use index for note backlink
This commit is contained in:
Gamosoft
parent
4b316b2cc7
commit
0cd5332a99
176
backend/main.py
176
backend/main.py
|
|
@ -46,7 +46,6 @@ from .utils import (
|
|||
get_backlinks,
|
||||
)
|
||||
from . import note_index
|
||||
from .note_index import USE_NOTE_INDEX
|
||||
from .plugins import PluginManager
|
||||
from .themes import get_available_themes, get_theme_css
|
||||
from .share import (
|
||||
|
|
@ -1486,186 +1485,15 @@ async def search(
|
|||
|
||||
@api_router.get("/graph", tags=["Graph"])
|
||||
async def get_graph():
|
||||
"""Get graph data for note visualization with wikilink and markdown link detection."""
|
||||
"""Graph data (nodes + resolved wikilink/markdown edges) for the visualizer."""
|
||||
try:
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
# Fast path: index already holds resolved edges. Ensure a scan has
|
||||
# populated it (first request after startup, or after an invalidate)
|
||||
# then ask the facade for a snapshot.
|
||||
if USE_NOTE_INDEX:
|
||||
if not note_index.get_index().is_built():
|
||||
scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
|
||||
indexed = note_index.try_graph_data()
|
||||
if indexed is not None:
|
||||
nodes_paths, edges_tuples = indexed
|
||||
nodes_paths, edges_tuples = note_index.get_graph_data()
|
||||
return {
|
||||
"nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths],
|
||||
"edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples],
|
||||
}
|
||||
|
||||
notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
|
||||
nodes = []
|
||||
edges = []
|
||||
|
||||
# Build set of valid note names/paths for matching
|
||||
note_paths = set()
|
||||
note_paths_lower = {} # Map lowercase path -> actual path for case-insensitive matching
|
||||
note_names = {} # Map name -> path for quick lookup
|
||||
|
||||
for note in notes:
|
||||
if note.get('type') == 'note':
|
||||
note_paths.add(note['path'])
|
||||
note_paths.add(note['path'].replace('.md', ''))
|
||||
# Store lowercase path -> actual path mapping for case-insensitive matching
|
||||
note_paths_lower[note['path'].lower()] = note['path']
|
||||
note_paths_lower[note['path'].replace('.md', '').lower()] = note['path']
|
||||
# Store name -> path mapping (without extension)
|
||||
name = note['name'].replace('.md', '')
|
||||
note_names[name.lower()] = note['path']
|
||||
note_names[note['name'].lower()] = note['path']
|
||||
|
||||
# Build graph structure with link detection
|
||||
for note in notes:
|
||||
if note.get('type') == 'note':
|
||||
nodes.append({
|
||||
"id": note['path'],
|
||||
"label": note['name'].replace('.md', '')
|
||||
})
|
||||
|
||||
# Read note content to find links
|
||||
content = get_note_content(config['storage']['notes_dir'], note['path'])
|
||||
if content:
|
||||
# Find wikilinks: [[target]] or [[target|display]]
|
||||
wikilinks = re.findall(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', content)
|
||||
|
||||
# Find standard markdown internal links: [text](path) - any local path (not http/https)
|
||||
# Match links that don't start with http://, https://, mailto:, #, etc.
|
||||
markdown_links = re.findall(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', content)
|
||||
|
||||
# Get source note's folder for resolving relative links
|
||||
# Use forward slashes consistently (note_paths uses forward slashes)
|
||||
source_folder = str(Path(note['path']).parent).replace('\\', '/')
|
||||
if source_folder == '.':
|
||||
source_folder = ''
|
||||
|
||||
# Process wikilinks
|
||||
for target in wikilinks:
|
||||
target = target.strip()
|
||||
target_lower = target.lower()
|
||||
|
||||
# Try to match target to an existing note
|
||||
target_path = None
|
||||
|
||||
# 1. Try resolving relative to source note's folder first
|
||||
if source_folder and '/' not in target:
|
||||
relative_path = f"{source_folder}/{target}"
|
||||
relative_path_lower = relative_path.lower()
|
||||
|
||||
if relative_path in note_paths:
|
||||
target_path = relative_path if relative_path.endswith('.md') else relative_path + '.md'
|
||||
elif relative_path + '.md' in note_paths:
|
||||
target_path = relative_path + '.md'
|
||||
elif relative_path_lower in note_paths_lower:
|
||||
target_path = note_paths_lower[relative_path_lower]
|
||||
elif relative_path_lower + '.md' in note_paths_lower:
|
||||
target_path = note_paths_lower[relative_path_lower + '.md']
|
||||
|
||||
# 2. Exact path match (absolute or already has folder)
|
||||
if not target_path:
|
||||
if target in note_paths:
|
||||
target_path = target if target.endswith('.md') else target + '.md'
|
||||
elif target + '.md' in note_paths:
|
||||
target_path = target + '.md'
|
||||
# 3. Case-insensitive path match (e.g., [[Folder/Note]] -> folder/note.md)
|
||||
elif target_lower in note_paths_lower:
|
||||
target_path = note_paths_lower[target_lower]
|
||||
elif target_lower + '.md' in note_paths_lower:
|
||||
target_path = note_paths_lower[target_lower + '.md']
|
||||
# 4. Just note name (case-insensitive) - global match
|
||||
elif target_lower in note_names:
|
||||
target_path = note_names[target_lower]
|
||||
|
||||
if target_path and target_path != note['path']:
|
||||
edges.append({
|
||||
"source": note['path'],
|
||||
"target": target_path,
|
||||
"type": "wikilink"
|
||||
})
|
||||
|
||||
# Process markdown links
|
||||
for _, link_path in markdown_links:
|
||||
# Skip anchor-only links and external protocols
|
||||
if not link_path or link_path.startswith('#'):
|
||||
continue
|
||||
|
||||
# Remove anchor part if present (e.g., "note.md#section" -> "note.md")
|
||||
link_path = link_path.split('#')[0]
|
||||
if not link_path:
|
||||
continue
|
||||
|
||||
# Normalize path: remove ./ prefix, handle URL encoding
|
||||
link_path = urllib.parse.unquote(link_path)
|
||||
if link_path.startswith('./'):
|
||||
link_path = link_path[2:]
|
||||
|
||||
# Add .md extension if not present and doesn't have other extension
|
||||
link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md'
|
||||
|
||||
# Try to match target to an existing note
|
||||
target_path = None
|
||||
|
||||
# 1. First, try resolving relative to source note's folder
|
||||
if source_folder and not link_path.startswith('/'):
|
||||
relative_path = f"{source_folder}/{link_path}"
|
||||
relative_path_with_md = f"{source_folder}/{link_path_with_md}"
|
||||
relative_path_lower = relative_path.lower()
|
||||
relative_path_with_md_lower = relative_path_with_md.lower()
|
||||
|
||||
if relative_path in note_paths:
|
||||
target_path = relative_path if relative_path.endswith('.md') else relative_path + '.md'
|
||||
elif relative_path_with_md in note_paths:
|
||||
target_path = relative_path_with_md
|
||||
elif relative_path_lower in note_paths_lower:
|
||||
target_path = note_paths_lower[relative_path_lower]
|
||||
elif relative_path_with_md_lower in note_paths_lower:
|
||||
target_path = note_paths_lower[relative_path_with_md_lower]
|
||||
|
||||
# 2. Try exact path match from root (for absolute paths or notes at root)
|
||||
if not target_path:
|
||||
link_path_lower = link_path.lower()
|
||||
link_path_with_md_lower = link_path_with_md.lower()
|
||||
|
||||
if link_path in note_paths:
|
||||
target_path = link_path if link_path.endswith('.md') else link_path + '.md'
|
||||
elif link_path_with_md in note_paths:
|
||||
target_path = link_path_with_md
|
||||
# Case-insensitive path match
|
||||
elif link_path_lower in note_paths_lower:
|
||||
target_path = note_paths_lower[link_path_lower]
|
||||
elif link_path_with_md_lower in note_paths_lower:
|
||||
target_path = note_paths_lower[link_path_with_md_lower]
|
||||
|
||||
# No global filename fallback for markdown links - they must resolve as paths
|
||||
|
||||
if target_path and target_path != note['path']:
|
||||
edges.append({
|
||||
"source": note['path'],
|
||||
"target": target_path,
|
||||
"type": "markdown"
|
||||
})
|
||||
|
||||
# Remove duplicate edges
|
||||
seen = set()
|
||||
unique_edges = []
|
||||
for edge in edges:
|
||||
key = (edge['source'], edge['target'])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique_edges.append(edge)
|
||||
|
||||
return {"nodes": nodes, "edges": unique_edges}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to generate graph data"))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,77 +1,12 @@
|
|||
"""
|
||||
NoteIndex — unified in-memory index of everything that's expensive to recompute
|
||||
from disk: links, tags, full-text search, and per-note metadata.
|
||||
NoteIndex — in-memory index of vault state: notes, folders, tags, links, search.
|
||||
|
||||
+---------------------------------------------------------------+
|
||||
| ONE module, ONE singleton, ONE lock, ONE rollback switch. |
|
||||
| All call sites in utils.py / main.py are one-line shims. |
|
||||
+---------------------------------------------------------------+
|
||||
Backs /api/notes, /api/tags, /api/backlinks, /api/graph, /api/search. Keeps
|
||||
everything thread-safe under one RLock. Process-memory only — rebuilds on the
|
||||
first /api/notes request after each process start (~1-3s on a 10K-note vault).
|
||||
|
||||
Why this exists
|
||||
---------------
|
||||
Without an index, every backlink lookup, every graph render, and every text
|
||||
search re-walks the entire vault and re-reads every file. That's O(N) per
|
||||
request and scales linearly with vault size — 4-second note loads on a 10K
|
||||
vault, 5-10s graph renders, multi-second searches. With an index, those
|
||||
endpoints become O(matches) — milliseconds regardless of vault size.
|
||||
|
||||
What's indexed
|
||||
--------------
|
||||
* Note metadata (path -> name, folder, mtime, size, type, tags)
|
||||
* Folders (set of folder paths)
|
||||
* Tags forward (path -> sorted tags)
|
||||
* Tags backward (tag -> set of paths)
|
||||
* Links forward (source_path -> {target_path: "wikilink"|"markdown"})
|
||||
* Links backward (target_path -> set of source paths) strict
|
||||
* Wikilink tokens (lowercased token -> set of source paths) loose
|
||||
* Search inverted (lowercased term -> set of paths)
|
||||
|
||||
What's NOT cached
|
||||
-----------------
|
||||
File content. Reading file content for line-level context (backlinks, search
|
||||
snippets) is done only for the small set of matched files, never for the
|
||||
whole vault.
|
||||
|
||||
Threading model
|
||||
---------------
|
||||
Everything mutating goes through one RLock. Reads return snapshot copies, so
|
||||
callers can iterate freely after they release the lock. The index is
|
||||
designed to be called from FastAPI request handlers concurrently.
|
||||
|
||||
Lifetime
|
||||
--------
|
||||
Process-memory only. No persistence to disk — keeping the app lightweight
|
||||
matters more than the ~500ms saved on cold restart for a 10K-note vault.
|
||||
The index rebuilds on the first `/api/notes` request after every restart
|
||||
(typically 1-3 seconds, dominated by reading every file once for tag/link
|
||||
extraction). F5 / browser reload doesn't touch this — only Python process
|
||||
restart does.
|
||||
|
||||
Rollback switch
|
||||
---------------
|
||||
USE_NOTE_INDEX (below) is the single, global on/off. Flip it to False and
|
||||
every facade function becomes a no-op or returns None, and every call site
|
||||
in utils.py / main.py falls through to the legacy file-scanning behavior.
|
||||
|
||||
ROLLBACK RECIPE (decide the index isn't worth it)
|
||||
-------------------------------------------------
|
||||
Set USE_NOTE_INDEX = False below. That's it. Every try_* facade returns
|
||||
None, every on_* facade becomes a no-op, callers fall through to the
|
||||
legacy file-scanning paths that are preserved as the function bodies.
|
||||
|
||||
COMMIT RECIPE (you're happy, drop the legacy paths)
|
||||
---------------------------------------------------
|
||||
1. Here: delete USE_NOTE_INDEX and inline `if not USE_NOTE_INDEX` to True
|
||||
in every try_/on_ facade.
|
||||
2. In backend/utils.py:
|
||||
- get_backlinks: drop the `candidates is None` branch — the
|
||||
index always provides them.
|
||||
- search_notes: drop the `candidates is None` fallback.
|
||||
- get_all_tags / get_notes_by_tag: drop the legacy aggregation
|
||||
tail block.
|
||||
3. In backend/main.py:
|
||||
- /api/graph: drop the legacy fallback block at the bottom of
|
||||
the endpoint, keep only `try_graph_data`.
|
||||
Updated incrementally on every save/delete/move/rename through the on_*
|
||||
facades at the bottom of this file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -87,40 +22,20 @@ from pathlib import Path
|
|||
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Configuration
|
||||
# ============================================================================
|
||||
|
||||
# Single global rollback switch. Flip to False to disable the index and every
|
||||
# call site in utils.py / main.py falls back to the legacy code paths.
|
||||
USE_NOTE_INDEX: bool = True
|
||||
|
||||
# Parallelism cutoff for full rescans. Below this many markdown files, threads
|
||||
# add more overhead than they save (small vaults stay sequential).
|
||||
# Below this many markdown files, parallel tag extraction adds more overhead
|
||||
# than it saves.
|
||||
_PARALLEL_CUTOFF = 50
|
||||
_PARALLEL_WORKERS = min(8, (os.cpu_count() or 4))
|
||||
|
||||
# Search tokenization. Lowercase, split on anything that's not a word char.
|
||||
# Min length 2 keeps single-letter noise out of the index without losing
|
||||
# common short queries like "go", "ai".
|
||||
# Search tokenization. Min length 2 keeps single-letter noise out without
|
||||
# losing common short queries like "go", "ai".
|
||||
_SEARCH_TOKEN_RE = re.compile(r"[A-Za-z0-9_\-]{2,}")
|
||||
_SEARCH_MIN_QUERY_LEN = 2
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Shared regexes (same as legacy get_backlinks / /api/graph). Kept here so the
|
||||
# index and the legacy paths are guaranteed to extract the same tokens.
|
||||
# ============================================================================
|
||||
|
||||
WIKILINK_RE = re.compile(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]')
|
||||
MDLINK_RE = re.compile(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)')
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Public extraction helpers — used by the index AND by lifecycle hooks in
|
||||
# utils.py (so a single save can update the index without re-reading the file).
|
||||
# ============================================================================
|
||||
|
||||
def extract_links_from_content(content: str) -> Dict[str, List[str]]:
|
||||
"""Pull raw wikilink targets and markdown-link paths out of note content."""
|
||||
wikilinks = [m.strip() for m in WIKILINK_RE.findall(content)]
|
||||
|
|
@ -129,68 +44,50 @@ def extract_links_from_content(content: str) -> Dict[str, List[str]]:
|
|||
|
||||
|
||||
def extract_search_terms(content: str) -> Set[str]:
|
||||
"""Tokenize content for the inverted search index. Lowercased, deduped."""
|
||||
"""Tokenize content for the inverted search index."""
|
||||
return {m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(content)}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Data record for one note's metadata snapshot
|
||||
# ============================================================================
|
||||
|
||||
@dataclass
|
||||
class NoteRecord:
|
||||
"""Snapshot of one note's metadata. Kept tiny — no content, no resolved
|
||||
links (those live in the inverted indexes)."""
|
||||
"""One note's metadata. No content, no resolved links."""
|
||||
path: str # vault-relative POSIX
|
||||
name: str # stem (no extension)
|
||||
folder: str # vault-relative POSIX, "" for root
|
||||
modified: str # ISO timestamp
|
||||
size: int # bytes
|
||||
type: str # "note" | "image" | "audio" | ...
|
||||
mtime: float # raw stat mtime, for staleness checks
|
||||
tags: Tuple[str, ...] = field(default_factory=tuple) # sorted, deduped
|
||||
mtime: float # raw stat mtime
|
||||
tags: Tuple[str, ...] = field(default_factory=tuple)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# The index itself
|
||||
# ============================================================================
|
||||
|
||||
class NoteIndex:
|
||||
"""Thread-safe in-memory index of vault state. Every read returns a
|
||||
snapshot copy so callers don't have to hold the lock."""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Construction
|
||||
# ------------------------------------------------------------------
|
||||
"""Thread-safe in-memory index of vault state. Reads return copies so
|
||||
callers don't have to hold the lock."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
|
||||
# Note metadata
|
||||
self._notes: Dict[str, NoteRecord] = {} # path -> record
|
||||
self._folders: Set[str] = set()
|
||||
|
||||
# Tag indexes
|
||||
self._tags_forward: Dict[str, Tuple[str, ...]] = {} # path -> tags
|
||||
self._tags_backward: Dict[str, Set[str]] = {} # tag -> paths
|
||||
|
||||
# Link indexes
|
||||
self._raw_links: Dict[str, Dict[str, List[str]]] = {} # path -> {"wikilinks":[], "mdlinks":[]}
|
||||
self._links_forward: Dict[str, Dict[str, str]] = {} # src -> {tgt: type}
|
||||
self._links_backward: Dict[str, Set[str]] = {} # tgt -> {srcs}
|
||||
self._wikilink_tokens: Dict[str, Set[str]] = {} # token -> {srcs} (loose)
|
||||
self._wikilink_tokens: Dict[str, Set[str]] = {} # lower token -> {srcs} (loose match)
|
||||
|
||||
# Full-text inverted index
|
||||
self._search_terms: Dict[str, Set[str]] = {} # term -> {paths}
|
||||
|
||||
# Build state. _built tracks the cheap part (notes/tags/links).
|
||||
# _search_built tracks the expensive search index separately — that
|
||||
# one is built lazily on first search request to keep startup fast.
|
||||
# _search_built tracked separately — the search index is built lazily
|
||||
# on the first /api/search call, after the cheaper notes/tags/links
|
||||
# part is already built.
|
||||
self._built = False
|
||||
self._search_built = False
|
||||
self._raw_fingerprint: Optional[int] = None # short-circuits no-op rebuilds
|
||||
|
||||
# Observability counters (lock-free reads OK; rough is fine)
|
||||
self._stats = {
|
||||
"build_count": 0,
|
||||
"last_build_ms": 0.0,
|
||||
|
|
@ -201,17 +98,12 @@ class NoteIndex:
|
|||
"last_search_build_ms": 0.0,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Status / lifecycle gates
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def is_built(self) -> bool:
|
||||
with self._lock:
|
||||
return self._built
|
||||
|
||||
def invalidate(self) -> None:
|
||||
"""Mark as needing rebuild on next scan. Used as a safety net when
|
||||
an operation's incremental update is too fiddly to track precisely."""
|
||||
"""Mark as needing rebuild on next scan."""
|
||||
with self._lock:
|
||||
self._built = False
|
||||
self._raw_fingerprint = None
|
||||
|
|
@ -236,25 +128,14 @@ class NoteIndex:
|
|||
return self._search_built
|
||||
|
||||
def ensure_search_index_built(self, notes_dir: str) -> bool:
|
||||
"""Build the full-text search index by reading every markdown file.
|
||||
|
||||
Cheap if already built (instant return). Called lazily by search_notes
|
||||
on the first search request so app startup stays fast. Subsequent
|
||||
searches reuse the built index.
|
||||
|
||||
Concurrency: while one thread is building, others can still read the
|
||||
(still-cold) index — they'll just take the legacy path until the
|
||||
build completes. Returns True if the index is built (now or already).
|
||||
"""
|
||||
# Cheap pre-check without lock.
|
||||
"""Build the search index on demand. Returns True when ready, False
|
||||
only if the main index isn't built yet. File I/O happens OUTSIDE the
|
||||
lock so other reads aren't blocked."""
|
||||
if self._search_built:
|
||||
return True
|
||||
|
||||
# Slow path: do the read+tokenize OUTSIDE the lock so other reads
|
||||
# aren't blocked. Snapshot the paths we need to read under the lock,
|
||||
# release, do I/O, then re-acquire to install the result.
|
||||
with self._lock:
|
||||
if self._search_built: # raced with another builder
|
||||
if self._search_built:
|
||||
return True
|
||||
if not self._built:
|
||||
return False
|
||||
|
|
@ -265,16 +146,11 @@ class NoteIndex:
|
|||
terms_per_path: Dict[str, Set[str]] = {}
|
||||
for rel in paths_to_read:
|
||||
try:
|
||||
full = base / rel
|
||||
with open(full, "r", encoding="utf-8") as f:
|
||||
with open(base / rel, "r", encoding="utf-8") as f:
|
||||
terms_per_path[rel] = extract_search_terms(f.read())
|
||||
except Exception:
|
||||
terms_per_path[rel] = set()
|
||||
|
||||
# Re-acquire and install. If state changed under us (vault was
|
||||
# heavily mutated during the build), still install — the index
|
||||
# will be slightly stale until the next note save, which is the
|
||||
# same liveness guarantee as the per-save update path.
|
||||
with self._lock:
|
||||
if self._search_built:
|
||||
return True
|
||||
|
|
@ -287,24 +163,15 @@ class NoteIndex:
|
|||
self._stats["last_search_build_ms"] = (time.perf_counter() - t0) * 1000
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Bulk population — called from a full scan_notes_fast_walk
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def bulk_set(
|
||||
self,
|
||||
notes_meta: List[NoteRecord],
|
||||
folders: Iterable[str],
|
||||
sources_raw: Dict[str, Dict[str, List[str]]],
|
||||
) -> None:
|
||||
"""Replace the entire index atomically. Builds notes/tags/links
|
||||
(cheap on top of an already-completed scan). The search index is
|
||||
NOT touched here — it's built lazily on the first search request
|
||||
(see ensure_search_index_built) so app startup stays fast.
|
||||
|
||||
Fast-paths when the input fingerprints to the same state we already
|
||||
hold (typical warm scan where nothing changed in the vault).
|
||||
"""
|
||||
"""Replace the entire index from a fresh scan. Short-circuits when
|
||||
the input fingerprints to the same state we already hold (warm
|
||||
scan, nothing changed)."""
|
||||
new_fp = _fingerprint(notes_meta, sources_raw)
|
||||
with self._lock:
|
||||
if self._built and self._raw_fingerprint == new_fp:
|
||||
|
|
@ -319,8 +186,6 @@ class NoteIndex:
|
|||
|
||||
self._rebuild_tags_unlocked()
|
||||
self._rebuild_links_unlocked()
|
||||
# Search index: if it was built and still references known notes,
|
||||
# prune stale entries; full rebuild is deferred until a search.
|
||||
if self._search_built:
|
||||
self._prune_search_unlocked()
|
||||
|
||||
|
|
@ -331,19 +196,13 @@ class NoteIndex:
|
|||
self._stats["last_build_ms"] = (time.perf_counter() - t0) * 1000
|
||||
self._stats["last_built_at"] = datetime.now(tz=timezone.utc).isoformat()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Incremental updates — called from save/delete/rename handlers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def update_note(
|
||||
self,
|
||||
record: NoteRecord,
|
||||
raw_links: Dict[str, List[str]],
|
||||
content: Optional[str] = None,
|
||||
) -> None:
|
||||
"""A single note's content changed (or was just created). Patches the
|
||||
index in place: re-extracts its links, updates the tag indexes, and
|
||||
if `content` is provided, refreshes its search-term entries."""
|
||||
"""Patch the index in place after a note is created or saved."""
|
||||
with self._lock:
|
||||
old_record = self._notes.get(record.path)
|
||||
old_tags = old_record.tags if old_record else ()
|
||||
|
|
@ -352,7 +211,6 @@ class NoteIndex:
|
|||
if record.folder:
|
||||
self._folders.add(record.folder)
|
||||
|
||||
# Tags: diff old vs new and patch the backward index.
|
||||
new_tags = record.tags
|
||||
if old_tags != new_tags:
|
||||
for t in set(old_tags) - set(new_tags):
|
||||
|
|
@ -365,13 +223,11 @@ class NoteIndex:
|
|||
self._tags_backward.setdefault(t, set()).add(record.path)
|
||||
self._tags_forward[record.path] = new_tags
|
||||
|
||||
# Links: re-resolve this one source against the current vault.
|
||||
self._raw_links[record.path] = raw_links
|
||||
self._resolve_single_source_unlocked(record.path)
|
||||
|
||||
# Search: only patch the search index if it's already been built.
|
||||
# Otherwise we'd be doing work for an index nobody is using yet
|
||||
# (the first /api/search call will build it from disk).
|
||||
# Search index only gets patched if it's already been built —
|
||||
# otherwise the first /api/search will build it from scratch.
|
||||
if content is not None and self._search_built:
|
||||
self._update_search_for_note_unlocked(record.path, content)
|
||||
|
||||
|
|
@ -437,11 +293,9 @@ class NoteIndex:
|
|||
self._stats["incremental_updates"] += 1
|
||||
|
||||
def rename_note(self, old_path: str, new_path: str) -> None:
|
||||
"""A note was renamed/moved. Move all references from old_path to
|
||||
new_path in every index. Because the new path can change the source's
|
||||
own relative-folder resolution AND can change how other notes resolve
|
||||
their wikilinks to it (different parent folder, different stem),
|
||||
the simplest correct path is to migrate raw state then re-resolve."""
|
||||
"""Move all references from old_path to new_path. The path change can
|
||||
affect how other notes' wikilinks resolve, so we wipe the resolved
|
||||
link indexes for this note and rely on the next bulk_set to rebuild."""
|
||||
if old_path == new_path:
|
||||
return
|
||||
with self._lock:
|
||||
|
|
@ -465,7 +319,6 @@ class NoteIndex:
|
|||
if new_record.folder:
|
||||
self._folders.add(new_record.folder)
|
||||
|
||||
# Tags forward + backward — swap the key.
|
||||
if old_path in self._tags_forward:
|
||||
tags = self._tags_forward.pop(old_path)
|
||||
self._tags_forward[new_path] = tags
|
||||
|
|
@ -475,24 +328,17 @@ class NoteIndex:
|
|||
bucket.discard(old_path)
|
||||
bucket.add(new_path)
|
||||
|
||||
# Search: swap path in every term bucket. Bounded by terms-per-note.
|
||||
for paths in self._search_terms.values():
|
||||
if old_path in paths:
|
||||
paths.discard(old_path)
|
||||
paths.add(new_path)
|
||||
|
||||
# Raw links: migrate.
|
||||
if old_path in self._raw_links:
|
||||
self._raw_links[new_path] = self._raw_links.pop(old_path)
|
||||
|
||||
# Strict link indexes: drop both old src and old target everywhere,
|
||||
# then re-resolve. Other sources that linked TO old_path by name
|
||||
# may now resolve to new_path (or not), so we need a re-resolve.
|
||||
#
|
||||
# Implementation: drop forward/backward for old_path, drop old
|
||||
# entries in other sources' forwards that pointed to old_path,
|
||||
# invalidate the index. The next scan rebuilds (which we trigger
|
||||
# implicitly because the caller flips _built = False below).
|
||||
# Other sources that linked to old_path by stem name may now
|
||||
# resolve to new_path (or not), so we wipe both forward & backward
|
||||
# for old_path and let the next bulk_set re-resolve from scratch.
|
||||
self._links_forward.pop(old_path, None)
|
||||
for bucket in self._links_backward.values():
|
||||
bucket.discard(old_path)
|
||||
|
|
@ -504,7 +350,6 @@ class NoteIndex:
|
|||
if not fwd:
|
||||
del self._links_forward[src]
|
||||
|
||||
# Loose wikilink index: replace path values.
|
||||
empty_keys = []
|
||||
for key, sources in self._wikilink_tokens.items():
|
||||
if old_path in sources:
|
||||
|
|
@ -516,17 +361,12 @@ class NoteIndex:
|
|||
del self._wikilink_tokens[k]
|
||||
|
||||
self._raw_fingerprint = None
|
||||
self._built = False # force re-resolve on next bulk_set/scan
|
||||
self._built = False # force re-resolve on next bulk_set
|
||||
self._stats["incremental_updates"] += 1
|
||||
|
||||
def rename_folder_prefix(self, old_prefix: str, new_prefix: str) -> None:
|
||||
"""A folder was renamed/moved. Migrate every entry whose path starts
|
||||
with `old_prefix/` to `new_prefix/`. Much cheaper than a full rebuild:
|
||||
for a 1000-note folder rename, this is microseconds of key swaps
|
||||
vs ~400ms of disk re-scan.
|
||||
|
||||
Both prefixes are normalized to forward slashes, no trailing slash.
|
||||
"""
|
||||
"""Migrate every entry under `old_prefix/` to `new_prefix/`. Much
|
||||
cheaper than a full rebuild — microseconds of key swaps."""
|
||||
old_prefix = old_prefix.rstrip("/")
|
||||
new_prefix = new_prefix.rstrip("/")
|
||||
if old_prefix == new_prefix:
|
||||
|
|
@ -534,14 +374,9 @@ class NoteIndex:
|
|||
with self._lock:
|
||||
affected_paths = [p for p in self._notes if p == old_prefix or p.startswith(old_prefix + "/")]
|
||||
for old_path in affected_paths:
|
||||
suffix = old_path[len(old_prefix):] # includes leading "/" or nothing
|
||||
new_path = new_prefix + suffix
|
||||
# Reuse rename_note's heavy lifting — already correct, just
|
||||
# called many times. Acceptable for folder operations.
|
||||
# (Inlining would be a perf gain but multiplies bug surface.)
|
||||
self._rename_note_unlocked(old_path, new_path)
|
||||
suffix = old_path[len(old_prefix):]
|
||||
self._rename_note_unlocked(old_path, new_prefix + suffix)
|
||||
|
||||
# Migrate the folder set too.
|
||||
folders_to_rename = [
|
||||
f for f in self._folders if f == old_prefix or f.startswith(old_prefix + "/")
|
||||
]
|
||||
|
|
@ -554,7 +389,7 @@ class NoteIndex:
|
|||
self._built = False
|
||||
|
||||
def remove_folder_prefix(self, prefix: str) -> None:
|
||||
"""A folder was deleted. Drop everything under it."""
|
||||
"""Drop every entry under `prefix/`."""
|
||||
prefix = prefix.rstrip("/")
|
||||
with self._lock:
|
||||
affected = [p for p in self._notes if p == prefix or p.startswith(prefix + "/")]
|
||||
|
|
@ -597,7 +432,7 @@ class NoteIndex:
|
|||
return nodes, edges
|
||||
|
||||
def get_all_tags(self) -> Dict[str, int]:
|
||||
"""Snapshot: {tag: count}, sorted by tag name."""
|
||||
"""Snapshot {tag: count}, sorted by tag name."""
|
||||
with self._lock:
|
||||
return {tag: len(paths) for tag, paths in sorted(self._tags_backward.items())}
|
||||
|
||||
|
|
@ -610,23 +445,19 @@ class NoteIndex:
|
|||
with self._lock:
|
||||
return self._notes.get(path)
|
||||
|
||||
def all_note_records(self) -> List[Tuple[str, NoteRecord]]:
|
||||
"""Snapshot list of (path, record) for every indexed markdown note."""
|
||||
with self._lock:
|
||||
return [(p, r) for p, r in self._notes.items() if r.type == "note"]
|
||||
|
||||
def try_get_extraction(
|
||||
self,
|
||||
rel_path: str,
|
||||
mtime: float,
|
||||
) -> Optional[Tuple[List[str], Dict[str, List[str]]]]:
|
||||
"""Return (tags, raw_links) from the index when fresh.
|
||||
|
||||
The caller passes the file's current mtime; we hand back the cached
|
||||
extraction only when the recorded mtime matches exactly. Lets
|
||||
scan_notes_fast_walk skip the per-file read on a snapshot-warm
|
||||
startup — the bulk of cold-load latency on large vaults.
|
||||
|
||||
Returns None when:
|
||||
- The note isn't in the index (new file)
|
||||
- mtime differs (file changed since snapshot)
|
||||
- We've somehow lost the raw_links entry (defensive)
|
||||
"""
|
||||
"""Return (tags, raw_links) from the index only if the recorded mtime
|
||||
matches `mtime` exactly. Lets scan_notes_fast_walk skip the per-file
|
||||
read on a warm scan."""
|
||||
with self._lock:
|
||||
rec = self._notes.get(rel_path)
|
||||
if rec is None or rec.mtime != mtime:
|
||||
|
|
@ -643,34 +474,18 @@ class NoteIndex:
|
|||
)
|
||||
|
||||
def get_search_candidates(self, query: str) -> Optional[Set[str]]:
|
||||
# Index can only narrow candidates if it's been built.
|
||||
"""Superset of paths whose content COULD contain `query`. Caller
|
||||
still runs the substring match per candidate for confirmation +
|
||||
snippet extraction. Returns None when the query is too short or
|
||||
tokenizes to nothing — caller should iterate every note instead."""
|
||||
if not self._search_built:
|
||||
return None
|
||||
"""Return the set of paths that COULD contain `query` as a substring,
|
||||
based on the inverted term index.
|
||||
|
||||
Returns None when the query is too short to use the index (caller
|
||||
should fall through to the legacy full-scan). When the query
|
||||
tokenizes to nothing useful (all stopword-like noise), returns the
|
||||
set of every indexed path so the caller still does a substring
|
||||
check on each (no false negatives).
|
||||
|
||||
IMPORTANT: this is a SUPERSET — the caller must still run the
|
||||
substring match per candidate to confirm and extract context.
|
||||
Token-AND can include docs that have both tokens but not as the
|
||||
adjacent substring the user searched for.
|
||||
"""
|
||||
if len(query) < _SEARCH_MIN_QUERY_LEN:
|
||||
return None
|
||||
tokens = [m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(query)]
|
||||
if not tokens:
|
||||
# Query has no tokenizable parts (pure punctuation, single char,
|
||||
# etc.). Index can't help; fall through.
|
||||
return None
|
||||
with self._lock:
|
||||
# Intersection of all token buckets. Empty set means no doc
|
||||
# contains ALL of the tokens (and therefore none contain the
|
||||
# query as substring either).
|
||||
candidate = self._search_terms.get(tokens[0])
|
||||
if candidate is None:
|
||||
return set()
|
||||
|
|
@ -685,11 +500,9 @@ class NoteIndex:
|
|||
return result
|
||||
|
||||
def stats(self) -> Dict[str, Any]:
|
||||
"""Snapshot of internal counters + size metrics. Cheap to call —
|
||||
no traversal beyond `len()` on the top-level dicts."""
|
||||
"""Snapshot of counters + size metrics."""
|
||||
with self._lock:
|
||||
return {
|
||||
"enabled": USE_NOTE_INDEX,
|
||||
"built": self._built,
|
||||
"search_built": self._search_built,
|
||||
"notes": len(self._notes),
|
||||
|
|
@ -720,11 +533,8 @@ class NoteIndex:
|
|||
self._tags_backward.setdefault(tag, set()).add(path)
|
||||
|
||||
def _rebuild_links_unlocked(self) -> None:
|
||||
"""Full link re-resolution. Builds a shared Resolver lookup once
|
||||
(O(N)) and reuses it for every source (O(K) each) — total O(N+K*L)
|
||||
rather than the O(N*K) you'd get by building a fresh Resolver per
|
||||
source.
|
||||
"""
|
||||
"""Full link re-resolution. Reuses one _Resolver across all sources
|
||||
(O(N+K*L) instead of O(N*K))."""
|
||||
self._links_forward.clear()
|
||||
self._links_backward.clear()
|
||||
self._wikilink_tokens.clear()
|
||||
|
|
@ -734,9 +544,7 @@ class NoteIndex:
|
|||
self._resolve_single_source_unlocked(source_path, resolver=resolver, skip_cleanup=True)
|
||||
|
||||
def _prune_search_unlocked(self) -> None:
|
||||
"""Drop search-term entries for paths no longer in the index. Cheap
|
||||
compared to a full rebuild — only touches terms whose bucket still
|
||||
references a now-deleted path."""
|
||||
"""Drop search-term entries for paths no longer in the index."""
|
||||
live_paths = set(self._notes.keys())
|
||||
empty_terms = []
|
||||
for term, paths in self._search_terms.items():
|
||||
|
|
@ -750,7 +558,6 @@ class NoteIndex:
|
|||
|
||||
def _update_search_for_note_unlocked(self, path: str, content: str) -> None:
|
||||
"""Replace one note's terms in the inverted index."""
|
||||
# Drop old entries for this path.
|
||||
empty_terms = []
|
||||
for term, paths in self._search_terms.items():
|
||||
if path in paths:
|
||||
|
|
@ -759,7 +566,6 @@ class NoteIndex:
|
|||
empty_terms.append(term)
|
||||
for term in empty_terms:
|
||||
del self._search_terms[term]
|
||||
# Add new entries.
|
||||
for term in extract_search_terms(content):
|
||||
self._search_terms.setdefault(term, set()).add(path)
|
||||
|
||||
|
|
@ -799,16 +605,13 @@ class NoteIndex:
|
|||
|
||||
targets: Dict[str, str] = {}
|
||||
|
||||
# Wikilinks first so they win the "first wins" dedup that legacy
|
||||
# /api/graph also implements.
|
||||
# Wikilinks first (they win the "first wins" dedup that /api/graph uses).
|
||||
for target in raw.get("wikilinks", []):
|
||||
resolved = resolver.resolve_wikilink(target, source_folder)
|
||||
if resolved and resolved != source_path and resolved not in targets:
|
||||
targets[resolved] = "wikilink"
|
||||
# Loose wikilink reverse index — populated regardless of strict
|
||||
# resolution success, because the legacy get_backlinks uses
|
||||
# token-stem matching independent of where the link actually
|
||||
# navigates.
|
||||
# Loose token index — populated even when strict resolution fails,
|
||||
# because backlink matching uses stem comparison independently.
|
||||
t_lower = target.strip().lower()
|
||||
if t_lower:
|
||||
self._wikilink_tokens.setdefault(t_lower, set()).add(source_path)
|
||||
|
|
@ -827,8 +630,8 @@ class NoteIndex:
|
|||
self._links_backward.setdefault(t, set()).add(source_path)
|
||||
|
||||
def _rename_note_unlocked(self, old_path: str, new_path: str) -> None:
|
||||
"""Same as rename_note() but assumes the caller already holds the lock.
|
||||
Used by folder-prefix rename to avoid re-acquiring the lock per file."""
|
||||
"""rename_note() body without acquiring the lock — used by
|
||||
rename_folder_prefix to avoid re-locking per file."""
|
||||
if old_path == new_path:
|
||||
return
|
||||
old_record = self._notes.pop(old_path, None)
|
||||
|
|
@ -936,9 +739,8 @@ class NoteIndex:
|
|||
# ============================================================================
|
||||
|
||||
class _Resolver:
|
||||
"""Build the lookup tables once per resolution batch, then call
|
||||
resolve_* repeatedly. Reused across sources within a single rebuild so
|
||||
we don't pay O(N) to construct it on every source."""
|
||||
"""Link-target lookup tables. Build once per resolution batch, then call
|
||||
resolve_* repeatedly across many sources."""
|
||||
|
||||
def __init__(self, all_notes: Set[str]) -> None:
|
||||
self.note_paths: Set[str] = set(all_notes)
|
||||
|
|
@ -1029,8 +831,7 @@ def _fingerprint(
|
|||
notes_meta: List[NoteRecord],
|
||||
sources_raw: Dict[str, Dict[str, List[str]]],
|
||||
) -> int:
|
||||
"""Cheap content hash of a scan result. Used to short-circuit bulk_set
|
||||
when nothing has changed in the vault."""
|
||||
"""Cheap hash of a scan result. Short-circuits bulk_set when unchanged."""
|
||||
notes_fp = hash(frozenset((n.path, n.mtime) for n in notes_meta))
|
||||
raw_items = (
|
||||
(src, tuple(raw.get("wikilinks", [])), tuple(raw.get("mdlinks", [])))
|
||||
|
|
@ -1040,11 +841,7 @@ def _fingerprint(
|
|||
|
||||
|
||||
# ============================================================================
|
||||
# Module-level singleton + facade
|
||||
#
|
||||
# Every utils.py / main.py call site uses these — never instantiates its own
|
||||
# index. The facade functions are no-ops (or return None) when
|
||||
# USE_NOTE_INDEX is False, so call sites don't have to repeat the flag check.
|
||||
# Module singleton + facade. Callers in utils.py / main.py go through these.
|
||||
# ============================================================================
|
||||
|
||||
_index = NoteIndex()
|
||||
|
|
@ -1054,12 +851,10 @@ def get_index() -> NoteIndex:
|
|||
return _index
|
||||
|
||||
|
||||
# --- Lifecycle facade (one-line calls from utils.py mutators) ----------------
|
||||
# --- Lifecycle hooks (one-line calls from utils.py mutators) ----------------
|
||||
|
||||
def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None:
|
||||
"""A note was created or updated on disk. Patch the index in place."""
|
||||
if not USE_NOTE_INDEX:
|
||||
return
|
||||
"""A note was created or updated on disk."""
|
||||
try:
|
||||
rel_path = full_path.relative_to(Path(notes_dir)).as_posix()
|
||||
st = full_path.stat()
|
||||
|
|
@ -1074,15 +869,12 @@ def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None:
|
|||
mtime=st.st_mtime,
|
||||
tags=tuple(_parse_tags_for_record(content)),
|
||||
)
|
||||
raw_links = extract_links_from_content(content)
|
||||
_index.update_note(record, raw_links, content=content)
|
||||
_index.update_note(record, extract_links_from_content(content), content=content)
|
||||
except Exception as e:
|
||||
print(f"note_index: on_note_saved failed for {full_path}: {e}")
|
||||
|
||||
|
||||
def on_note_deleted(notes_dir: str, full_path: Path) -> None:
|
||||
if not USE_NOTE_INDEX:
|
||||
return
|
||||
try:
|
||||
rel_path = full_path.relative_to(Path(notes_dir)).as_posix()
|
||||
_index.remove_note(rel_path)
|
||||
|
|
@ -1091,8 +883,6 @@ def on_note_deleted(notes_dir: str, full_path: Path) -> None:
|
|||
|
||||
|
||||
def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None:
|
||||
if not USE_NOTE_INDEX:
|
||||
return
|
||||
try:
|
||||
base = Path(notes_dir)
|
||||
_index.rename_note(
|
||||
|
|
@ -1104,10 +894,7 @@ def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) ->
|
|||
|
||||
|
||||
def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None:
|
||||
"""A folder was moved/renamed. Re-keys every entry under it (cheap,
|
||||
no disk reads) rather than invalidating the whole index."""
|
||||
if not USE_NOTE_INDEX:
|
||||
return
|
||||
"""Re-key every entry under the folder. No disk reads."""
|
||||
try:
|
||||
base = Path(notes_dir)
|
||||
_index.rename_folder_prefix(
|
||||
|
|
@ -1116,12 +903,10 @@ def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path)
|
|||
)
|
||||
except Exception as e:
|
||||
print(f"note_index: on_folder_renamed failed: {e}")
|
||||
_index.invalidate() # fail-safe
|
||||
_index.invalidate()
|
||||
|
||||
|
||||
def on_folder_deleted(notes_dir: str, full_path: Path) -> None:
|
||||
if not USE_NOTE_INDEX:
|
||||
return
|
||||
try:
|
||||
rel_prefix = full_path.relative_to(Path(notes_dir)).as_posix()
|
||||
_index.remove_folder_prefix(rel_prefix)
|
||||
|
|
@ -1135,9 +920,7 @@ def populate_from_scan(
|
|||
folders: Iterable[str],
|
||||
sources_raw: Dict[str, Dict[str, List[str]]],
|
||||
) -> None:
|
||||
"""Bulk-replace the index from a fresh scan. No-op when off."""
|
||||
if not USE_NOTE_INDEX:
|
||||
return
|
||||
"""Bulk-replace the index after a full scan_notes_fast_walk."""
|
||||
try:
|
||||
_index.bulk_set(notes_meta, folders, sources_raw)
|
||||
except Exception as e:
|
||||
|
|
@ -1145,44 +928,34 @@ def populate_from_scan(
|
|||
|
||||
|
||||
def ensure_search_index(notes_dir: str) -> bool:
|
||||
"""Lazily build the full-text search index on first /api/search request.
|
||||
Cheap after the first call. Returns True if the index is now usable."""
|
||||
if not USE_NOTE_INDEX:
|
||||
return False
|
||||
"""Lazy-build the search index on first /api/search. Returns False only
|
||||
if the main index isn't built yet."""
|
||||
if not _index.is_built():
|
||||
return False
|
||||
return _index.ensure_search_index_built(notes_dir)
|
||||
|
||||
|
||||
# --- Read facade (returns None when off — callers fall through to legacy) ----
|
||||
# --- Read facade -------------------------------------------------------------
|
||||
|
||||
def try_backlink_candidates(target_path: str) -> Optional[Set[str]]:
|
||||
if not USE_NOTE_INDEX or not _index.is_built():
|
||||
return None
|
||||
def get_backlink_candidates(target_path: str) -> Set[str]:
|
||||
return _index.get_backlink_candidate_sources(target_path)
|
||||
|
||||
|
||||
def try_graph_data() -> Optional[Tuple[List[str], List[Tuple[str, str, str]]]]:
|
||||
if not USE_NOTE_INDEX or not _index.is_built():
|
||||
return None
|
||||
def get_graph_data() -> Tuple[List[str], List[Tuple[str, str, str]]]:
|
||||
return _index.get_graph_data()
|
||||
|
||||
|
||||
def try_search_candidates(query: str) -> Optional[Set[str]]:
|
||||
if not USE_NOTE_INDEX or not _index.is_built():
|
||||
return None
|
||||
def get_search_candidates(query: str) -> Optional[Set[str]]:
|
||||
"""Returns None when query is too short / untokenizable — caller iterates
|
||||
every indexed note instead."""
|
||||
return _index.get_search_candidates(query)
|
||||
|
||||
|
||||
def try_all_tags() -> Optional[Dict[str, int]]:
|
||||
if not USE_NOTE_INDEX or not _index.is_built():
|
||||
return None
|
||||
def get_all_tags() -> Dict[str, int]:
|
||||
return _index.get_all_tags()
|
||||
|
||||
|
||||
def try_notes_by_tag(tag: str) -> Optional[Set[str]]:
|
||||
if not USE_NOTE_INDEX or not _index.is_built():
|
||||
return None
|
||||
def get_paths_for_tag(tag: str) -> Set[str]:
|
||||
return _index.get_paths_for_tag(tag)
|
||||
|
||||
|
||||
|
|
@ -1190,30 +963,16 @@ def try_get_extraction(
|
|||
rel_path: str,
|
||||
mtime: float,
|
||||
) -> Optional[Tuple[List[str], Dict[str, List[str]]]]:
|
||||
"""Serve (tags, raw_links) from the index for a single file, when fresh.
|
||||
|
||||
Used by scan_notes_fast_walk to skip the cold per-file read after a
|
||||
snapshot load. Returns None when the index can't help (off, not built,
|
||||
file unknown, mtime mismatch) — caller falls back to reading the file.
|
||||
"""
|
||||
if not USE_NOTE_INDEX or not _index.is_built():
|
||||
return None
|
||||
"""(tags, raw_links) from the index when mtime matches, else None so the
|
||||
caller reads the file. Used by scan_notes_fast_walk."""
|
||||
return _index.try_get_extraction(rel_path, mtime)
|
||||
|
||||
|
||||
# --- Observability -----------------------------------------------------------
|
||||
|
||||
def stats() -> Dict[str, Any]:
|
||||
return _index.stats()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Late import to avoid a circular dependency with utils.parse_tags.
|
||||
# We need tag parsing inside on_note_saved but utils.py imports this module.
|
||||
# ============================================================================
|
||||
|
||||
# Late import — utils imports this module, so we delay the reverse direction.
|
||||
def _parse_tags_for_record(content: str) -> List[str]:
|
||||
"""Thin shim to utils.parse_tags. Late-bound so this module imports cleanly
|
||||
before utils.py is loaded."""
|
||||
from .utils import parse_tags
|
||||
return parse_tags(content)
|
||||
|
|
|
|||
162
backend/utils.py
162
backend/utils.py
|
|
@ -2,8 +2,7 @@
|
|||
Utility functions for file operations, search, and markdown processing.
|
||||
|
||||
Heavy work (backlinks, graph, full-text search, tag aggregation) is delegated
|
||||
to the in-memory index in note_index.py — every facade function there is a
|
||||
no-op when USE_NOTE_INDEX is False, so call sites here stay flag-free.
|
||||
to the in-memory index in note_index.py.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
|
@ -163,27 +162,25 @@ def _scan_cache_set(key: Tuple[str, bool], value: Tuple[List[Dict], List[str]])
|
|||
|
||||
|
||||
def _scan_cache_invalidate() -> None:
|
||||
"""Drop the TTL scan cache. Called from every mutation handler so callers
|
||||
that hit the cache immediately after a save/delete/move see the new state
|
||||
instead of stale data from the previous walk."""
|
||||
"""Drop the TTL scan cache. Called from every mutation handler."""
|
||||
with _SCAN_WALK_CACHE_LOCK:
|
||||
_SCAN_WALK_CACHE.clear()
|
||||
|
||||
|
||||
def _ensure_index_built(notes_dir: str) -> None:
|
||||
"""Trigger a fresh scan when the NoteIndex hasn't been populated yet.
|
||||
Idempotent: the scan short-circuits when nothing changed in the vault."""
|
||||
if not note_index.get_index().is_built():
|
||||
scan_notes_fast_walk(notes_dir, use_cache=False, include_media=False)
|
||||
|
||||
|
||||
def get_tags_and_links_cached(
|
||||
file_path: Path,
|
||||
rel_path: Optional[str] = None,
|
||||
) -> Tuple[List[str], Dict[str, List[str]]]:
|
||||
"""Single-read fused extraction of tags + raw links, cached by mtime.
|
||||
|
||||
Three-tier lookup, cheapest first:
|
||||
1. In-process per-file caches (warm scans on the live server)
|
||||
2. NoteIndex (warm after snapshot load — avoids file reads on cold start)
|
||||
3. Read the file from disk
|
||||
|
||||
rel_path enables tier 2. scan_notes_fast_walk passes it; ad-hoc callers
|
||||
can omit it and just get tiers 1 + 3.
|
||||
"""
|
||||
"""Fused tags + raw-links extraction, mtime-cached. Tries in-process
|
||||
per-file caches first, then the NoteIndex (when rel_path is provided),
|
||||
then reads the file."""
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime
|
||||
file_key = str(file_path)
|
||||
|
|
@ -196,8 +193,6 @@ def get_tags_and_links_cached(
|
|||
if tags_ok and links_ok:
|
||||
return tag_cached[1], link_cached[1]
|
||||
|
||||
# Try the NoteIndex before reading the file. On a snapshot-warm
|
||||
# startup, this turns a 10K-file cold scan into a no-I/O walk.
|
||||
if rel_path is not None:
|
||||
indexed = note_index.try_get_extraction(rel_path, mtime)
|
||||
if indexed is not None:
|
||||
|
|
@ -293,11 +288,11 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
|
|||
_scan_cache_set(cache_key, normalized_value)
|
||||
return normalized_value
|
||||
|
||||
# Walk the vault once. Tag/link extraction is deferred out of the walk
|
||||
# (the expensive part is the file I/O) and parallelized below across files.
|
||||
# Walk the vault once. Tag/link extraction is parallelized below across
|
||||
# the markdown files we collected here.
|
||||
notes: List[Dict] = []
|
||||
folders_set = set()
|
||||
md_to_extract: List[Tuple[int, Path, str]] = [] # (note_index, full_path, rel_path)
|
||||
md_to_extract: List[Tuple[int, Path, str]] = [] # (index, full_path, rel_path)
|
||||
|
||||
for root, dirnames, filenames in os.walk(notes_path):
|
||||
dirnames[:] = [d for d in dirnames if not d.startswith('.')]
|
||||
|
|
@ -339,19 +334,10 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
|
|||
if is_markdown:
|
||||
md_to_extract.append((len(notes) - 1, full_path, rel_str))
|
||||
|
||||
# Fused tag + raw-link extraction. mtime-keyed cache makes warm scans
|
||||
# mostly free; cold scans win from parallel I/O.
|
||||
#
|
||||
# We don't read full content here — that would double the parsing cost
|
||||
# for every scan, and the search index is the only thing that needs it.
|
||||
# Search is built lazily on the first search request (see
|
||||
# note_index.ensure_search_index_built), which keeps startup fast and
|
||||
# only pays the cost when a user actually searches.
|
||||
# Tag + raw-link extraction in parallel. Search-index terms are built
|
||||
# later, lazily, on the first /api/search call.
|
||||
sources_raw: Dict[str, Dict[str, List[str]]] = {}
|
||||
if md_to_extract:
|
||||
# Pass rel_path so each call can hit the NoteIndex before falling
|
||||
# back to a file read — critical for fast startup after a snapshot
|
||||
# load on large vaults.
|
||||
path_pairs = [(full, rel) for (_, full, rel) in md_to_extract]
|
||||
if len(md_to_extract) >= 50:
|
||||
workers = min(8, (os.cpu_count() or 4))
|
||||
|
|
@ -367,8 +353,7 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
|
|||
notes[idx]["tags"] = tags
|
||||
sources_raw[rel_str] = links
|
||||
|
||||
# Populate the unified index. Cheap when the fingerprint matches (warm
|
||||
# scan, nothing changed) — short-circuits internally. No-op when off.
|
||||
# Push the result into the NoteIndex. Short-circuits on warm scans.
|
||||
notes_meta = [
|
||||
NoteRecord(
|
||||
path=n["path"],
|
||||
|
|
@ -555,27 +540,28 @@ def delete_note(notes_dir: str, note_path: str) -> bool:
|
|||
|
||||
|
||||
def search_notes(notes_dir: str, query: str) -> List[Dict]:
|
||||
"""Full-text search through note contents.
|
||||
|
||||
Only searches inside note content (not filenames or paths). The index, when
|
||||
available, narrows down the candidate files via a token-AND match on its
|
||||
inverted index — then the existing per-line snippet extraction runs on the
|
||||
narrow set. Output is byte-identical to a full scan.
|
||||
"""
|
||||
"""Full-text search through note contents. Narrow the candidate set via
|
||||
the inverted index, then run the snippet extractor on each candidate."""
|
||||
from html import escape
|
||||
results: List[Dict] = []
|
||||
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
|
||||
|
||||
# Build the search index lazily on the first search request. Subsequent
|
||||
# searches hit the warm index; the first one pays the cost (read every
|
||||
# file once) and is roughly as slow as the legacy full scan would be.
|
||||
_ensure_index_built(notes_dir)
|
||||
note_index.ensure_search_index(notes_dir)
|
||||
candidates = note_index.try_search_candidates(query)
|
||||
candidates = note_index.get_search_candidates(query)
|
||||
|
||||
for note in notes:
|
||||
idx = note_index.get_index()
|
||||
if candidates is None:
|
||||
# Query too short to tokenize — iterate every indexed note instead.
|
||||
candidate_records = idx.all_note_records()
|
||||
else:
|
||||
candidate_records = [(p, idx.get_note_record(p)) for p in candidates]
|
||||
candidate_records = [(p, r) for (p, r) in candidate_records if r is not None and r.type == "note"]
|
||||
|
||||
candidate_records.sort(key=lambda pr: pr[1].mtime, reverse=True)
|
||||
iterable = [{"path": p, "name": r.name} for (p, r) in candidate_records]
|
||||
|
||||
for note in iterable:
|
||||
path = note["path"]
|
||||
if candidates is not None and path not in candidates:
|
||||
continue
|
||||
md_file = Path(notes_dir) / path
|
||||
try:
|
||||
with open(md_file, 'r', encoding='utf-8') as f:
|
||||
|
|
@ -888,32 +874,16 @@ def parse_tags(content: str) -> List[str]:
|
|||
|
||||
|
||||
def get_all_tags(notes_dir: str) -> Dict[str, int]:
|
||||
"""All tags in the vault with note counts. Index-backed when available."""
|
||||
# Fast path FIRST — when the index is ready it knows every tag count,
|
||||
# so we skip the scan entirely (no disk walk, no JSON of 10K notes).
|
||||
indexed = note_index.try_all_tags()
|
||||
if indexed is not None:
|
||||
return indexed
|
||||
|
||||
# Legacy fallback: scan the vault and aggregate.
|
||||
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
|
||||
tag_counts: Dict[str, int] = {}
|
||||
for note in notes:
|
||||
for tag in note.get("tags", []):
|
||||
tag_counts[tag] = tag_counts.get(tag, 0) + 1
|
||||
return dict(sorted(tag_counts.items()))
|
||||
"""All tags in the vault with note counts."""
|
||||
_ensure_index_built(notes_dir)
|
||||
return note_index.get_all_tags()
|
||||
|
||||
|
||||
def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]:
|
||||
"""All notes carrying `tag` (case-insensitive). Index-backed when available."""
|
||||
tag_lower = tag.lower()
|
||||
|
||||
# Fast path FIRST: every field we need lives in the NoteRecord. Skip
|
||||
# the vault scan when the index can answer directly.
|
||||
indexed_paths = note_index.try_notes_by_tag(tag_lower)
|
||||
if indexed_paths is not None:
|
||||
"""All notes carrying `tag` (case-insensitive)."""
|
||||
_ensure_index_built(notes_dir)
|
||||
idx = note_index.get_index()
|
||||
records = [idx.get_note_record(p) for p in indexed_paths]
|
||||
records = [idx.get_note_record(p) for p in note_index.get_paths_for_tag(tag.lower())]
|
||||
matching = [
|
||||
{
|
||||
"name": r.name,
|
||||
|
|
@ -926,28 +896,9 @@ def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]:
|
|||
for r in records
|
||||
if r is not None and r.type == "note"
|
||||
]
|
||||
# Match legacy sort order (scan returns notes sorted by modified desc).
|
||||
matching.sort(key=lambda n: n["modified"], reverse=True)
|
||||
return matching
|
||||
|
||||
# Legacy fallback.
|
||||
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
|
||||
matching: List[Dict] = []
|
||||
for note in notes:
|
||||
if note.get("type") != "note":
|
||||
continue
|
||||
tags = note.get("tags", [])
|
||||
if tag_lower in tags:
|
||||
matching.append({
|
||||
"name": note["name"],
|
||||
"path": note["path"],
|
||||
"folder": note["folder"],
|
||||
"modified": note["modified"],
|
||||
"size": note["size"],
|
||||
"tags": tags,
|
||||
})
|
||||
return matching
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Template Functions
|
||||
|
|
@ -1189,13 +1140,8 @@ def _extract_backlink_references(
|
|||
|
||||
|
||||
def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
|
||||
"""Find all notes that link TO the specified note.
|
||||
|
||||
Index-backed when available: the index hands us a superset of candidate
|
||||
source files (O(1)), we read only those, then the per-line matcher
|
||||
filters down to true matches with line-level context. Output is
|
||||
byte-identical to a full vault scan.
|
||||
"""
|
||||
"""All notes that link TO `target_note_path`. The index narrows the
|
||||
candidate set; we only read the candidate files for line context."""
|
||||
target_path = target_note_path
|
||||
target_path_lower = target_path.lower()
|
||||
target_path_no_ext_lower = target_path_lower.replace('.md', '')
|
||||
|
|
@ -1205,22 +1151,15 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
|
|||
Path(target_path).stem.lower(),
|
||||
}
|
||||
|
||||
# scan_notes_fast_walk is TTL-cached, so calling it here is a dict-lookup
|
||||
# in the warm case. We need it for the canonical ordering of `notes`.
|
||||
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
|
||||
|
||||
candidates = note_index.try_backlink_candidates(target_path)
|
||||
_ensure_index_built(notes_dir)
|
||||
idx = note_index.get_index()
|
||||
candidates = note_index.get_backlink_candidates(target_path)
|
||||
records = [(p, idx.get_note_record(p)) for p in candidates]
|
||||
records = [(p, r) for (p, r) in records if r is not None and r.type == "note" and p != target_path]
|
||||
records.sort(key=lambda pr: pr[1].mtime, reverse=True)
|
||||
|
||||
backlinks: List[Dict] = []
|
||||
for note in notes:
|
||||
if note.get('type') != 'note':
|
||||
continue
|
||||
source_path = note['path']
|
||||
if source_path == target_path:
|
||||
continue
|
||||
if candidates is not None and source_path not in candidates:
|
||||
continue
|
||||
|
||||
for source_path, record in records:
|
||||
refs = _extract_backlink_references(
|
||||
notes_dir,
|
||||
source_path,
|
||||
|
|
@ -1232,9 +1171,8 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
|
|||
if refs:
|
||||
backlinks.append({
|
||||
"path": source_path,
|
||||
"name": note['name'].replace('.md', ''),
|
||||
"name": record.name.replace('.md', ''),
|
||||
"references": refs[:3],
|
||||
})
|
||||
|
||||
return backlinks
|
||||
|
||||
|
|
|
|||
|
|
@ -591,7 +591,6 @@ The index is rebuilt on the first scan after each process start and updated incr
|
|||
**Response:**
|
||||
```json
|
||||
{
|
||||
"enabled": true,
|
||||
"built": true,
|
||||
"search_built": false,
|
||||
"notes": 142,
|
||||
|
|
@ -615,7 +614,6 @@ The index is rebuilt on the first scan after each process start and updated incr
|
|||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `enabled` | Whether the index is active (always `true` in current builds) |
|
||||
| `built` | `true` after the first vault scan completes |
|
||||
| `search_built` | `true` after the first `/api/search` request (search index is built lazily) |
|
||||
| `notes` | Number of note records currently held in the index |
|
||||
|
|
|
|||
Loading…
Reference in New Issue