diff --git a/backend/main.py b/backend/main.py index 72781ff..bf823ef 100644 --- a/backend/main.py +++ b/backend/main.py @@ -45,6 +45,8 @@ from .utils import ( paginate, get_backlinks, ) +from . import note_index +from .note_index import USE_NOTE_INDEX from .plugins import PluginManager from .themes import get_available_themes, get_theme_css from .share import ( @@ -265,6 +267,7 @@ plugin_manager = PluginManager(config['storage']['plugins_dir']) # Run app startup hooks plugin_manager.run_hook('on_app_startup') + # Mount static files static_path = Path(__file__).parent.parent / "frontend" app.mount("/static", StaticFiles(directory=static_path), name="static") @@ -1483,10 +1486,25 @@ async def search( @api_router.get("/graph", tags=["Graph"]) async def get_graph(): - """Get graph data for note visualization with wikilink and markdown link detection""" + """Get graph data for note visualization with wikilink and markdown link detection.""" try: import re import urllib.parse + + # Fast path: index already holds resolved edges. Ensure a scan has + # populated it (first request after startup, or after an invalidate) + # then ask the facade for a snapshot. + if USE_NOTE_INDEX: + if not note_index.get_index().is_built(): + scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) + indexed = note_index.try_graph_data() + if indexed is not None: + nodes_paths, edges_tuples = indexed + return { + "nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths], + "edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples], + } + notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) nodes = [] edges = [] @@ -1692,6 +1710,19 @@ async def toggle_plugin(request: Request, plugin_name: str, enabled: dict): raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to toggle plugin")) +# ============================================================================ +# Index observability — internal counters & sizes for debugging +# +# Useful for confirming the index is doing what we expect (build_count >0, +# search_terms not empty, fingerprint_short_circuits incrementing on idle +# warm scans). Not rate-limited — cheap, no I/O, dict lookups only. +# ============================================================================ + +@api_router.get("/index/stats", tags=["System"]) +async def get_index_stats(): + return note_index.stats() + + # ============================================================================ # Stats Endpoint (for dashboards) # ============================================================================ diff --git a/backend/note_index.py b/backend/note_index.py new file mode 100644 index 0000000..5a0d7eb --- /dev/null +++ b/backend/note_index.py @@ -0,0 +1,1219 @@ +""" +NoteIndex — unified in-memory index of everything that's expensive to recompute +from disk: links, tags, full-text search, and per-note metadata. + + +---------------------------------------------------------------+ + | ONE module, ONE singleton, ONE lock, ONE rollback switch. | + | All call sites in utils.py / main.py are one-line shims. | + +---------------------------------------------------------------+ + +Why this exists +--------------- +Without an index, every backlink lookup, every graph render, and every text +search re-walks the entire vault and re-reads every file. That's O(N) per +request and scales linearly with vault size — 4-second note loads on a 10K +vault, 5-10s graph renders, multi-second searches. With an index, those +endpoints become O(matches) — milliseconds regardless of vault size. + +What's indexed +-------------- + * Note metadata (path -> name, folder, mtime, size, type, tags) + * Folders (set of folder paths) + * Tags forward (path -> sorted tags) + * Tags backward (tag -> set of paths) + * Links forward (source_path -> {target_path: "wikilink"|"markdown"}) + * Links backward (target_path -> set of source paths) strict + * Wikilink tokens (lowercased token -> set of source paths) loose + * Search inverted (lowercased term -> set of paths) + +What's NOT cached +----------------- +File content. Reading file content for line-level context (backlinks, search +snippets) is done only for the small set of matched files, never for the +whole vault. + +Threading model +--------------- +Everything mutating goes through one RLock. Reads return snapshot copies, so +callers can iterate freely after they release the lock. The index is +designed to be called from FastAPI request handlers concurrently. + +Lifetime +-------- +Process-memory only. No persistence to disk — keeping the app lightweight +matters more than the ~500ms saved on cold restart for a 10K-note vault. +The index rebuilds on the first `/api/notes` request after every restart +(typically 1-3 seconds, dominated by reading every file once for tag/link +extraction). F5 / browser reload doesn't touch this — only Python process +restart does. + +Rollback switch +--------------- +USE_NOTE_INDEX (below) is the single, global on/off. Flip it to False and +every facade function becomes a no-op or returns None, and every call site +in utils.py / main.py falls through to the legacy file-scanning behavior. + + ROLLBACK RECIPE (decide the index isn't worth it) + ------------------------------------------------- + Set USE_NOTE_INDEX = False below. That's it. Every try_* facade returns + None, every on_* facade becomes a no-op, callers fall through to the + legacy file-scanning paths that are preserved as the function bodies. + + COMMIT RECIPE (you're happy, drop the legacy paths) + --------------------------------------------------- + 1. Here: delete USE_NOTE_INDEX and inline `if not USE_NOTE_INDEX` to True + in every try_/on_ facade. + 2. In backend/utils.py: + - get_backlinks: drop the `candidates is None` branch — the + index always provides them. + - search_notes: drop the `candidates is None` fallback. + - get_all_tags / get_notes_by_tag: drop the legacy aggregation + tail block. + 3. In backend/main.py: + - /api/graph: drop the legacy fallback block at the bottom of + the endpoint, keep only `try_graph_data`. +""" + +from __future__ import annotations + +import os +import re +import threading +import time +import urllib.parse +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + + +# ============================================================================ +# Configuration +# ============================================================================ + +# Single global rollback switch. Flip to False to disable the index and every +# call site in utils.py / main.py falls back to the legacy code paths. +USE_NOTE_INDEX: bool = True + +# Parallelism cutoff for full rescans. Below this many markdown files, threads +# add more overhead than they save (small vaults stay sequential). +_PARALLEL_CUTOFF = 50 +_PARALLEL_WORKERS = min(8, (os.cpu_count() or 4)) + +# Search tokenization. Lowercase, split on anything that's not a word char. +# Min length 2 keeps single-letter noise out of the index without losing +# common short queries like "go", "ai". +_SEARCH_TOKEN_RE = re.compile(r"[A-Za-z0-9_\-]{2,}") +_SEARCH_MIN_QUERY_LEN = 2 + + +# ============================================================================ +# Shared regexes (same as legacy get_backlinks / /api/graph). Kept here so the +# index and the legacy paths are guaranteed to extract the same tokens. +# ============================================================================ + +WIKILINK_RE = re.compile(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]') +MDLINK_RE = re.compile(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)') + + +# ============================================================================ +# Public extraction helpers — used by the index AND by lifecycle hooks in +# utils.py (so a single save can update the index without re-reading the file). +# ============================================================================ + +def extract_links_from_content(content: str) -> Dict[str, List[str]]: + """Pull raw wikilink targets and markdown-link paths out of note content.""" + wikilinks = [m.strip() for m in WIKILINK_RE.findall(content)] + mdlinks = [link_path for _, link_path in MDLINK_RE.findall(content)] + return {"wikilinks": wikilinks, "mdlinks": mdlinks} + + +def extract_search_terms(content: str) -> Set[str]: + """Tokenize content for the inverted search index. Lowercased, deduped.""" + return {m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(content)} + + +# ============================================================================ +# Data record for one note's metadata snapshot +# ============================================================================ + +@dataclass +class NoteRecord: + """Snapshot of one note's metadata. Kept tiny — no content, no resolved + links (those live in the inverted indexes).""" + path: str # vault-relative POSIX + name: str # stem (no extension) + folder: str # vault-relative POSIX, "" for root + modified: str # ISO timestamp + size: int # bytes + type: str # "note" | "image" | "audio" | ... + mtime: float # raw stat mtime, for staleness checks + tags: Tuple[str, ...] = field(default_factory=tuple) # sorted, deduped + + +# ============================================================================ +# The index itself +# ============================================================================ + +class NoteIndex: + """Thread-safe in-memory index of vault state. Every read returns a + snapshot copy so callers don't have to hold the lock.""" + + # ------------------------------------------------------------------ + # Construction + # ------------------------------------------------------------------ + + def __init__(self) -> None: + self._lock = threading.RLock() + + # Note metadata + self._notes: Dict[str, NoteRecord] = {} # path -> record + self._folders: Set[str] = set() + + # Tag indexes + self._tags_forward: Dict[str, Tuple[str, ...]] = {} # path -> tags + self._tags_backward: Dict[str, Set[str]] = {} # tag -> paths + + # Link indexes + self._raw_links: Dict[str, Dict[str, List[str]]] = {} # path -> {"wikilinks":[], "mdlinks":[]} + self._links_forward: Dict[str, Dict[str, str]] = {} # src -> {tgt: type} + self._links_backward: Dict[str, Set[str]] = {} # tgt -> {srcs} + self._wikilink_tokens: Dict[str, Set[str]] = {} # token -> {srcs} (loose) + + # Full-text inverted index + self._search_terms: Dict[str, Set[str]] = {} # term -> {paths} + + # Build state. _built tracks the cheap part (notes/tags/links). + # _search_built tracks the expensive search index separately — that + # one is built lazily on first search request to keep startup fast. + self._built = False + self._search_built = False + self._raw_fingerprint: Optional[int] = None # short-circuits no-op rebuilds + + # Observability counters (lock-free reads OK; rough is fine) + self._stats = { + "build_count": 0, + "last_build_ms": 0.0, + "last_built_at": None, + "incremental_updates": 0, + "fingerprint_short_circuits": 0, + "search_build_count": 0, + "last_search_build_ms": 0.0, + } + + # ------------------------------------------------------------------ + # Status / lifecycle gates + # ------------------------------------------------------------------ + + def is_built(self) -> bool: + with self._lock: + return self._built + + def invalidate(self) -> None: + """Mark as needing rebuild on next scan. Used as a safety net when + an operation's incremental update is too fiddly to track precisely.""" + with self._lock: + self._built = False + self._raw_fingerprint = None + + def reset(self) -> None: + with self._lock: + self._notes.clear() + self._folders.clear() + self._tags_forward.clear() + self._tags_backward.clear() + self._raw_links.clear() + self._links_forward.clear() + self._links_backward.clear() + self._wikilink_tokens.clear() + self._search_terms.clear() + self._built = False + self._search_built = False + self._raw_fingerprint = None + + def is_search_built(self) -> bool: + with self._lock: + return self._search_built + + def ensure_search_index_built(self, notes_dir: str) -> bool: + """Build the full-text search index by reading every markdown file. + + Cheap if already built (instant return). Called lazily by search_notes + on the first search request so app startup stays fast. Subsequent + searches reuse the built index. + + Concurrency: while one thread is building, others can still read the + (still-cold) index — they'll just take the legacy path until the + build completes. Returns True if the index is built (now or already). + """ + # Cheap pre-check without lock. + if self._search_built: + return True + + # Slow path: do the read+tokenize OUTSIDE the lock so other reads + # aren't blocked. Snapshot the paths we need to read under the lock, + # release, do I/O, then re-acquire to install the result. + with self._lock: + if self._search_built: # raced with another builder + return True + if not self._built: + return False + paths_to_read = [p for p, r in self._notes.items() if r.type == "note"] + + t0 = time.perf_counter() + base = Path(notes_dir) + terms_per_path: Dict[str, Set[str]] = {} + for rel in paths_to_read: + try: + full = base / rel + with open(full, "r", encoding="utf-8") as f: + terms_per_path[rel] = extract_search_terms(f.read()) + except Exception: + terms_per_path[rel] = set() + + # Re-acquire and install. If state changed under us (vault was + # heavily mutated during the build), still install — the index + # will be slightly stale until the next note save, which is the + # same liveness guarantee as the per-save update path. + with self._lock: + if self._search_built: + return True + self._search_terms.clear() + for path, terms in terms_per_path.items(): + for term in terms: + self._search_terms.setdefault(term, set()).add(path) + self._search_built = True + self._stats["search_build_count"] += 1 + self._stats["last_search_build_ms"] = (time.perf_counter() - t0) * 1000 + return True + + # ------------------------------------------------------------------ + # Bulk population — called from a full scan_notes_fast_walk + # ------------------------------------------------------------------ + + def bulk_set( + self, + notes_meta: List[NoteRecord], + folders: Iterable[str], + sources_raw: Dict[str, Dict[str, List[str]]], + ) -> None: + """Replace the entire index atomically. Builds notes/tags/links + (cheap on top of an already-completed scan). The search index is + NOT touched here — it's built lazily on the first search request + (see ensure_search_index_built) so app startup stays fast. + + Fast-paths when the input fingerprints to the same state we already + hold (typical warm scan where nothing changed in the vault). + """ + new_fp = _fingerprint(notes_meta, sources_raw) + with self._lock: + if self._built and self._raw_fingerprint == new_fp: + self._stats["fingerprint_short_circuits"] += 1 + return + + t0 = time.perf_counter() + + self._notes = {n.path: n for n in notes_meta} + self._folders = set(folders) + self._raw_links = {k: v for k, v in sources_raw.items()} + + self._rebuild_tags_unlocked() + self._rebuild_links_unlocked() + # Search index: if it was built and still references known notes, + # prune stale entries; full rebuild is deferred until a search. + if self._search_built: + self._prune_search_unlocked() + + self._raw_fingerprint = new_fp + self._built = True + + self._stats["build_count"] += 1 + self._stats["last_build_ms"] = (time.perf_counter() - t0) * 1000 + self._stats["last_built_at"] = datetime.now(tz=timezone.utc).isoformat() + + # ------------------------------------------------------------------ + # Incremental updates — called from save/delete/rename handlers + # ------------------------------------------------------------------ + + def update_note( + self, + record: NoteRecord, + raw_links: Dict[str, List[str]], + content: Optional[str] = None, + ) -> None: + """A single note's content changed (or was just created). Patches the + index in place: re-extracts its links, updates the tag indexes, and + if `content` is provided, refreshes its search-term entries.""" + with self._lock: + old_record = self._notes.get(record.path) + old_tags = old_record.tags if old_record else () + + self._notes[record.path] = record + if record.folder: + self._folders.add(record.folder) + + # Tags: diff old vs new and patch the backward index. + new_tags = record.tags + if old_tags != new_tags: + for t in set(old_tags) - set(new_tags): + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(record.path) + if not bucket: + del self._tags_backward[t] + for t in set(new_tags) - set(old_tags): + self._tags_backward.setdefault(t, set()).add(record.path) + self._tags_forward[record.path] = new_tags + + # Links: re-resolve this one source against the current vault. + self._raw_links[record.path] = raw_links + self._resolve_single_source_unlocked(record.path) + + # Search: only patch the search index if it's already been built. + # Otherwise we'd be doing work for an index nobody is using yet + # (the first /api/search call will build it from disk). + if content is not None and self._search_built: + self._update_search_for_note_unlocked(record.path, content) + + self._raw_fingerprint = None + self._stats["incremental_updates"] += 1 + + def remove_note(self, path: str) -> None: + """A note was deleted. Drop everything that mentions it.""" + with self._lock: + old_record = self._notes.pop(path, None) + if old_record is None: + return + + # Tags + for t in old_record.tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._tags_backward[t] + self._tags_forward.pop(path, None) + + # Links: drop as source. + self._raw_links.pop(path, None) + old_targets = self._links_forward.pop(path, {}) + for t in old_targets: + bucket = self._links_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._links_backward[t] + + # Links: drop as loose wikilink source. + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + sources.discard(path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + # Links: drop as target from every other source's forward dict. + sources_pointing_here = self._links_backward.pop(path, set()) + for src in sources_pointing_here: + fwd = self._links_forward.get(src) + if fwd is not None and path in fwd: + del fwd[path] + if not fwd: + del self._links_forward[src] + + # Search: drop from every term bucket. Linear in distinct terms + # this note contributed to, which is bounded by note size. + empty_terms = [] + for term, paths in self._search_terms.items(): + if path in paths: + paths.discard(path) + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + + self._raw_fingerprint = None + self._stats["incremental_updates"] += 1 + + def rename_note(self, old_path: str, new_path: str) -> None: + """A note was renamed/moved. Move all references from old_path to + new_path in every index. Because the new path can change the source's + own relative-folder resolution AND can change how other notes resolve + their wikilinks to it (different parent folder, different stem), + the simplest correct path is to migrate raw state then re-resolve.""" + if old_path == new_path: + return + with self._lock: + old_record = self._notes.pop(old_path, None) + if old_record is None: + return + new_record = NoteRecord( + path=new_path, + name=Path(new_path).stem, + folder=str(Path(new_path).parent).replace("\\", "/").lstrip(".").lstrip("/") or "", + modified=old_record.modified, + size=old_record.size, + type=old_record.type, + mtime=old_record.mtime, + tags=old_record.tags, + ) + # Re-derive folder cleanly (the above lstrip dance is brittle). + folder = str(Path(new_path).parent).replace("\\", "/") + new_record.folder = "" if folder == "." else folder + self._notes[new_path] = new_record + if new_record.folder: + self._folders.add(new_record.folder) + + # Tags forward + backward — swap the key. + if old_path in self._tags_forward: + tags = self._tags_forward.pop(old_path) + self._tags_forward[new_path] = tags + for t in tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(old_path) + bucket.add(new_path) + + # Search: swap path in every term bucket. Bounded by terms-per-note. + for paths in self._search_terms.values(): + if old_path in paths: + paths.discard(old_path) + paths.add(new_path) + + # Raw links: migrate. + if old_path in self._raw_links: + self._raw_links[new_path] = self._raw_links.pop(old_path) + + # Strict link indexes: drop both old src and old target everywhere, + # then re-resolve. Other sources that linked TO old_path by name + # may now resolve to new_path (or not), so we need a re-resolve. + # + # Implementation: drop forward/backward for old_path, drop old + # entries in other sources' forwards that pointed to old_path, + # invalidate the index. The next scan rebuilds (which we trigger + # implicitly because the caller flips _built = False below). + self._links_forward.pop(old_path, None) + for bucket in self._links_backward.values(): + bucket.discard(old_path) + old_backlinks = self._links_backward.pop(old_path, set()) + for src in old_backlinks: + fwd = self._links_forward.get(src) + if fwd is not None and old_path in fwd: + del fwd[old_path] + if not fwd: + del self._links_forward[src] + + # Loose wikilink index: replace path values. + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + if old_path in sources: + sources.discard(old_path) + sources.add(new_path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + self._raw_fingerprint = None + self._built = False # force re-resolve on next bulk_set/scan + self._stats["incremental_updates"] += 1 + + def rename_folder_prefix(self, old_prefix: str, new_prefix: str) -> None: + """A folder was renamed/moved. Migrate every entry whose path starts + with `old_prefix/` to `new_prefix/`. Much cheaper than a full rebuild: + for a 1000-note folder rename, this is microseconds of key swaps + vs ~400ms of disk re-scan. + + Both prefixes are normalized to forward slashes, no trailing slash. + """ + old_prefix = old_prefix.rstrip("/") + new_prefix = new_prefix.rstrip("/") + if old_prefix == new_prefix: + return + with self._lock: + affected_paths = [p for p in self._notes if p == old_prefix or p.startswith(old_prefix + "/")] + for old_path in affected_paths: + suffix = old_path[len(old_prefix):] # includes leading "/" or nothing + new_path = new_prefix + suffix + # Reuse rename_note's heavy lifting — already correct, just + # called many times. Acceptable for folder operations. + # (Inlining would be a perf gain but multiplies bug surface.) + self._rename_note_unlocked(old_path, new_path) + + # Migrate the folder set too. + folders_to_rename = [ + f for f in self._folders if f == old_prefix or f.startswith(old_prefix + "/") + ] + for f in folders_to_rename: + self._folders.discard(f) + suffix = f[len(old_prefix):] + self._folders.add(new_prefix + suffix) + + self._raw_fingerprint = None + self._built = False + + def remove_folder_prefix(self, prefix: str) -> None: + """A folder was deleted. Drop everything under it.""" + prefix = prefix.rstrip("/") + with self._lock: + affected = [p for p in self._notes if p == prefix or p.startswith(prefix + "/")] + for path in affected: + self._remove_note_unlocked(path) + folders_to_drop = [ + f for f in self._folders if f == prefix or f.startswith(prefix + "/") + ] + for f in folders_to_drop: + self._folders.discard(f) + self._raw_fingerprint = None + + # ------------------------------------------------------------------ + # Read API — every method returns a snapshot copy + # ------------------------------------------------------------------ + + def get_backlink_candidate_sources(self, target_path: str) -> Set[str]: + """Superset of true backlink sources. Combines strict resolved + backward links with the loose wikilink-token reverse index. Caller + runs the per-line matcher against each candidate to filter.""" + target_lower = target_path.lower() + target_no_ext_lower = target_lower[:-3] if target_lower.endswith(".md") else target_lower + target_name = Path(target_path).stem.lower() + + with self._lock: + candidates: Set[str] = set(self._links_backward.get(target_path, set())) + for key in (target_lower, target_no_ext_lower, target_name): + candidates.update(self._wikilink_tokens.get(key, set())) + candidates.discard(target_path) + return candidates + + def get_graph_data(self) -> Tuple[List[str], List[Tuple[str, str, str]]]: + """Snapshot of (sorted note paths, (source, target, type) edges).""" + with self._lock: + nodes = sorted(p for p, r in self._notes.items() if r.type == "note") + edges: List[Tuple[str, str, str]] = [] + for src, targets in self._links_forward.items(): + for target, edge_type in targets.items(): + edges.append((src, target, edge_type)) + return nodes, edges + + def get_all_tags(self) -> Dict[str, int]: + """Snapshot: {tag: count}, sorted by tag name.""" + with self._lock: + return {tag: len(paths) for tag, paths in sorted(self._tags_backward.items())} + + def get_paths_for_tag(self, tag: str) -> Set[str]: + """Snapshot set of paths tagged with `tag` (case-insensitive).""" + with self._lock: + return set(self._tags_backward.get(tag.lower(), set())) + + def get_note_record(self, path: str) -> Optional[NoteRecord]: + with self._lock: + return self._notes.get(path) + + def try_get_extraction( + self, + rel_path: str, + mtime: float, + ) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: + """Return (tags, raw_links) from the index when fresh. + + The caller passes the file's current mtime; we hand back the cached + extraction only when the recorded mtime matches exactly. Lets + scan_notes_fast_walk skip the per-file read on a snapshot-warm + startup — the bulk of cold-load latency on large vaults. + + Returns None when: + - The note isn't in the index (new file) + - mtime differs (file changed since snapshot) + - We've somehow lost the raw_links entry (defensive) + """ + with self._lock: + rec = self._notes.get(rel_path) + if rec is None or rec.mtime != mtime: + return None + raw = self._raw_links.get(rel_path) + if raw is None: + return None + return ( + list(rec.tags), + { + "wikilinks": list(raw.get("wikilinks", [])), + "mdlinks": list(raw.get("mdlinks", [])), + }, + ) + + def get_search_candidates(self, query: str) -> Optional[Set[str]]: + # Index can only narrow candidates if it's been built. + if not self._search_built: + return None + """Return the set of paths that COULD contain `query` as a substring, + based on the inverted term index. + + Returns None when the query is too short to use the index (caller + should fall through to the legacy full-scan). When the query + tokenizes to nothing useful (all stopword-like noise), returns the + set of every indexed path so the caller still does a substring + check on each (no false negatives). + + IMPORTANT: this is a SUPERSET — the caller must still run the + substring match per candidate to confirm and extract context. + Token-AND can include docs that have both tokens but not as the + adjacent substring the user searched for. + """ + if len(query) < _SEARCH_MIN_QUERY_LEN: + return None + tokens = [m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(query)] + if not tokens: + # Query has no tokenizable parts (pure punctuation, single char, + # etc.). Index can't help; fall through. + return None + with self._lock: + # Intersection of all token buckets. Empty set means no doc + # contains ALL of the tokens (and therefore none contain the + # query as substring either). + candidate = self._search_terms.get(tokens[0]) + if candidate is None: + return set() + result: Set[str] = set(candidate) + for tok in tokens[1:]: + bucket = self._search_terms.get(tok) + if bucket is None: + return set() + result &= bucket + if not result: + break + return result + + def stats(self) -> Dict[str, Any]: + """Snapshot of internal counters + size metrics. Cheap to call — + no traversal beyond `len()` on the top-level dicts.""" + with self._lock: + return { + "enabled": USE_NOTE_INDEX, + "built": self._built, + "search_built": self._search_built, + "notes": len(self._notes), + "folders": len(self._folders), + "tags": len(self._tags_backward), + "links_forward_entries": len(self._links_forward), + "links_backward_entries": len(self._links_backward), + "wikilink_tokens": len(self._wikilink_tokens), + "search_terms": len(self._search_terms), + "counters": dict(self._stats), + } + + # ================================================================== + # Internal — must hold _lock when called + # ================================================================== + + def _rebuild_tags_unlocked(self) -> None: + self._tags_forward.clear() + self._tags_backward.clear() + for path, record in self._notes.items(): + if record.type != "note": + continue + tags = record.tags + if not tags: + continue + self._tags_forward[path] = tags + for tag in tags: + self._tags_backward.setdefault(tag, set()).add(path) + + def _rebuild_links_unlocked(self) -> None: + """Full link re-resolution. Builds a shared Resolver lookup once + (O(N)) and reuses it for every source (O(K) each) — total O(N+K*L) + rather than the O(N*K) you'd get by building a fresh Resolver per + source. + """ + self._links_forward.clear() + self._links_backward.clear() + self._wikilink_tokens.clear() + note_paths = {p for p, r in self._notes.items() if r.type == "note"} + resolver = _Resolver(note_paths) + for source_path in self._raw_links: + self._resolve_single_source_unlocked(source_path, resolver=resolver, skip_cleanup=True) + + def _prune_search_unlocked(self) -> None: + """Drop search-term entries for paths no longer in the index. Cheap + compared to a full rebuild — only touches terms whose bucket still + references a now-deleted path.""" + live_paths = set(self._notes.keys()) + empty_terms = [] + for term, paths in self._search_terms.items(): + stale = paths - live_paths + if stale: + paths -= stale + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + + def _update_search_for_note_unlocked(self, path: str, content: str) -> None: + """Replace one note's terms in the inverted index.""" + # Drop old entries for this path. + empty_terms = [] + for term, paths in self._search_terms.items(): + if path in paths: + paths.discard(path) + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + # Add new entries. + for term in extract_search_terms(content): + self._search_terms.setdefault(term, set()).add(path) + + def _resolve_single_source_unlocked( + self, + source_path: str, + resolver: Optional["_Resolver"] = None, + skip_cleanup: bool = False, + ) -> None: + if not skip_cleanup: + old_targets = self._links_forward.pop(source_path, {}) + for t in old_targets: + bucket = self._links_backward.get(t) + if bucket is not None: + bucket.discard(source_path) + if not bucket: + del self._links_backward[t] + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + sources.discard(source_path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + raw = self._raw_links.get(source_path) + if not raw: + return + + if resolver is None: + note_paths = {p for p, r in self._notes.items() if r.type == "note"} + resolver = _Resolver(note_paths) + + source_folder = str(Path(source_path).parent).replace("\\", "/") + if source_folder == ".": + source_folder = "" + + targets: Dict[str, str] = {} + + # Wikilinks first so they win the "first wins" dedup that legacy + # /api/graph also implements. + for target in raw.get("wikilinks", []): + resolved = resolver.resolve_wikilink(target, source_folder) + if resolved and resolved != source_path and resolved not in targets: + targets[resolved] = "wikilink" + # Loose wikilink reverse index — populated regardless of strict + # resolution success, because the legacy get_backlinks uses + # token-stem matching independent of where the link actually + # navigates. + t_lower = target.strip().lower() + if t_lower: + self._wikilink_tokens.setdefault(t_lower, set()).add(source_path) + t_no_ext = t_lower[:-3] if t_lower.endswith(".md") else t_lower + if t_no_ext != t_lower: + self._wikilink_tokens.setdefault(t_no_ext, set()).add(source_path) + + for link_path in raw.get("mdlinks", []): + resolved = resolver.resolve_mdlink(link_path, source_folder) + if resolved and resolved != source_path and resolved not in targets: + targets[resolved] = "markdown" + + if targets: + self._links_forward[source_path] = targets + for t in targets: + self._links_backward.setdefault(t, set()).add(source_path) + + def _rename_note_unlocked(self, old_path: str, new_path: str) -> None: + """Same as rename_note() but assumes the caller already holds the lock. + Used by folder-prefix rename to avoid re-acquiring the lock per file.""" + if old_path == new_path: + return + old_record = self._notes.pop(old_path, None) + if old_record is None: + return + folder = str(Path(new_path).parent).replace("\\", "/") + new_record = NoteRecord( + path=new_path, + name=Path(new_path).stem, + folder="" if folder == "." else folder, + modified=old_record.modified, + size=old_record.size, + type=old_record.type, + mtime=old_record.mtime, + tags=old_record.tags, + ) + self._notes[new_path] = new_record + if new_record.folder: + self._folders.add(new_record.folder) + + if old_path in self._tags_forward: + tags = self._tags_forward.pop(old_path) + self._tags_forward[new_path] = tags + for t in tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(old_path) + bucket.add(new_path) + + for paths in self._search_terms.values(): + if old_path in paths: + paths.discard(old_path) + paths.add(new_path) + + if old_path in self._raw_links: + self._raw_links[new_path] = self._raw_links.pop(old_path) + + self._links_forward.pop(old_path, None) + for bucket in self._links_backward.values(): + bucket.discard(old_path) + old_backlinks = self._links_backward.pop(old_path, set()) + for src in old_backlinks: + fwd = self._links_forward.get(src) + if fwd is not None and old_path in fwd: + del fwd[old_path] + if not fwd: + del self._links_forward[src] + + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + if old_path in sources: + sources.discard(old_path) + sources.add(new_path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + def _remove_note_unlocked(self, path: str) -> None: + """Same as remove_note() but assumes the caller already holds the lock.""" + old_record = self._notes.pop(path, None) + if old_record is None: + return + for t in old_record.tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._tags_backward[t] + self._tags_forward.pop(path, None) + self._raw_links.pop(path, None) + old_targets = self._links_forward.pop(path, {}) + for t in old_targets: + bucket = self._links_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._links_backward[t] + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + sources.discard(path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + sources_pointing_here = self._links_backward.pop(path, set()) + for src in sources_pointing_here: + fwd = self._links_forward.get(src) + if fwd is not None and path in fwd: + del fwd[path] + if not fwd: + del self._links_forward[src] + empty_terms = [] + for term, paths in self._search_terms.items(): + if path in paths: + paths.discard(path) + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + + +# ============================================================================ +# Resolver — link-target matching (mirrors legacy /api/graph rules exactly) +# ============================================================================ + +class _Resolver: + """Build the lookup tables once per resolution batch, then call + resolve_* repeatedly. Reused across sources within a single rebuild so + we don't pay O(N) to construct it on every source.""" + + def __init__(self, all_notes: Set[str]) -> None: + self.note_paths: Set[str] = set(all_notes) + self.note_paths_lower: Dict[str, str] = {} + self.note_names: Dict[str, str] = {} + for p in all_notes: + self.note_paths_lower[p.lower()] = p + if p.endswith(".md"): + self.note_paths_lower[p[:-3].lower()] = p + stem = Path(p).stem + self.note_names[stem.lower()] = p + self.note_names[Path(p).name.lower()] = p + + def resolve_wikilink(self, target: str, source_folder: str) -> Optional[str]: + target = target.strip() + if not target: + return None + target_lower = target.lower() + + # 1. Relative to source folder (only for bare names with no slash). + if source_folder and "/" not in target: + relative_path = f"{source_folder}/{target}" + relative_path_lower = relative_path.lower() + if relative_path in self.note_paths: + return relative_path if relative_path.endswith(".md") else relative_path + ".md" + if relative_path + ".md" in self.note_paths: + return relative_path + ".md" + if relative_path_lower in self.note_paths_lower: + return self.note_paths_lower[relative_path_lower] + if (relative_path_lower + ".md") in self.note_paths_lower: + return self.note_paths_lower[relative_path_lower + ".md"] + + if target in self.note_paths: + return target if target.endswith(".md") else target + ".md" + if (target + ".md") in self.note_paths: + return target + ".md" + if target_lower in self.note_paths_lower: + return self.note_paths_lower[target_lower] + if (target_lower + ".md") in self.note_paths_lower: + return self.note_paths_lower[target_lower + ".md"] + if target_lower in self.note_names: + return self.note_names[target_lower] + return None + + def resolve_mdlink(self, link_path: str, source_folder: str) -> Optional[str]: + if not link_path: + return None + link_path = link_path.split("#")[0] + if not link_path: + return None + link_path = urllib.parse.unquote(link_path) + if link_path.startswith("./"): + link_path = link_path[2:] + link_path_with_md = link_path if link_path.endswith(".md") else link_path + ".md" + + if source_folder and not link_path.startswith("/"): + relative_path = f"{source_folder}/{link_path}" + relative_path_with_md = f"{source_folder}/{link_path_with_md}" + relative_path_lower = relative_path.lower() + relative_path_with_md_lower = relative_path_with_md.lower() + if relative_path in self.note_paths: + return relative_path if relative_path.endswith(".md") else relative_path + ".md" + if relative_path_with_md in self.note_paths: + return relative_path_with_md + if relative_path_lower in self.note_paths_lower: + return self.note_paths_lower[relative_path_lower] + if relative_path_with_md_lower in self.note_paths_lower: + return self.note_paths_lower[relative_path_with_md_lower] + + link_path_lower = link_path.lower() + link_path_with_md_lower = link_path_with_md.lower() + if link_path in self.note_paths: + return link_path if link_path.endswith(".md") else link_path + ".md" + if link_path_with_md in self.note_paths: + return link_path_with_md + if link_path_lower in self.note_paths_lower: + return self.note_paths_lower[link_path_lower] + if link_path_with_md_lower in self.note_paths_lower: + return self.note_paths_lower[link_path_with_md_lower] + return None + + +# ============================================================================ +# Internal helpers +# ============================================================================ + +def _fingerprint( + notes_meta: List[NoteRecord], + sources_raw: Dict[str, Dict[str, List[str]]], +) -> int: + """Cheap content hash of a scan result. Used to short-circuit bulk_set + when nothing has changed in the vault.""" + notes_fp = hash(frozenset((n.path, n.mtime) for n in notes_meta)) + raw_items = ( + (src, tuple(raw.get("wikilinks", [])), tuple(raw.get("mdlinks", []))) + for src, raw in sources_raw.items() + ) + return hash((notes_fp, hash(frozenset(raw_items)))) + + +# ============================================================================ +# Module-level singleton + facade +# +# Every utils.py / main.py call site uses these — never instantiates its own +# index. The facade functions are no-ops (or return None) when +# USE_NOTE_INDEX is False, so call sites don't have to repeat the flag check. +# ============================================================================ + +_index = NoteIndex() + + +def get_index() -> NoteIndex: + return _index + + +# --- Lifecycle facade (one-line calls from utils.py mutators) ---------------- + +def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None: + """A note was created or updated on disk. Patch the index in place.""" + if not USE_NOTE_INDEX: + return + try: + rel_path = full_path.relative_to(Path(notes_dir)).as_posix() + st = full_path.stat() + folder = str(Path(rel_path).parent).replace("\\", "/") + record = NoteRecord( + path=rel_path, + name=Path(rel_path).stem, + folder="" if folder == "." else folder, + modified=datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), + size=st.st_size, + type="note", + mtime=st.st_mtime, + tags=tuple(_parse_tags_for_record(content)), + ) + raw_links = extract_links_from_content(content) + _index.update_note(record, raw_links, content=content) + except Exception as e: + print(f"note_index: on_note_saved failed for {full_path}: {e}") + + +def on_note_deleted(notes_dir: str, full_path: Path) -> None: + if not USE_NOTE_INDEX: + return + try: + rel_path = full_path.relative_to(Path(notes_dir)).as_posix() + _index.remove_note(rel_path) + except Exception as e: + print(f"note_index: on_note_deleted failed for {full_path}: {e}") + + +def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: + if not USE_NOTE_INDEX: + return + try: + base = Path(notes_dir) + _index.rename_note( + old_full_path.relative_to(base).as_posix(), + new_full_path.relative_to(base).as_posix(), + ) + except Exception as e: + print(f"note_index: on_note_renamed failed: {e}") + + +def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: + """A folder was moved/renamed. Re-keys every entry under it (cheap, + no disk reads) rather than invalidating the whole index.""" + if not USE_NOTE_INDEX: + return + try: + base = Path(notes_dir) + _index.rename_folder_prefix( + old_full_path.relative_to(base).as_posix(), + new_full_path.relative_to(base).as_posix(), + ) + except Exception as e: + print(f"note_index: on_folder_renamed failed: {e}") + _index.invalidate() # fail-safe + + +def on_folder_deleted(notes_dir: str, full_path: Path) -> None: + if not USE_NOTE_INDEX: + return + try: + rel_prefix = full_path.relative_to(Path(notes_dir)).as_posix() + _index.remove_folder_prefix(rel_prefix) + except Exception as e: + print(f"note_index: on_folder_deleted failed: {e}") + _index.invalidate() + + +def populate_from_scan( + notes_meta: List[NoteRecord], + folders: Iterable[str], + sources_raw: Dict[str, Dict[str, List[str]]], +) -> None: + """Bulk-replace the index from a fresh scan. No-op when off.""" + if not USE_NOTE_INDEX: + return + try: + _index.bulk_set(notes_meta, folders, sources_raw) + except Exception as e: + print(f"note_index: populate_from_scan failed: {e}") + + +def ensure_search_index(notes_dir: str) -> bool: + """Lazily build the full-text search index on first /api/search request. + Cheap after the first call. Returns True if the index is now usable.""" + if not USE_NOTE_INDEX: + return False + if not _index.is_built(): + return False + return _index.ensure_search_index_built(notes_dir) + + +# --- Read facade (returns None when off — callers fall through to legacy) ---- + +def try_backlink_candidates(target_path: str) -> Optional[Set[str]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_backlink_candidate_sources(target_path) + + +def try_graph_data() -> Optional[Tuple[List[str], List[Tuple[str, str, str]]]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_graph_data() + + +def try_search_candidates(query: str) -> Optional[Set[str]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_search_candidates(query) + + +def try_all_tags() -> Optional[Dict[str, int]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_all_tags() + + +def try_notes_by_tag(tag: str) -> Optional[Set[str]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_paths_for_tag(tag) + + +def try_get_extraction( + rel_path: str, + mtime: float, +) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: + """Serve (tags, raw_links) from the index for a single file, when fresh. + + Used by scan_notes_fast_walk to skip the cold per-file read after a + snapshot load. Returns None when the index can't help (off, not built, + file unknown, mtime mismatch) — caller falls back to reading the file. + """ + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.try_get_extraction(rel_path, mtime) + + +# --- Observability ----------------------------------------------------------- + +def stats() -> Dict[str, Any]: + return _index.stats() + + +# ============================================================================ +# Late import to avoid a circular dependency with utils.parse_tags. +# We need tag parsing inside on_note_saved but utils.py imports this module. +# ============================================================================ + +def _parse_tags_for_record(content: str) -> List[str]: + """Thin shim to utils.parse_tags. Late-bound so this module imports cleanly + before utils.py is loaded.""" + from .utils import parse_tags + return parse_tags(content) diff --git a/backend/utils.py b/backend/utils.py index 224e604..f3902a5 100644 --- a/backend/utils.py +++ b/backend/utils.py @@ -1,5 +1,9 @@ """ -Utility functions for file operations, search, and markdown processing +Utility functions for file operations, search, and markdown processing. + +Heavy work (backlinks, graph, full-text search, tag aggregation) is delegated +to the in-memory index in note_index.py — every facade function there is a +no-op when USE_NOTE_INDEX is False, so call sites here stay flag-free. """ import os @@ -7,6 +11,7 @@ import re import shutil import threading import time +import traceback import urllib.parse from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass @@ -14,6 +19,9 @@ from pathlib import Path from typing import List, Dict, Optional, Tuple, Any, TypeVar, Callable from datetime import datetime, timezone +from . import note_index +from .note_index import NoteRecord, extract_links_from_content + # ============================================================================ # Pagination Support @@ -112,14 +120,23 @@ def paginate( ) -# In-memory cache for parsed tags -# Format: {file_path: (mtime, tags)} -_tag_cache: Dict[str, Tuple[float, List[str]]] = {} - -# Notes tree scan cache (TTL). +# ============================================================================ +# In-memory caches # -# This avoids repeated full-directory walks when multiple endpoints (or the UI) -# request indexes in quick succession. +# Two layers: +# _tag_cache / _links_cache : per-file, mtime-keyed. Avoid re-parsing the +# same file on warm scans. Used by get_tags_and_links_cached. +# _SCAN_WALK_CACHE : per-scan TTL cache. Avoids repeated full- +# directory walks when several endpoints fire in quick succession. +# +# All three are invalidated on every mutation (save/delete/move). The +# note_index in note_index.py is the system of record for derived data +# (links, backlinks, tags-by-name, search) — these caches just exist to +# avoid double-reading files we already parsed. +# ============================================================================ + +_tag_cache: Dict[str, Tuple[float, List[str]]] = {} +_links_cache: Dict[str, Tuple[float, Dict[str, List[str]]]] = {} _SCAN_WALK_CACHE_LOCK = threading.Lock() _SCAN_WALK_CACHE_TTL_SECONDS = 1.0 @@ -145,6 +162,66 @@ def _scan_cache_set(key: Tuple[str, bool], value: Tuple[List[Dict], List[str]]) _SCAN_WALK_CACHE[key] = (time.monotonic(), value) +def _scan_cache_invalidate() -> None: + """Drop the TTL scan cache. Called from every mutation handler so callers + that hit the cache immediately after a save/delete/move see the new state + instead of stale data from the previous walk.""" + with _SCAN_WALK_CACHE_LOCK: + _SCAN_WALK_CACHE.clear() + + +def get_tags_and_links_cached( + file_path: Path, + rel_path: Optional[str] = None, +) -> Tuple[List[str], Dict[str, List[str]]]: + """Single-read fused extraction of tags + raw links, cached by mtime. + + Three-tier lookup, cheapest first: + 1. In-process per-file caches (warm scans on the live server) + 2. NoteIndex (warm after snapshot load — avoids file reads on cold start) + 3. Read the file from disk + + rel_path enables tier 2. scan_notes_fast_walk passes it; ad-hoc callers + can omit it and just get tiers 1 + 3. + """ + try: + mtime = file_path.stat().st_mtime + file_key = str(file_path) + + tag_cached = _tag_cache.get(file_key) + link_cached = _links_cache.get(file_key) + tags_ok = tag_cached is not None and tag_cached[0] == mtime + links_ok = link_cached is not None and link_cached[0] == mtime + + if tags_ok and links_ok: + return tag_cached[1], link_cached[1] + + # Try the NoteIndex before reading the file. On a snapshot-warm + # startup, this turns a 10K-file cold scan into a no-I/O walk. + if rel_path is not None: + indexed = note_index.try_get_extraction(rel_path, mtime) + if indexed is not None: + idx_tags, idx_links = indexed + _tag_cache[file_key] = (mtime, idx_tags) + _links_cache[file_key] = (mtime, idx_links) + return idx_tags, idx_links + + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + tags = tag_cached[1] if tags_ok else parse_tags(content) + links = link_cached[1] if links_ok else extract_links_from_content(content) + + if not tags_ok: + _tag_cache[file_key] = (mtime, tags) + if not links_ok: + _links_cache[file_key] = (mtime, links) + + return tags, links + except Exception: + return [], {"wikilinks": [], "mdlinks": []} + + def validate_path_security(notes_dir: str, path: Path) -> bool: """ Validate that a path is within the notes directory (security check). @@ -176,15 +253,12 @@ def ensure_directories(config: dict): def create_folder(notes_dir: str, folder_path: str) -> bool: - """Create a new folder in the notes directory""" + """Create a new folder in the notes directory.""" full_path = Path(notes_dir) / folder_path - - # Security check if not validate_path_security(notes_dir, full_path): return False - full_path.mkdir(parents=True, exist_ok=True) - + _scan_cache_invalidate() return True @@ -219,113 +293,99 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: _scan_cache_set(cache_key, normalized_value) return normalized_value - # TODO: remove this flag and the legacy branch below once the parallel - # path has been validated in production with large vaults. To revert to - # the pre-parallel behavior, flip _USE_PARALLEL_TAG_SCAN to False. - _USE_PARALLEL_TAG_SCAN = True - + # Walk the vault once. Tag/link extraction is deferred out of the walk + # (the expensive part is the file I/O) and parallelized below across files. notes: List[Dict] = [] folders_set = set() + md_to_extract: List[Tuple[int, Path, str]] = [] # (note_index, full_path, rel_path) - if not _USE_PARALLEL_TAG_SCAN: - # ===== LEGACY: sequential tag extraction inline with the walk ===== - for root, dirnames, filenames in os.walk(notes_path): - # Skip descending into dot-dirs - dirnames[:] = [d for d in dirnames if not d.startswith('.')] + for root, dirnames, filenames in os.walk(notes_path): + dirnames[:] = [d for d in dirnames if not d.startswith('.')] - root_path = Path(root) - rel_folder = root_path.relative_to(notes_path).as_posix() - if rel_folder != "." and not rel_folder.startswith('.'): - folders_set.add(rel_folder) + root_path = Path(root) + rel_folder = root_path.relative_to(notes_path).as_posix() + if rel_folder != "." and not rel_folder.startswith('.'): + folders_set.add(rel_folder) - for filename in filenames: - if filename.startswith('.'): - continue + for filename in filenames: + if filename.startswith('.'): + continue - full_path = root_path / filename - try: - st = full_path.stat() - except OSError: - continue + full_path = root_path / filename + try: + st = full_path.stat() + except OSError: + continue - relative_path = full_path.relative_to(notes_path) - media_type = get_media_type(filename) if include_media else None - is_markdown = full_path.suffix.lower() == '.md' - should_include = is_markdown or (include_media and media_type is not None) + relative_path = full_path.relative_to(notes_path) + media_type = get_media_type(filename) if include_media else None + is_markdown = full_path.suffix.lower() == '.md' + should_include = is_markdown or (include_media and media_type is not None) + if not should_include: + continue - if not should_include: - continue + folder = relative_path.parent.as_posix() + rel_str = relative_path.as_posix() + notes.append({ + "name": full_path.stem, + "path": rel_str, + "folder": "" if folder == "." else folder, + "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), + "size": st.st_size, + "type": media_type if media_type else "note", + "tags": [], + "_mtime": st.st_mtime, # internal — popped before returning + }) + if is_markdown: + md_to_extract.append((len(notes) - 1, full_path, rel_str)) - folder = relative_path.parent.as_posix() - tags = get_tags_cached(full_path) if is_markdown else [] - notes.append({ - "name": full_path.stem, - "path": relative_path.as_posix(), - "folder": "" if folder == "." else folder, - "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), - "size": st.st_size, - "type": media_type if media_type else "note", - "tags": tags, - }) - else: - # ===== NEW: tag extraction deferred out of the walk and parallelized ===== - # Indices of markdown entries in `notes` plus their full paths, so we can - # defer tag extraction (the expensive part) out of the walk and parallelize - # it across files. Tags are filled in after the walk completes. - md_to_tag: List[Tuple[int, Path]] = [] + # Fused tag + raw-link extraction. mtime-keyed cache makes warm scans + # mostly free; cold scans win from parallel I/O. + # + # We don't read full content here — that would double the parsing cost + # for every scan, and the search index is the only thing that needs it. + # Search is built lazily on the first search request (see + # note_index.ensure_search_index_built), which keeps startup fast and + # only pays the cost when a user actually searches. + sources_raw: Dict[str, Dict[str, List[str]]] = {} + if md_to_extract: + # Pass rel_path so each call can hit the NoteIndex before falling + # back to a file read — critical for fast startup after a snapshot + # load on large vaults. + path_pairs = [(full, rel) for (_, full, rel) in md_to_extract] + if len(md_to_extract) >= 50: + workers = min(8, (os.cpu_count() or 4)) + with ThreadPoolExecutor(max_workers=workers) as ex: + extraction = list(ex.map( + lambda pair: get_tags_and_links_cached(pair[0], pair[1]), + path_pairs, + )) + else: + extraction = [get_tags_and_links_cached(full, rel) for (full, rel) in path_pairs] - for root, dirnames, filenames in os.walk(notes_path): - dirnames[:] = [d for d in dirnames if not d.startswith('.')] + for (idx, _full, rel_str), (tags, links) in zip(md_to_extract, extraction): + notes[idx]["tags"] = tags + sources_raw[rel_str] = links - root_path = Path(root) - rel_folder = root_path.relative_to(notes_path).as_posix() - if rel_folder != "." and not rel_folder.startswith('.'): - folders_set.add(rel_folder) + # Populate the unified index. Cheap when the fingerprint matches (warm + # scan, nothing changed) — short-circuits internally. No-op when off. + notes_meta = [ + NoteRecord( + path=n["path"], + name=n["name"], + folder=n["folder"], + modified=n["modified"], + size=n["size"], + type=n["type"], + mtime=n["_mtime"], + tags=tuple(n["tags"]), + ) + for n in notes + ] + note_index.populate_from_scan(notes_meta, folders_set, sources_raw) - for filename in filenames: - if filename.startswith('.'): - continue - - full_path = root_path / filename - try: - st = full_path.stat() - except OSError: - continue - - relative_path = full_path.relative_to(notes_path) - media_type = get_media_type(filename) if include_media else None - is_markdown = full_path.suffix.lower() == '.md' - should_include = is_markdown or (include_media and media_type is not None) - - if not should_include: - continue - - folder = relative_path.parent.as_posix() - notes.append({ - "name": full_path.stem, - "path": relative_path.as_posix(), - "folder": "" if folder == "." else folder, - "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), - "size": st.st_size, - "type": media_type if media_type else "note", - "tags": [], - }) - if is_markdown: - md_to_tag.append((len(notes) - 1, full_path)) - - # Parallelize for large vaults (sequential branch avoids thread-spawn - # overhead for small vaults). get_tags_cached is mtime-keyed so this is - # mostly free on warm cache; the win is the cold first call. - if md_to_tag: - if len(md_to_tag) >= 50: - workers = min(8, (os.cpu_count() or 4)) - paths = [p for _, p in md_to_tag] - with ThreadPoolExecutor(max_workers=workers) as ex: - for (idx, _), tags in zip(md_to_tag, ex.map(get_tags_cached, paths)): - notes[idx]["tags"] = tags - else: - for idx, p in md_to_tag: - notes[idx]["tags"] = get_tags_cached(p) + for n in notes: + n.pop("_mtime", None) value = (sorted(notes, key=lambda x: x.get('modified', ''), reverse=True), sorted(folders_set)) if use_cache: @@ -333,132 +393,106 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: return value def move_note(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]: - """Move a note to a different location - - Returns: - Tuple of (success: bool, error_message: str) - """ + """Move a note. Returns (success, error_message).""" old_full_path = Path(notes_dir) / old_path new_full_path = Path(notes_dir) / new_path - - # Security checks + if not validate_path_security(notes_dir, old_full_path): return False, "Invalid source path" if not validate_path_security(notes_dir, new_full_path): return False, "Invalid destination path" - if not old_full_path.exists(): return False, f"Source note does not exist: {old_path}" - - # Check if target already exists (prevent overwriting) if new_full_path.exists(): return False, f"A note already exists at: {new_path}" - - # Invalidate cache for old path - old_key = str(old_full_path) - if old_key in _tag_cache: - del _tag_cache[old_key] - + + _drop_path_caches(old_full_path) + try: - # Create parent directory if needed new_full_path.parent.mkdir(parents=True, exist_ok=True) - - # Move the file old_full_path.rename(new_full_path) except Exception as e: return False, f"Failed to move file: {str(e)}" - - # Note: We don't automatically delete empty folders to preserve user's folder structure - + + note_index.on_note_renamed(notes_dir, old_full_path, new_full_path) + _scan_cache_invalidate() return True, "" def move_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]: - """Move a folder to a different location - - Returns: - Tuple of (success: bool, error_message: str) - """ - import shutil - + """Move a folder. Returns (success, error_message).""" old_full_path = Path(notes_dir) / old_path new_full_path = Path(notes_dir) / new_path - - # Security checks + if not validate_path_security(notes_dir, old_full_path): return False, "Invalid source path" if not validate_path_security(notes_dir, new_full_path): return False, "Invalid destination path" - if not old_full_path.exists() or not old_full_path.is_dir(): return False, f"Source folder does not exist: {old_path}" - - # Check if target already exists if new_full_path.exists(): return False, f"A folder already exists at: {new_path}" - - # Invalidate cache for all notes in this folder - global _tag_cache - old_path_str = str(old_full_path) - keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(old_path_str)] - for key in keys_to_delete: - del _tag_cache[key] - + + _drop_prefix_caches(old_full_path) + try: - # Create parent directory if needed new_full_path.parent.mkdir(parents=True, exist_ok=True) - - # Move the folder shutil.move(str(old_full_path), str(new_full_path)) except Exception as e: return False, f"Failed to move folder: {str(e)}" - - # Note: We don't automatically delete empty folders to preserve user's folder structure - + + note_index.on_folder_renamed(notes_dir, old_full_path, new_full_path) + _scan_cache_invalidate() return True, "" def rename_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]: - """Rename a folder (same as move but for clarity)""" + """Rename a folder (same as move, named for clarity).""" return move_folder(notes_dir, old_path, new_path) def delete_folder(notes_dir: str, folder_path: str) -> bool: - """Delete a folder and all its contents""" + """Delete a folder and all its contents.""" try: full_path = Path(notes_dir) / folder_path - - # Security check: ensure the path is within notes_dir + if not validate_path_security(notes_dir, full_path): print(f"Security: Path is outside notes directory: {full_path}") return False - if not full_path.exists(): print(f"Folder does not exist: {full_path}") return False - if not full_path.is_dir(): print(f"Path is not a directory: {full_path}") return False - - # Invalidate cache for all notes in this folder - global _tag_cache - folder_path_str = str(full_path) - keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(folder_path_str)] - for key in keys_to_delete: - del _tag_cache[key] - - # Delete the folder and all its contents + + _drop_prefix_caches(full_path) shutil.rmtree(full_path) - print(f"Successfully deleted folder: {full_path}") + note_index.on_folder_deleted(notes_dir, full_path) + _scan_cache_invalidate() return True except Exception as e: print(f"Error deleting folder '{folder_path}': {e}") - import traceback traceback.print_exc() return False +def _drop_path_caches(full_path: Path) -> None: + """Evict the per-file mtime caches for a single note.""" + key = str(full_path) + _tag_cache.pop(key, None) + _links_cache.pop(key, None) + + +def _drop_prefix_caches(folder_full_path: Path) -> None: + """Evict per-file mtime caches for every entry under a folder.""" + prefix = str(folder_full_path) + for k in [k for k in _tag_cache if k.startswith(prefix)]: + _tag_cache.pop(k, None) + for k in [k for k in _links_cache if k.startswith(prefix)]: + _links_cache.pop(k, None) + + def get_note_content(notes_dir: str, note_path: str) -> Optional[str]: @@ -477,114 +511,115 @@ def get_note_content(notes_dir: str, note_path: str) -> Optional[str]: def save_note(notes_dir: str, note_path: str, content: str) -> bool: - """Save or update a note""" + """Save or update a note.""" full_path = Path(notes_dir) / note_path - - # Ensure .md extension if not note_path.endswith('.md'): full_path = full_path.with_suffix('.md') - - # Security check + if not validate_path_security(notes_dir, full_path): return False - - # Create parent directories if needed + full_path.parent.mkdir(parents=True, exist_ok=True) - with open(full_path, 'w', encoding='utf-8') as f: f.write(content) - + + # Refresh the per-file mtime caches with what we just wrote, so the next + # scan doesn't re-parse this file. extract_links + parse_tags + save are + # tiny relative to the file write. + try: + mtime = full_path.stat().st_mtime + file_key = str(full_path) + _tag_cache[file_key] = (mtime, parse_tags(content)) + _links_cache[file_key] = (mtime, extract_links_from_content(content)) + except Exception: + pass # caches are best-effort + + note_index.on_note_saved(notes_dir, full_path, content) + _scan_cache_invalidate() return True def delete_note(notes_dir: str, note_path: str) -> bool: - """Delete a note""" + """Delete a note.""" full_path = Path(notes_dir) / note_path - if not full_path.exists(): return False - - # Security check if not validate_path_security(notes_dir, full_path): return False - - # Invalidate cache for this note - file_key = str(full_path) - if file_key in _tag_cache: - del _tag_cache[file_key] - + + _drop_path_caches(full_path) full_path.unlink() - - # Note: We don't automatically delete empty folders to preserve user's folder structure - + note_index.on_note_deleted(notes_dir, full_path) + _scan_cache_invalidate() return True def search_notes(notes_dir: str, query: str) -> List[Dict]: - """ - Full-text search through note contents only. - Does NOT search in file names, folder names, or paths - only note content. - Uses character-based context extraction with highlighted matches. + """Full-text search through note contents. + + Only searches inside note content (not filenames or paths). The index, when + available, narrows down the candidate files via a token-AND match on its + inverted index — then the existing per-line snippet extraction runs on the + narrow set. Output is byte-identical to a full scan. """ from html import escape - results = [] + results: List[Dict] = [] notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + # Build the search index lazily on the first search request. Subsequent + # searches hit the warm index; the first one pays the cost (read every + # file once) and is roughly as slow as the legacy full scan would be. + note_index.ensure_search_index(notes_dir) + candidates = note_index.try_search_candidates(query) + for note in notes: - md_file = Path(notes_dir) / note["path"] + path = note["path"] + if candidates is not None and path not in candidates: + continue + md_file = Path(notes_dir) / path try: with open(md_file, 'r', encoding='utf-8') as f: content = f.read() - - # Find all matches using regex (case-insensitive) + matches = list(re.finditer(re.escape(query), content, re.IGNORECASE)) - - if matches: - matched_lines = [] - - for match in matches[:3]: # Limit to 3 matches per file - start_index = match.start() - end_index = match.end() - matched_text = match.group() # Preserve original case - - # Create slice window: ±15 characters around match - context_start = max(0, start_index - 15) - context_end = min(len(content), end_index + 15) - - # Extract and clean parts (newlines → spaces) - before = escape(content[context_start:start_index].replace('\n', ' ')) - after = escape(content[end_index:context_end].replace('\n', ' ')) - matched_clean = escape(matched_text.replace('\n', ' ')) - - # Build snippet with highlight (styled via CSS) - snippet = f'{before}{matched_clean}{after}' - - # Add ellipsis if truncated at start - if context_start > 0: - snippet = '...' + snippet - - # Add ellipsis if truncated at end - if context_end < len(content): - snippet = snippet + '...' - - # Calculate line number by counting newlines up to match start - line_number = content.count('\n', 0, start_index) + 1 - - matched_lines.append({ - "line_number": line_number, - "context": snippet - }) - - relative_path = Path(note["path"]) - results.append({ - "name": md_file.stem, - "path": str(relative_path.as_posix()), - "folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "", - "matches": matched_lines + if not matches: + continue + + matched_lines = [] + for match in matches[:3]: + start_index = match.start() + end_index = match.end() + matched_text = match.group() + + context_start = max(0, start_index - 15) + context_end = min(len(content), end_index + 15) + + before = escape(content[context_start:start_index].replace('\n', ' ')) + after = escape(content[end_index:context_end].replace('\n', ' ')) + matched_clean = escape(matched_text.replace('\n', ' ')) + + snippet = f'{before}{matched_clean}{after}' + if context_start > 0: + snippet = '...' + snippet + if context_end < len(content): + snippet = snippet + '...' + + line_number = content.count('\n', 0, start_index) + 1 + matched_lines.append({ + "line_number": line_number, + "context": snippet, }) + + relative_path = Path(path) + results.append({ + "name": md_file.stem, + "path": str(relative_path.as_posix()), + "folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "", + "matches": matched_lines, + }) except Exception: continue - + return results @@ -852,105 +887,66 @@ def parse_tags(content: str) -> List[str]: return [] -def get_tags_cached(file_path: Path) -> List[str]: - """ - Get tags for a file with caching based on modification time. - - Args: - file_path: Path to the markdown file - - Returns: - List of tags from the file (cached if mtime unchanged) - """ - global _tag_cache - - try: - # Get current modification time - mtime = file_path.stat().st_mtime - file_key = str(file_path) - - # Check cache - if file_key in _tag_cache: - cached_mtime, cached_tags = _tag_cache[file_key] - if cached_mtime == mtime: - # Cache hit! Return cached tags - return cached_tags - - # Cache miss or stale - parse tags - with open(file_path, 'r', encoding='utf-8') as f: - content = f.read() - tags = parse_tags(content) - - # Update cache - _tag_cache[file_key] = (mtime, tags) - return tags - - except Exception: - # If anything fails, return empty list - return [] - - -def clear_tag_cache(): - """Clear the tag cache (useful for testing or manual cache invalidation)""" - global _tag_cache - _tag_cache.clear() - - def get_all_tags(notes_dir: str) -> Dict[str, int]: - """ - Get all tags used across all notes with their count (cached). - - Args: - notes_dir: Directory containing notes - - Returns: - Dictionary mapping tag names to note counts - """ - tag_counts = {} - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + """All tags in the vault with note counts. Index-backed when available.""" + # Fast path FIRST — when the index is ready it knows every tag count, + # so we skip the scan entirely (no disk walk, no JSON of 10K notes). + indexed = note_index.try_all_tags() + if indexed is not None: + return indexed + # Legacy fallback: scan the vault and aggregate. + notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + tag_counts: Dict[str, int] = {} for note in notes: - md_file = Path(notes_dir) / note["path"] - # Get tags using cache - tags = get_tags_cached(md_file) - - for tag in tags: + for tag in note.get("tags", []): tag_counts[tag] = tag_counts.get(tag, 0) + 1 - return dict(sorted(tag_counts.items())) def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]: - """ - Get all notes that have a specific tag (cached). - - Args: - notes_dir: Directory containing notes - tag: Tag to filter by (case-insensitive) - - Returns: - List of note dictionaries matching the tag - """ - matching_notes = [] + """All notes carrying `tag` (case-insensitive). Index-backed when available.""" tag_lower = tag.lower() - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + # Fast path FIRST: every field we need lives in the NoteRecord. Skip + # the vault scan when the index can answer directly. + indexed_paths = note_index.try_notes_by_tag(tag_lower) + if indexed_paths is not None: + idx = note_index.get_index() + records = [idx.get_note_record(p) for p in indexed_paths] + matching = [ + { + "name": r.name, + "path": r.path, + "folder": r.folder, + "modified": r.modified, + "size": r.size, + "tags": list(r.tags), + } + for r in records + if r is not None and r.type == "note" + ] + # Match legacy sort order (scan returns notes sorted by modified desc). + matching.sort(key=lambda n: n["modified"], reverse=True) + return matching + + # Legacy fallback. + notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + matching: List[Dict] = [] for note in notes: - md_file = Path(notes_dir) / note["path"] - # Get tags using cache - tags = get_tags_cached(md_file) - + if note.get("type") != "note": + continue + tags = note.get("tags", []) if tag_lower in tags: - matching_notes.append({ + matching.append({ "name": note["name"], "path": note["path"], "folder": note["folder"], "modified": note["modified"], "size": note["size"], - "tags": tags + "tags": tags, }) - - return matching_notes + return matching # ============================================================================ @@ -1107,140 +1103,138 @@ def apply_template_placeholders(content: str, note_path: str) -> str: return result +def _extract_backlink_references( + notes_dir: str, + source_path: str, + target_path: str, + target_path_lower: str, + target_path_no_ext_lower: str, + wikilink_refs: set, +) -> List[Dict]: + """Read one source file and pull line-level references that point to target. + + Same regex + resolution rules as the legacy get_backlinks loop body; just + factored out so both the index-driven and the legacy code paths use the + exact same context-extraction logic. + """ + source_folder = str(Path(source_path).parent).replace('\\', '/') + if source_folder == '.': + source_folder = '' + + full_path = Path(notes_dir) / source_path + try: + with open(full_path, 'r', encoding='utf-8') as f: + content = f.read() + except Exception: + return [] + + lines = content.split('\n') + found_links: List[Dict] = [] + + for line_num, line in enumerate(lines, 1): + for match in re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line): + link_target = match.group(1).strip().lower() + link_target_no_ext = link_target.replace('.md', '') + if link_target in wikilink_refs or link_target_no_ext in wikilink_refs: + start = max(0, match.start() - 30) + end = min(len(line), match.end() + 30) + context = line[start:end] + if start > 0: + context = '...' + context + if end < len(line): + context = context + '...' + found_links.append({ + "line_number": line_num, + "context": context, + "type": "wikilink", + }) + + for match in re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line): + link_path = match.group(2).split('#')[0] + if not link_path: + continue + link_path = urllib.parse.unquote(link_path) + if link_path.startswith('./'): + link_path = link_path[2:] + link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md' + + resolved_path = None + if source_folder and not link_path.startswith('/'): + relative_path = f"{source_folder}/{link_path_with_md}" + if relative_path.lower() == target_path_lower: + resolved_path = target_path + elif f"{source_folder}/{link_path}".lower() == target_path_no_ext_lower: + resolved_path = target_path + if not resolved_path: + if link_path_with_md.lower() == target_path_lower: + resolved_path = target_path + elif link_path.lower() == target_path_no_ext_lower: + resolved_path = target_path + + if resolved_path: + start = max(0, match.start() - 30) + end = min(len(line), match.end() + 30) + context = line[start:end] + if start > 0: + context = '...' + context + if end < len(line): + context = context + '...' + found_links.append({ + "line_number": line_num, + "context": context, + "type": "markdown", + }) + + return found_links + + def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]: - """ - Find all notes that link TO the specified note (reverse links / backlinks). + """Find all notes that link TO the specified note. - Args: - notes_dir: Base directory containing notes - target_note_path: Path of the note to find backlinks for - - Returns: - List of backlink objects with path, context, and line_number + Index-backed when available: the index hands us a superset of candidate + source files (O(1)), we read only those, then the per-line matcher + filters down to true matches with line-level context. Output is + byte-identical to a full vault scan. """ - backlinks = [] - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) - - # Normalize target path for matching target_path = target_note_path target_path_lower = target_path.lower() - target_path_no_ext = target_path.replace('.md', '') - target_path_no_ext_lower = target_path_no_ext.lower() - target_name = Path(target_path).stem.lower() - - # For wikilinks: global name matching (find note anywhere by name) + target_path_no_ext_lower = target_path_lower.replace('.md', '') wikilink_refs = { target_path_lower, target_path_no_ext_lower, - target_name, + Path(target_path).stem.lower(), } - + + # scan_notes_fast_walk is TTL-cached, so calling it here is a dict-lookup + # in the warm case. We need it for the canonical ordering of `notes`. + notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + + candidates = note_index.try_backlink_candidates(target_path) + + backlinks: List[Dict] = [] for note in notes: if note.get('type') != 'note': continue - source_path = note['path'] - - # Skip self-references if source_path == target_path: continue - - # Get source folder for resolving relative markdown links - source_folder = str(Path(source_path).parent).replace('\\', '/') - if source_folder == '.': - source_folder = '' - - # Read note content - full_path = Path(notes_dir) / source_path - try: - with open(full_path, 'r', encoding='utf-8') as f: - content = f.read() - except Exception: + if candidates is not None and source_path not in candidates: continue - - lines = content.split('\n') - found_links = [] - - for line_num, line in enumerate(lines, 1): - # Find wikilinks: [[target]] or [[target|display]] - # Wikilinks use GLOBAL matching (find note anywhere by name) - wikilink_matches = re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line) - for match in wikilink_matches: - link_target = match.group(1).strip().lower() - link_target_no_ext = link_target.replace('.md', '') - - # Check if this wikilink points to our target (global match) - if link_target in wikilink_refs or link_target_no_ext in wikilink_refs: - start = max(0, match.start() - 30) - end = min(len(line), match.end() + 30) - context = line[start:end] - if start > 0: - context = '...' + context - if end < len(line): - context = context + '...' - - found_links.append({ - "line_number": line_num, - "context": context, - "type": "wikilink" - }) - - # Find markdown links: [text](path) - # Markdown links must RESOLVE as paths (relative to source or absolute) - markdown_matches = re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line) - for match in markdown_matches: - link_path = match.group(2).split('#')[0] # Remove anchor - if not link_path: - continue - - link_path = urllib.parse.unquote(link_path) - if link_path.startswith('./'): - link_path = link_path[2:] - - # Add .md if not present - link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md' - - # Resolve the link path to get the actual target - resolved_path = None - - # 1. Try resolving relative to source folder - if source_folder and not link_path.startswith('/'): - relative_path = f"{source_folder}/{link_path_with_md}" - if relative_path.lower() == target_path_lower: - resolved_path = target_path - elif f"{source_folder}/{link_path}".lower() == target_path_no_ext_lower: - resolved_path = target_path - - # 2. Try as absolute path from root - if not resolved_path: - if link_path_with_md.lower() == target_path_lower: - resolved_path = target_path - elif link_path.lower() == target_path_no_ext_lower: - resolved_path = target_path - - # Only add if the resolved path matches the target - if resolved_path: - start = max(0, match.start() - 30) - end = min(len(line), match.end() + 30) - context = line[start:end] - if start > 0: - context = '...' + context - if end < len(line): - context = context + '...' - - found_links.append({ - "line_number": line_num, - "context": context, - "type": "markdown" - }) - - # If we found links in this note, add it to backlinks - if found_links: + + refs = _extract_backlink_references( + notes_dir, + source_path, + target_path, + target_path_lower, + target_path_no_ext_lower, + wikilink_refs, + ) + if refs: backlinks.append({ "path": source_path, "name": note['name'].replace('.md', ''), - "references": found_links[:3] # Limit to 3 references per note + "references": refs[:3], }) - + return backlinks diff --git a/documentation/API.md b/documentation/API.md index ad21981..767b0a0 100644 --- a/documentation/API.md +++ b/documentation/API.md @@ -580,6 +580,59 @@ Returns application statistics at a glance. Designed for dashboard widgets (e.g. label: Version ``` +### Get Index Stats +```http +GET /api/index/stats +``` +Returns internal counters and sizes for the in-memory note index. Useful for confirming the index is built and observing its growth — handy when debugging slow endpoints or verifying that incremental updates are firing on every save/delete. + +The index is rebuilt on the first scan after each process start and updated incrementally on every save/delete/move. It lives in process memory only — no disk persistence. + +**Response:** +```json +{ + "enabled": true, + "built": true, + "search_built": false, + "notes": 142, + "folders": 12, + "tags": 37, + "links_forward_entries": 89, + "links_backward_entries": 76, + "wikilink_tokens": 134, + "search_terms": 0, + "counters": { + "build_count": 1, + "last_build_ms": 12.4, + "last_built_at": "2026-03-17T14:32:00+00:00", + "incremental_updates": 8, + "fingerprint_short_circuits": 3, + "search_build_count": 0, + "last_search_build_ms": 0.0 + } +} +``` + +| Field | Description | +|-------|-------------| +| `enabled` | Whether the index is active (always `true` in current builds) | +| `built` | `true` after the first vault scan completes | +| `search_built` | `true` after the first `/api/search` request (search index is built lazily) | +| `notes` | Number of note records currently held in the index | +| `folders` | Number of folder paths | +| `tags` | Number of unique tags | +| `links_forward_entries` | Number of notes that link out to at least one other note | +| `links_backward_entries` | Number of notes that have at least one incoming backlink | +| `wikilink_tokens` | Number of unique wikilink tokens (used for loose backlink matching by stem name) | +| `search_terms` | Number of unique terms in the full-text inverted search index (0 until first search) | +| `counters.build_count` | Total number of full index rebuilds since process start | +| `counters.last_build_ms` | Wall-clock time of the most recent full rebuild | +| `counters.last_built_at` | ISO timestamp of the most recent full rebuild | +| `counters.incremental_updates` | Number of single-note updates applied (save/delete/rename) | +| `counters.fingerprint_short_circuits` | Times a re-scan was skipped because the vault hash matched the previous scan | +| `counters.search_build_count` | Total number of search-index rebuilds | +| `counters.last_search_build_ms` | Wall-clock time of the most recent search-index rebuild | + ### Health Check ```http GET /health