From 030565299a21d5c9b05df9da10f41777a858804e Mon Sep 17 00:00:00 2001 From: Gamosoft Date: Mon, 29 Jun 2026 17:08:52 +0200 Subject: [PATCH 01/18] parallel loading --- backend/utils.py | 132 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 99 insertions(+), 33 deletions(-) diff --git a/backend/utils.py b/backend/utils.py index 795aaf0..224e604 100644 --- a/backend/utils.py +++ b/backend/utils.py @@ -8,6 +8,7 @@ import shutil import threading import time import urllib.parse +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path from typing import List, Dict, Optional, Tuple, Any, TypeVar, Callable @@ -218,48 +219,113 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: _scan_cache_set(cache_key, normalized_value) return normalized_value + # TODO: remove this flag and the legacy branch below once the parallel + # path has been validated in production with large vaults. To revert to + # the pre-parallel behavior, flip _USE_PARALLEL_TAG_SCAN to False. + _USE_PARALLEL_TAG_SCAN = True + notes: List[Dict] = [] folders_set = set() - for root, dirnames, filenames in os.walk(notes_path): - # Skip descending into dot-dirs - dirnames[:] = [d for d in dirnames if not d.startswith('.')] + if not _USE_PARALLEL_TAG_SCAN: + # ===== LEGACY: sequential tag extraction inline with the walk ===== + for root, dirnames, filenames in os.walk(notes_path): + # Skip descending into dot-dirs + dirnames[:] = [d for d in dirnames if not d.startswith('.')] - root_path = Path(root) - rel_folder = root_path.relative_to(notes_path).as_posix() - if rel_folder != "." and not rel_folder.startswith('.'): - folders_set.add(rel_folder) + root_path = Path(root) + rel_folder = root_path.relative_to(notes_path).as_posix() + if rel_folder != "." and not rel_folder.startswith('.'): + folders_set.add(rel_folder) - for filename in filenames: - if filename.startswith('.'): - continue + for filename in filenames: + if filename.startswith('.'): + continue - full_path = root_path / filename - try: - st = full_path.stat() - except OSError: - continue + full_path = root_path / filename + try: + st = full_path.stat() + except OSError: + continue - relative_path = full_path.relative_to(notes_path) - media_type = get_media_type(filename) if include_media else None - is_markdown = full_path.suffix.lower() == '.md' - should_include = is_markdown or (include_media and media_type is not None) + relative_path = full_path.relative_to(notes_path) + media_type = get_media_type(filename) if include_media else None + is_markdown = full_path.suffix.lower() == '.md' + should_include = is_markdown or (include_media and media_type is not None) - if not should_include: - continue + if not should_include: + continue - folder = relative_path.parent.as_posix() - # Get tags for this note (cached) - tags = get_tags_cached(full_path) if is_markdown else [] - notes.append({ - "name": full_path.stem, - "path": relative_path.as_posix(), - "folder": "" if folder == "." else folder, - "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), - "size": st.st_size, - "type": media_type if media_type else "note", - "tags": tags, - }) + folder = relative_path.parent.as_posix() + tags = get_tags_cached(full_path) if is_markdown else [] + notes.append({ + "name": full_path.stem, + "path": relative_path.as_posix(), + "folder": "" if folder == "." else folder, + "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), + "size": st.st_size, + "type": media_type if media_type else "note", + "tags": tags, + }) + else: + # ===== NEW: tag extraction deferred out of the walk and parallelized ===== + # Indices of markdown entries in `notes` plus their full paths, so we can + # defer tag extraction (the expensive part) out of the walk and parallelize + # it across files. Tags are filled in after the walk completes. + md_to_tag: List[Tuple[int, Path]] = [] + + for root, dirnames, filenames in os.walk(notes_path): + dirnames[:] = [d for d in dirnames if not d.startswith('.')] + + root_path = Path(root) + rel_folder = root_path.relative_to(notes_path).as_posix() + if rel_folder != "." and not rel_folder.startswith('.'): + folders_set.add(rel_folder) + + for filename in filenames: + if filename.startswith('.'): + continue + + full_path = root_path / filename + try: + st = full_path.stat() + except OSError: + continue + + relative_path = full_path.relative_to(notes_path) + media_type = get_media_type(filename) if include_media else None + is_markdown = full_path.suffix.lower() == '.md' + should_include = is_markdown or (include_media and media_type is not None) + + if not should_include: + continue + + folder = relative_path.parent.as_posix() + notes.append({ + "name": full_path.stem, + "path": relative_path.as_posix(), + "folder": "" if folder == "." else folder, + "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), + "size": st.st_size, + "type": media_type if media_type else "note", + "tags": [], + }) + if is_markdown: + md_to_tag.append((len(notes) - 1, full_path)) + + # Parallelize for large vaults (sequential branch avoids thread-spawn + # overhead for small vaults). get_tags_cached is mtime-keyed so this is + # mostly free on warm cache; the win is the cold first call. + if md_to_tag: + if len(md_to_tag) >= 50: + workers = min(8, (os.cpu_count() or 4)) + paths = [p for _, p in md_to_tag] + with ThreadPoolExecutor(max_workers=workers) as ex: + for (idx, _), tags in zip(md_to_tag, ex.map(get_tags_cached, paths)): + notes[idx]["tags"] = tags + else: + for idx, p in md_to_tag: + notes[idx]["tags"] = get_tags_cached(p) value = (sorted(notes, key=lambda x: x.get('modified', ''), reverse=True), sorted(folders_set)) if use_cache: From 4b316b2cc73e2aa3a96c7f2615eb8d543b80a52b Mon Sep 17 00:00:00 2001 From: Gamosoft Date: Tue, 30 Jun 2026 11:14:42 +0200 Subject: [PATCH 02/18] global in memory index for large vaults --- backend/main.py | 33 +- backend/note_index.py | 1219 +++++++++++++++++++++++++++++++++++++++++ backend/utils.py | 880 +++++++++++++++-------------- documentation/API.md | 53 ++ 4 files changed, 1741 insertions(+), 444 deletions(-) create mode 100644 backend/note_index.py diff --git a/backend/main.py b/backend/main.py index 72781ff..bf823ef 100644 --- a/backend/main.py +++ b/backend/main.py @@ -45,6 +45,8 @@ from .utils import ( paginate, get_backlinks, ) +from . import note_index +from .note_index import USE_NOTE_INDEX from .plugins import PluginManager from .themes import get_available_themes, get_theme_css from .share import ( @@ -265,6 +267,7 @@ plugin_manager = PluginManager(config['storage']['plugins_dir']) # Run app startup hooks plugin_manager.run_hook('on_app_startup') + # Mount static files static_path = Path(__file__).parent.parent / "frontend" app.mount("/static", StaticFiles(directory=static_path), name="static") @@ -1483,10 +1486,25 @@ async def search( @api_router.get("/graph", tags=["Graph"]) async def get_graph(): - """Get graph data for note visualization with wikilink and markdown link detection""" + """Get graph data for note visualization with wikilink and markdown link detection.""" try: import re import urllib.parse + + # Fast path: index already holds resolved edges. Ensure a scan has + # populated it (first request after startup, or after an invalidate) + # then ask the facade for a snapshot. + if USE_NOTE_INDEX: + if not note_index.get_index().is_built(): + scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) + indexed = note_index.try_graph_data() + if indexed is not None: + nodes_paths, edges_tuples = indexed + return { + "nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths], + "edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples], + } + notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) nodes = [] edges = [] @@ -1692,6 +1710,19 @@ async def toggle_plugin(request: Request, plugin_name: str, enabled: dict): raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to toggle plugin")) +# ============================================================================ +# Index observability — internal counters & sizes for debugging +# +# Useful for confirming the index is doing what we expect (build_count >0, +# search_terms not empty, fingerprint_short_circuits incrementing on idle +# warm scans). Not rate-limited — cheap, no I/O, dict lookups only. +# ============================================================================ + +@api_router.get("/index/stats", tags=["System"]) +async def get_index_stats(): + return note_index.stats() + + # ============================================================================ # Stats Endpoint (for dashboards) # ============================================================================ diff --git a/backend/note_index.py b/backend/note_index.py new file mode 100644 index 0000000..5a0d7eb --- /dev/null +++ b/backend/note_index.py @@ -0,0 +1,1219 @@ +""" +NoteIndex — unified in-memory index of everything that's expensive to recompute +from disk: links, tags, full-text search, and per-note metadata. + + +---------------------------------------------------------------+ + | ONE module, ONE singleton, ONE lock, ONE rollback switch. | + | All call sites in utils.py / main.py are one-line shims. | + +---------------------------------------------------------------+ + +Why this exists +--------------- +Without an index, every backlink lookup, every graph render, and every text +search re-walks the entire vault and re-reads every file. That's O(N) per +request and scales linearly with vault size — 4-second note loads on a 10K +vault, 5-10s graph renders, multi-second searches. With an index, those +endpoints become O(matches) — milliseconds regardless of vault size. + +What's indexed +-------------- + * Note metadata (path -> name, folder, mtime, size, type, tags) + * Folders (set of folder paths) + * Tags forward (path -> sorted tags) + * Tags backward (tag -> set of paths) + * Links forward (source_path -> {target_path: "wikilink"|"markdown"}) + * Links backward (target_path -> set of source paths) strict + * Wikilink tokens (lowercased token -> set of source paths) loose + * Search inverted (lowercased term -> set of paths) + +What's NOT cached +----------------- +File content. Reading file content for line-level context (backlinks, search +snippets) is done only for the small set of matched files, never for the +whole vault. + +Threading model +--------------- +Everything mutating goes through one RLock. Reads return snapshot copies, so +callers can iterate freely after they release the lock. The index is +designed to be called from FastAPI request handlers concurrently. + +Lifetime +-------- +Process-memory only. No persistence to disk — keeping the app lightweight +matters more than the ~500ms saved on cold restart for a 10K-note vault. +The index rebuilds on the first `/api/notes` request after every restart +(typically 1-3 seconds, dominated by reading every file once for tag/link +extraction). F5 / browser reload doesn't touch this — only Python process +restart does. + +Rollback switch +--------------- +USE_NOTE_INDEX (below) is the single, global on/off. Flip it to False and +every facade function becomes a no-op or returns None, and every call site +in utils.py / main.py falls through to the legacy file-scanning behavior. + + ROLLBACK RECIPE (decide the index isn't worth it) + ------------------------------------------------- + Set USE_NOTE_INDEX = False below. That's it. Every try_* facade returns + None, every on_* facade becomes a no-op, callers fall through to the + legacy file-scanning paths that are preserved as the function bodies. + + COMMIT RECIPE (you're happy, drop the legacy paths) + --------------------------------------------------- + 1. Here: delete USE_NOTE_INDEX and inline `if not USE_NOTE_INDEX` to True + in every try_/on_ facade. + 2. In backend/utils.py: + - get_backlinks: drop the `candidates is None` branch — the + index always provides them. + - search_notes: drop the `candidates is None` fallback. + - get_all_tags / get_notes_by_tag: drop the legacy aggregation + tail block. + 3. In backend/main.py: + - /api/graph: drop the legacy fallback block at the bottom of + the endpoint, keep only `try_graph_data`. +""" + +from __future__ import annotations + +import os +import re +import threading +import time +import urllib.parse +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + + +# ============================================================================ +# Configuration +# ============================================================================ + +# Single global rollback switch. Flip to False to disable the index and every +# call site in utils.py / main.py falls back to the legacy code paths. +USE_NOTE_INDEX: bool = True + +# Parallelism cutoff for full rescans. Below this many markdown files, threads +# add more overhead than they save (small vaults stay sequential). +_PARALLEL_CUTOFF = 50 +_PARALLEL_WORKERS = min(8, (os.cpu_count() or 4)) + +# Search tokenization. Lowercase, split on anything that's not a word char. +# Min length 2 keeps single-letter noise out of the index without losing +# common short queries like "go", "ai". +_SEARCH_TOKEN_RE = re.compile(r"[A-Za-z0-9_\-]{2,}") +_SEARCH_MIN_QUERY_LEN = 2 + + +# ============================================================================ +# Shared regexes (same as legacy get_backlinks / /api/graph). Kept here so the +# index and the legacy paths are guaranteed to extract the same tokens. +# ============================================================================ + +WIKILINK_RE = re.compile(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]') +MDLINK_RE = re.compile(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)') + + +# ============================================================================ +# Public extraction helpers — used by the index AND by lifecycle hooks in +# utils.py (so a single save can update the index without re-reading the file). +# ============================================================================ + +def extract_links_from_content(content: str) -> Dict[str, List[str]]: + """Pull raw wikilink targets and markdown-link paths out of note content.""" + wikilinks = [m.strip() for m in WIKILINK_RE.findall(content)] + mdlinks = [link_path for _, link_path in MDLINK_RE.findall(content)] + return {"wikilinks": wikilinks, "mdlinks": mdlinks} + + +def extract_search_terms(content: str) -> Set[str]: + """Tokenize content for the inverted search index. Lowercased, deduped.""" + return {m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(content)} + + +# ============================================================================ +# Data record for one note's metadata snapshot +# ============================================================================ + +@dataclass +class NoteRecord: + """Snapshot of one note's metadata. Kept tiny — no content, no resolved + links (those live in the inverted indexes).""" + path: str # vault-relative POSIX + name: str # stem (no extension) + folder: str # vault-relative POSIX, "" for root + modified: str # ISO timestamp + size: int # bytes + type: str # "note" | "image" | "audio" | ... + mtime: float # raw stat mtime, for staleness checks + tags: Tuple[str, ...] = field(default_factory=tuple) # sorted, deduped + + +# ============================================================================ +# The index itself +# ============================================================================ + +class NoteIndex: + """Thread-safe in-memory index of vault state. Every read returns a + snapshot copy so callers don't have to hold the lock.""" + + # ------------------------------------------------------------------ + # Construction + # ------------------------------------------------------------------ + + def __init__(self) -> None: + self._lock = threading.RLock() + + # Note metadata + self._notes: Dict[str, NoteRecord] = {} # path -> record + self._folders: Set[str] = set() + + # Tag indexes + self._tags_forward: Dict[str, Tuple[str, ...]] = {} # path -> tags + self._tags_backward: Dict[str, Set[str]] = {} # tag -> paths + + # Link indexes + self._raw_links: Dict[str, Dict[str, List[str]]] = {} # path -> {"wikilinks":[], "mdlinks":[]} + self._links_forward: Dict[str, Dict[str, str]] = {} # src -> {tgt: type} + self._links_backward: Dict[str, Set[str]] = {} # tgt -> {srcs} + self._wikilink_tokens: Dict[str, Set[str]] = {} # token -> {srcs} (loose) + + # Full-text inverted index + self._search_terms: Dict[str, Set[str]] = {} # term -> {paths} + + # Build state. _built tracks the cheap part (notes/tags/links). + # _search_built tracks the expensive search index separately — that + # one is built lazily on first search request to keep startup fast. + self._built = False + self._search_built = False + self._raw_fingerprint: Optional[int] = None # short-circuits no-op rebuilds + + # Observability counters (lock-free reads OK; rough is fine) + self._stats = { + "build_count": 0, + "last_build_ms": 0.0, + "last_built_at": None, + "incremental_updates": 0, + "fingerprint_short_circuits": 0, + "search_build_count": 0, + "last_search_build_ms": 0.0, + } + + # ------------------------------------------------------------------ + # Status / lifecycle gates + # ------------------------------------------------------------------ + + def is_built(self) -> bool: + with self._lock: + return self._built + + def invalidate(self) -> None: + """Mark as needing rebuild on next scan. Used as a safety net when + an operation's incremental update is too fiddly to track precisely.""" + with self._lock: + self._built = False + self._raw_fingerprint = None + + def reset(self) -> None: + with self._lock: + self._notes.clear() + self._folders.clear() + self._tags_forward.clear() + self._tags_backward.clear() + self._raw_links.clear() + self._links_forward.clear() + self._links_backward.clear() + self._wikilink_tokens.clear() + self._search_terms.clear() + self._built = False + self._search_built = False + self._raw_fingerprint = None + + def is_search_built(self) -> bool: + with self._lock: + return self._search_built + + def ensure_search_index_built(self, notes_dir: str) -> bool: + """Build the full-text search index by reading every markdown file. + + Cheap if already built (instant return). Called lazily by search_notes + on the first search request so app startup stays fast. Subsequent + searches reuse the built index. + + Concurrency: while one thread is building, others can still read the + (still-cold) index — they'll just take the legacy path until the + build completes. Returns True if the index is built (now or already). + """ + # Cheap pre-check without lock. + if self._search_built: + return True + + # Slow path: do the read+tokenize OUTSIDE the lock so other reads + # aren't blocked. Snapshot the paths we need to read under the lock, + # release, do I/O, then re-acquire to install the result. + with self._lock: + if self._search_built: # raced with another builder + return True + if not self._built: + return False + paths_to_read = [p for p, r in self._notes.items() if r.type == "note"] + + t0 = time.perf_counter() + base = Path(notes_dir) + terms_per_path: Dict[str, Set[str]] = {} + for rel in paths_to_read: + try: + full = base / rel + with open(full, "r", encoding="utf-8") as f: + terms_per_path[rel] = extract_search_terms(f.read()) + except Exception: + terms_per_path[rel] = set() + + # Re-acquire and install. If state changed under us (vault was + # heavily mutated during the build), still install — the index + # will be slightly stale until the next note save, which is the + # same liveness guarantee as the per-save update path. + with self._lock: + if self._search_built: + return True + self._search_terms.clear() + for path, terms in terms_per_path.items(): + for term in terms: + self._search_terms.setdefault(term, set()).add(path) + self._search_built = True + self._stats["search_build_count"] += 1 + self._stats["last_search_build_ms"] = (time.perf_counter() - t0) * 1000 + return True + + # ------------------------------------------------------------------ + # Bulk population — called from a full scan_notes_fast_walk + # ------------------------------------------------------------------ + + def bulk_set( + self, + notes_meta: List[NoteRecord], + folders: Iterable[str], + sources_raw: Dict[str, Dict[str, List[str]]], + ) -> None: + """Replace the entire index atomically. Builds notes/tags/links + (cheap on top of an already-completed scan). The search index is + NOT touched here — it's built lazily on the first search request + (see ensure_search_index_built) so app startup stays fast. + + Fast-paths when the input fingerprints to the same state we already + hold (typical warm scan where nothing changed in the vault). + """ + new_fp = _fingerprint(notes_meta, sources_raw) + with self._lock: + if self._built and self._raw_fingerprint == new_fp: + self._stats["fingerprint_short_circuits"] += 1 + return + + t0 = time.perf_counter() + + self._notes = {n.path: n for n in notes_meta} + self._folders = set(folders) + self._raw_links = {k: v for k, v in sources_raw.items()} + + self._rebuild_tags_unlocked() + self._rebuild_links_unlocked() + # Search index: if it was built and still references known notes, + # prune stale entries; full rebuild is deferred until a search. + if self._search_built: + self._prune_search_unlocked() + + self._raw_fingerprint = new_fp + self._built = True + + self._stats["build_count"] += 1 + self._stats["last_build_ms"] = (time.perf_counter() - t0) * 1000 + self._stats["last_built_at"] = datetime.now(tz=timezone.utc).isoformat() + + # ------------------------------------------------------------------ + # Incremental updates — called from save/delete/rename handlers + # ------------------------------------------------------------------ + + def update_note( + self, + record: NoteRecord, + raw_links: Dict[str, List[str]], + content: Optional[str] = None, + ) -> None: + """A single note's content changed (or was just created). Patches the + index in place: re-extracts its links, updates the tag indexes, and + if `content` is provided, refreshes its search-term entries.""" + with self._lock: + old_record = self._notes.get(record.path) + old_tags = old_record.tags if old_record else () + + self._notes[record.path] = record + if record.folder: + self._folders.add(record.folder) + + # Tags: diff old vs new and patch the backward index. + new_tags = record.tags + if old_tags != new_tags: + for t in set(old_tags) - set(new_tags): + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(record.path) + if not bucket: + del self._tags_backward[t] + for t in set(new_tags) - set(old_tags): + self._tags_backward.setdefault(t, set()).add(record.path) + self._tags_forward[record.path] = new_tags + + # Links: re-resolve this one source against the current vault. + self._raw_links[record.path] = raw_links + self._resolve_single_source_unlocked(record.path) + + # Search: only patch the search index if it's already been built. + # Otherwise we'd be doing work for an index nobody is using yet + # (the first /api/search call will build it from disk). + if content is not None and self._search_built: + self._update_search_for_note_unlocked(record.path, content) + + self._raw_fingerprint = None + self._stats["incremental_updates"] += 1 + + def remove_note(self, path: str) -> None: + """A note was deleted. Drop everything that mentions it.""" + with self._lock: + old_record = self._notes.pop(path, None) + if old_record is None: + return + + # Tags + for t in old_record.tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._tags_backward[t] + self._tags_forward.pop(path, None) + + # Links: drop as source. + self._raw_links.pop(path, None) + old_targets = self._links_forward.pop(path, {}) + for t in old_targets: + bucket = self._links_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._links_backward[t] + + # Links: drop as loose wikilink source. + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + sources.discard(path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + # Links: drop as target from every other source's forward dict. + sources_pointing_here = self._links_backward.pop(path, set()) + for src in sources_pointing_here: + fwd = self._links_forward.get(src) + if fwd is not None and path in fwd: + del fwd[path] + if not fwd: + del self._links_forward[src] + + # Search: drop from every term bucket. Linear in distinct terms + # this note contributed to, which is bounded by note size. + empty_terms = [] + for term, paths in self._search_terms.items(): + if path in paths: + paths.discard(path) + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + + self._raw_fingerprint = None + self._stats["incremental_updates"] += 1 + + def rename_note(self, old_path: str, new_path: str) -> None: + """A note was renamed/moved. Move all references from old_path to + new_path in every index. Because the new path can change the source's + own relative-folder resolution AND can change how other notes resolve + their wikilinks to it (different parent folder, different stem), + the simplest correct path is to migrate raw state then re-resolve.""" + if old_path == new_path: + return + with self._lock: + old_record = self._notes.pop(old_path, None) + if old_record is None: + return + new_record = NoteRecord( + path=new_path, + name=Path(new_path).stem, + folder=str(Path(new_path).parent).replace("\\", "/").lstrip(".").lstrip("/") or "", + modified=old_record.modified, + size=old_record.size, + type=old_record.type, + mtime=old_record.mtime, + tags=old_record.tags, + ) + # Re-derive folder cleanly (the above lstrip dance is brittle). + folder = str(Path(new_path).parent).replace("\\", "/") + new_record.folder = "" if folder == "." else folder + self._notes[new_path] = new_record + if new_record.folder: + self._folders.add(new_record.folder) + + # Tags forward + backward — swap the key. + if old_path in self._tags_forward: + tags = self._tags_forward.pop(old_path) + self._tags_forward[new_path] = tags + for t in tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(old_path) + bucket.add(new_path) + + # Search: swap path in every term bucket. Bounded by terms-per-note. + for paths in self._search_terms.values(): + if old_path in paths: + paths.discard(old_path) + paths.add(new_path) + + # Raw links: migrate. + if old_path in self._raw_links: + self._raw_links[new_path] = self._raw_links.pop(old_path) + + # Strict link indexes: drop both old src and old target everywhere, + # then re-resolve. Other sources that linked TO old_path by name + # may now resolve to new_path (or not), so we need a re-resolve. + # + # Implementation: drop forward/backward for old_path, drop old + # entries in other sources' forwards that pointed to old_path, + # invalidate the index. The next scan rebuilds (which we trigger + # implicitly because the caller flips _built = False below). + self._links_forward.pop(old_path, None) + for bucket in self._links_backward.values(): + bucket.discard(old_path) + old_backlinks = self._links_backward.pop(old_path, set()) + for src in old_backlinks: + fwd = self._links_forward.get(src) + if fwd is not None and old_path in fwd: + del fwd[old_path] + if not fwd: + del self._links_forward[src] + + # Loose wikilink index: replace path values. + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + if old_path in sources: + sources.discard(old_path) + sources.add(new_path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + self._raw_fingerprint = None + self._built = False # force re-resolve on next bulk_set/scan + self._stats["incremental_updates"] += 1 + + def rename_folder_prefix(self, old_prefix: str, new_prefix: str) -> None: + """A folder was renamed/moved. Migrate every entry whose path starts + with `old_prefix/` to `new_prefix/`. Much cheaper than a full rebuild: + for a 1000-note folder rename, this is microseconds of key swaps + vs ~400ms of disk re-scan. + + Both prefixes are normalized to forward slashes, no trailing slash. + """ + old_prefix = old_prefix.rstrip("/") + new_prefix = new_prefix.rstrip("/") + if old_prefix == new_prefix: + return + with self._lock: + affected_paths = [p for p in self._notes if p == old_prefix or p.startswith(old_prefix + "/")] + for old_path in affected_paths: + suffix = old_path[len(old_prefix):] # includes leading "/" or nothing + new_path = new_prefix + suffix + # Reuse rename_note's heavy lifting — already correct, just + # called many times. Acceptable for folder operations. + # (Inlining would be a perf gain but multiplies bug surface.) + self._rename_note_unlocked(old_path, new_path) + + # Migrate the folder set too. + folders_to_rename = [ + f for f in self._folders if f == old_prefix or f.startswith(old_prefix + "/") + ] + for f in folders_to_rename: + self._folders.discard(f) + suffix = f[len(old_prefix):] + self._folders.add(new_prefix + suffix) + + self._raw_fingerprint = None + self._built = False + + def remove_folder_prefix(self, prefix: str) -> None: + """A folder was deleted. Drop everything under it.""" + prefix = prefix.rstrip("/") + with self._lock: + affected = [p for p in self._notes if p == prefix or p.startswith(prefix + "/")] + for path in affected: + self._remove_note_unlocked(path) + folders_to_drop = [ + f for f in self._folders if f == prefix or f.startswith(prefix + "/") + ] + for f in folders_to_drop: + self._folders.discard(f) + self._raw_fingerprint = None + + # ------------------------------------------------------------------ + # Read API — every method returns a snapshot copy + # ------------------------------------------------------------------ + + def get_backlink_candidate_sources(self, target_path: str) -> Set[str]: + """Superset of true backlink sources. Combines strict resolved + backward links with the loose wikilink-token reverse index. Caller + runs the per-line matcher against each candidate to filter.""" + target_lower = target_path.lower() + target_no_ext_lower = target_lower[:-3] if target_lower.endswith(".md") else target_lower + target_name = Path(target_path).stem.lower() + + with self._lock: + candidates: Set[str] = set(self._links_backward.get(target_path, set())) + for key in (target_lower, target_no_ext_lower, target_name): + candidates.update(self._wikilink_tokens.get(key, set())) + candidates.discard(target_path) + return candidates + + def get_graph_data(self) -> Tuple[List[str], List[Tuple[str, str, str]]]: + """Snapshot of (sorted note paths, (source, target, type) edges).""" + with self._lock: + nodes = sorted(p for p, r in self._notes.items() if r.type == "note") + edges: List[Tuple[str, str, str]] = [] + for src, targets in self._links_forward.items(): + for target, edge_type in targets.items(): + edges.append((src, target, edge_type)) + return nodes, edges + + def get_all_tags(self) -> Dict[str, int]: + """Snapshot: {tag: count}, sorted by tag name.""" + with self._lock: + return {tag: len(paths) for tag, paths in sorted(self._tags_backward.items())} + + def get_paths_for_tag(self, tag: str) -> Set[str]: + """Snapshot set of paths tagged with `tag` (case-insensitive).""" + with self._lock: + return set(self._tags_backward.get(tag.lower(), set())) + + def get_note_record(self, path: str) -> Optional[NoteRecord]: + with self._lock: + return self._notes.get(path) + + def try_get_extraction( + self, + rel_path: str, + mtime: float, + ) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: + """Return (tags, raw_links) from the index when fresh. + + The caller passes the file's current mtime; we hand back the cached + extraction only when the recorded mtime matches exactly. Lets + scan_notes_fast_walk skip the per-file read on a snapshot-warm + startup — the bulk of cold-load latency on large vaults. + + Returns None when: + - The note isn't in the index (new file) + - mtime differs (file changed since snapshot) + - We've somehow lost the raw_links entry (defensive) + """ + with self._lock: + rec = self._notes.get(rel_path) + if rec is None or rec.mtime != mtime: + return None + raw = self._raw_links.get(rel_path) + if raw is None: + return None + return ( + list(rec.tags), + { + "wikilinks": list(raw.get("wikilinks", [])), + "mdlinks": list(raw.get("mdlinks", [])), + }, + ) + + def get_search_candidates(self, query: str) -> Optional[Set[str]]: + # Index can only narrow candidates if it's been built. + if not self._search_built: + return None + """Return the set of paths that COULD contain `query` as a substring, + based on the inverted term index. + + Returns None when the query is too short to use the index (caller + should fall through to the legacy full-scan). When the query + tokenizes to nothing useful (all stopword-like noise), returns the + set of every indexed path so the caller still does a substring + check on each (no false negatives). + + IMPORTANT: this is a SUPERSET — the caller must still run the + substring match per candidate to confirm and extract context. + Token-AND can include docs that have both tokens but not as the + adjacent substring the user searched for. + """ + if len(query) < _SEARCH_MIN_QUERY_LEN: + return None + tokens = [m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(query)] + if not tokens: + # Query has no tokenizable parts (pure punctuation, single char, + # etc.). Index can't help; fall through. + return None + with self._lock: + # Intersection of all token buckets. Empty set means no doc + # contains ALL of the tokens (and therefore none contain the + # query as substring either). + candidate = self._search_terms.get(tokens[0]) + if candidate is None: + return set() + result: Set[str] = set(candidate) + for tok in tokens[1:]: + bucket = self._search_terms.get(tok) + if bucket is None: + return set() + result &= bucket + if not result: + break + return result + + def stats(self) -> Dict[str, Any]: + """Snapshot of internal counters + size metrics. Cheap to call — + no traversal beyond `len()` on the top-level dicts.""" + with self._lock: + return { + "enabled": USE_NOTE_INDEX, + "built": self._built, + "search_built": self._search_built, + "notes": len(self._notes), + "folders": len(self._folders), + "tags": len(self._tags_backward), + "links_forward_entries": len(self._links_forward), + "links_backward_entries": len(self._links_backward), + "wikilink_tokens": len(self._wikilink_tokens), + "search_terms": len(self._search_terms), + "counters": dict(self._stats), + } + + # ================================================================== + # Internal — must hold _lock when called + # ================================================================== + + def _rebuild_tags_unlocked(self) -> None: + self._tags_forward.clear() + self._tags_backward.clear() + for path, record in self._notes.items(): + if record.type != "note": + continue + tags = record.tags + if not tags: + continue + self._tags_forward[path] = tags + for tag in tags: + self._tags_backward.setdefault(tag, set()).add(path) + + def _rebuild_links_unlocked(self) -> None: + """Full link re-resolution. Builds a shared Resolver lookup once + (O(N)) and reuses it for every source (O(K) each) — total O(N+K*L) + rather than the O(N*K) you'd get by building a fresh Resolver per + source. + """ + self._links_forward.clear() + self._links_backward.clear() + self._wikilink_tokens.clear() + note_paths = {p for p, r in self._notes.items() if r.type == "note"} + resolver = _Resolver(note_paths) + for source_path in self._raw_links: + self._resolve_single_source_unlocked(source_path, resolver=resolver, skip_cleanup=True) + + def _prune_search_unlocked(self) -> None: + """Drop search-term entries for paths no longer in the index. Cheap + compared to a full rebuild — only touches terms whose bucket still + references a now-deleted path.""" + live_paths = set(self._notes.keys()) + empty_terms = [] + for term, paths in self._search_terms.items(): + stale = paths - live_paths + if stale: + paths -= stale + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + + def _update_search_for_note_unlocked(self, path: str, content: str) -> None: + """Replace one note's terms in the inverted index.""" + # Drop old entries for this path. + empty_terms = [] + for term, paths in self._search_terms.items(): + if path in paths: + paths.discard(path) + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + # Add new entries. + for term in extract_search_terms(content): + self._search_terms.setdefault(term, set()).add(path) + + def _resolve_single_source_unlocked( + self, + source_path: str, + resolver: Optional["_Resolver"] = None, + skip_cleanup: bool = False, + ) -> None: + if not skip_cleanup: + old_targets = self._links_forward.pop(source_path, {}) + for t in old_targets: + bucket = self._links_backward.get(t) + if bucket is not None: + bucket.discard(source_path) + if not bucket: + del self._links_backward[t] + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + sources.discard(source_path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + raw = self._raw_links.get(source_path) + if not raw: + return + + if resolver is None: + note_paths = {p for p, r in self._notes.items() if r.type == "note"} + resolver = _Resolver(note_paths) + + source_folder = str(Path(source_path).parent).replace("\\", "/") + if source_folder == ".": + source_folder = "" + + targets: Dict[str, str] = {} + + # Wikilinks first so they win the "first wins" dedup that legacy + # /api/graph also implements. + for target in raw.get("wikilinks", []): + resolved = resolver.resolve_wikilink(target, source_folder) + if resolved and resolved != source_path and resolved not in targets: + targets[resolved] = "wikilink" + # Loose wikilink reverse index — populated regardless of strict + # resolution success, because the legacy get_backlinks uses + # token-stem matching independent of where the link actually + # navigates. + t_lower = target.strip().lower() + if t_lower: + self._wikilink_tokens.setdefault(t_lower, set()).add(source_path) + t_no_ext = t_lower[:-3] if t_lower.endswith(".md") else t_lower + if t_no_ext != t_lower: + self._wikilink_tokens.setdefault(t_no_ext, set()).add(source_path) + + for link_path in raw.get("mdlinks", []): + resolved = resolver.resolve_mdlink(link_path, source_folder) + if resolved and resolved != source_path and resolved not in targets: + targets[resolved] = "markdown" + + if targets: + self._links_forward[source_path] = targets + for t in targets: + self._links_backward.setdefault(t, set()).add(source_path) + + def _rename_note_unlocked(self, old_path: str, new_path: str) -> None: + """Same as rename_note() but assumes the caller already holds the lock. + Used by folder-prefix rename to avoid re-acquiring the lock per file.""" + if old_path == new_path: + return + old_record = self._notes.pop(old_path, None) + if old_record is None: + return + folder = str(Path(new_path).parent).replace("\\", "/") + new_record = NoteRecord( + path=new_path, + name=Path(new_path).stem, + folder="" if folder == "." else folder, + modified=old_record.modified, + size=old_record.size, + type=old_record.type, + mtime=old_record.mtime, + tags=old_record.tags, + ) + self._notes[new_path] = new_record + if new_record.folder: + self._folders.add(new_record.folder) + + if old_path in self._tags_forward: + tags = self._tags_forward.pop(old_path) + self._tags_forward[new_path] = tags + for t in tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(old_path) + bucket.add(new_path) + + for paths in self._search_terms.values(): + if old_path in paths: + paths.discard(old_path) + paths.add(new_path) + + if old_path in self._raw_links: + self._raw_links[new_path] = self._raw_links.pop(old_path) + + self._links_forward.pop(old_path, None) + for bucket in self._links_backward.values(): + bucket.discard(old_path) + old_backlinks = self._links_backward.pop(old_path, set()) + for src in old_backlinks: + fwd = self._links_forward.get(src) + if fwd is not None and old_path in fwd: + del fwd[old_path] + if not fwd: + del self._links_forward[src] + + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + if old_path in sources: + sources.discard(old_path) + sources.add(new_path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + def _remove_note_unlocked(self, path: str) -> None: + """Same as remove_note() but assumes the caller already holds the lock.""" + old_record = self._notes.pop(path, None) + if old_record is None: + return + for t in old_record.tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._tags_backward[t] + self._tags_forward.pop(path, None) + self._raw_links.pop(path, None) + old_targets = self._links_forward.pop(path, {}) + for t in old_targets: + bucket = self._links_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._links_backward[t] + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + sources.discard(path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + sources_pointing_here = self._links_backward.pop(path, set()) + for src in sources_pointing_here: + fwd = self._links_forward.get(src) + if fwd is not None and path in fwd: + del fwd[path] + if not fwd: + del self._links_forward[src] + empty_terms = [] + for term, paths in self._search_terms.items(): + if path in paths: + paths.discard(path) + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + + +# ============================================================================ +# Resolver — link-target matching (mirrors legacy /api/graph rules exactly) +# ============================================================================ + +class _Resolver: + """Build the lookup tables once per resolution batch, then call + resolve_* repeatedly. Reused across sources within a single rebuild so + we don't pay O(N) to construct it on every source.""" + + def __init__(self, all_notes: Set[str]) -> None: + self.note_paths: Set[str] = set(all_notes) + self.note_paths_lower: Dict[str, str] = {} + self.note_names: Dict[str, str] = {} + for p in all_notes: + self.note_paths_lower[p.lower()] = p + if p.endswith(".md"): + self.note_paths_lower[p[:-3].lower()] = p + stem = Path(p).stem + self.note_names[stem.lower()] = p + self.note_names[Path(p).name.lower()] = p + + def resolve_wikilink(self, target: str, source_folder: str) -> Optional[str]: + target = target.strip() + if not target: + return None + target_lower = target.lower() + + # 1. Relative to source folder (only for bare names with no slash). + if source_folder and "/" not in target: + relative_path = f"{source_folder}/{target}" + relative_path_lower = relative_path.lower() + if relative_path in self.note_paths: + return relative_path if relative_path.endswith(".md") else relative_path + ".md" + if relative_path + ".md" in self.note_paths: + return relative_path + ".md" + if relative_path_lower in self.note_paths_lower: + return self.note_paths_lower[relative_path_lower] + if (relative_path_lower + ".md") in self.note_paths_lower: + return self.note_paths_lower[relative_path_lower + ".md"] + + if target in self.note_paths: + return target if target.endswith(".md") else target + ".md" + if (target + ".md") in self.note_paths: + return target + ".md" + if target_lower in self.note_paths_lower: + return self.note_paths_lower[target_lower] + if (target_lower + ".md") in self.note_paths_lower: + return self.note_paths_lower[target_lower + ".md"] + if target_lower in self.note_names: + return self.note_names[target_lower] + return None + + def resolve_mdlink(self, link_path: str, source_folder: str) -> Optional[str]: + if not link_path: + return None + link_path = link_path.split("#")[0] + if not link_path: + return None + link_path = urllib.parse.unquote(link_path) + if link_path.startswith("./"): + link_path = link_path[2:] + link_path_with_md = link_path if link_path.endswith(".md") else link_path + ".md" + + if source_folder and not link_path.startswith("/"): + relative_path = f"{source_folder}/{link_path}" + relative_path_with_md = f"{source_folder}/{link_path_with_md}" + relative_path_lower = relative_path.lower() + relative_path_with_md_lower = relative_path_with_md.lower() + if relative_path in self.note_paths: + return relative_path if relative_path.endswith(".md") else relative_path + ".md" + if relative_path_with_md in self.note_paths: + return relative_path_with_md + if relative_path_lower in self.note_paths_lower: + return self.note_paths_lower[relative_path_lower] + if relative_path_with_md_lower in self.note_paths_lower: + return self.note_paths_lower[relative_path_with_md_lower] + + link_path_lower = link_path.lower() + link_path_with_md_lower = link_path_with_md.lower() + if link_path in self.note_paths: + return link_path if link_path.endswith(".md") else link_path + ".md" + if link_path_with_md in self.note_paths: + return link_path_with_md + if link_path_lower in self.note_paths_lower: + return self.note_paths_lower[link_path_lower] + if link_path_with_md_lower in self.note_paths_lower: + return self.note_paths_lower[link_path_with_md_lower] + return None + + +# ============================================================================ +# Internal helpers +# ============================================================================ + +def _fingerprint( + notes_meta: List[NoteRecord], + sources_raw: Dict[str, Dict[str, List[str]]], +) -> int: + """Cheap content hash of a scan result. Used to short-circuit bulk_set + when nothing has changed in the vault.""" + notes_fp = hash(frozenset((n.path, n.mtime) for n in notes_meta)) + raw_items = ( + (src, tuple(raw.get("wikilinks", [])), tuple(raw.get("mdlinks", []))) + for src, raw in sources_raw.items() + ) + return hash((notes_fp, hash(frozenset(raw_items)))) + + +# ============================================================================ +# Module-level singleton + facade +# +# Every utils.py / main.py call site uses these — never instantiates its own +# index. The facade functions are no-ops (or return None) when +# USE_NOTE_INDEX is False, so call sites don't have to repeat the flag check. +# ============================================================================ + +_index = NoteIndex() + + +def get_index() -> NoteIndex: + return _index + + +# --- Lifecycle facade (one-line calls from utils.py mutators) ---------------- + +def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None: + """A note was created or updated on disk. Patch the index in place.""" + if not USE_NOTE_INDEX: + return + try: + rel_path = full_path.relative_to(Path(notes_dir)).as_posix() + st = full_path.stat() + folder = str(Path(rel_path).parent).replace("\\", "/") + record = NoteRecord( + path=rel_path, + name=Path(rel_path).stem, + folder="" if folder == "." else folder, + modified=datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), + size=st.st_size, + type="note", + mtime=st.st_mtime, + tags=tuple(_parse_tags_for_record(content)), + ) + raw_links = extract_links_from_content(content) + _index.update_note(record, raw_links, content=content) + except Exception as e: + print(f"note_index: on_note_saved failed for {full_path}: {e}") + + +def on_note_deleted(notes_dir: str, full_path: Path) -> None: + if not USE_NOTE_INDEX: + return + try: + rel_path = full_path.relative_to(Path(notes_dir)).as_posix() + _index.remove_note(rel_path) + except Exception as e: + print(f"note_index: on_note_deleted failed for {full_path}: {e}") + + +def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: + if not USE_NOTE_INDEX: + return + try: + base = Path(notes_dir) + _index.rename_note( + old_full_path.relative_to(base).as_posix(), + new_full_path.relative_to(base).as_posix(), + ) + except Exception as e: + print(f"note_index: on_note_renamed failed: {e}") + + +def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: + """A folder was moved/renamed. Re-keys every entry under it (cheap, + no disk reads) rather than invalidating the whole index.""" + if not USE_NOTE_INDEX: + return + try: + base = Path(notes_dir) + _index.rename_folder_prefix( + old_full_path.relative_to(base).as_posix(), + new_full_path.relative_to(base).as_posix(), + ) + except Exception as e: + print(f"note_index: on_folder_renamed failed: {e}") + _index.invalidate() # fail-safe + + +def on_folder_deleted(notes_dir: str, full_path: Path) -> None: + if not USE_NOTE_INDEX: + return + try: + rel_prefix = full_path.relative_to(Path(notes_dir)).as_posix() + _index.remove_folder_prefix(rel_prefix) + except Exception as e: + print(f"note_index: on_folder_deleted failed: {e}") + _index.invalidate() + + +def populate_from_scan( + notes_meta: List[NoteRecord], + folders: Iterable[str], + sources_raw: Dict[str, Dict[str, List[str]]], +) -> None: + """Bulk-replace the index from a fresh scan. No-op when off.""" + if not USE_NOTE_INDEX: + return + try: + _index.bulk_set(notes_meta, folders, sources_raw) + except Exception as e: + print(f"note_index: populate_from_scan failed: {e}") + + +def ensure_search_index(notes_dir: str) -> bool: + """Lazily build the full-text search index on first /api/search request. + Cheap after the first call. Returns True if the index is now usable.""" + if not USE_NOTE_INDEX: + return False + if not _index.is_built(): + return False + return _index.ensure_search_index_built(notes_dir) + + +# --- Read facade (returns None when off — callers fall through to legacy) ---- + +def try_backlink_candidates(target_path: str) -> Optional[Set[str]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_backlink_candidate_sources(target_path) + + +def try_graph_data() -> Optional[Tuple[List[str], List[Tuple[str, str, str]]]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_graph_data() + + +def try_search_candidates(query: str) -> Optional[Set[str]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_search_candidates(query) + + +def try_all_tags() -> Optional[Dict[str, int]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_all_tags() + + +def try_notes_by_tag(tag: str) -> Optional[Set[str]]: + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.get_paths_for_tag(tag) + + +def try_get_extraction( + rel_path: str, + mtime: float, +) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: + """Serve (tags, raw_links) from the index for a single file, when fresh. + + Used by scan_notes_fast_walk to skip the cold per-file read after a + snapshot load. Returns None when the index can't help (off, not built, + file unknown, mtime mismatch) — caller falls back to reading the file. + """ + if not USE_NOTE_INDEX or not _index.is_built(): + return None + return _index.try_get_extraction(rel_path, mtime) + + +# --- Observability ----------------------------------------------------------- + +def stats() -> Dict[str, Any]: + return _index.stats() + + +# ============================================================================ +# Late import to avoid a circular dependency with utils.parse_tags. +# We need tag parsing inside on_note_saved but utils.py imports this module. +# ============================================================================ + +def _parse_tags_for_record(content: str) -> List[str]: + """Thin shim to utils.parse_tags. Late-bound so this module imports cleanly + before utils.py is loaded.""" + from .utils import parse_tags + return parse_tags(content) diff --git a/backend/utils.py b/backend/utils.py index 224e604..f3902a5 100644 --- a/backend/utils.py +++ b/backend/utils.py @@ -1,5 +1,9 @@ """ -Utility functions for file operations, search, and markdown processing +Utility functions for file operations, search, and markdown processing. + +Heavy work (backlinks, graph, full-text search, tag aggregation) is delegated +to the in-memory index in note_index.py — every facade function there is a +no-op when USE_NOTE_INDEX is False, so call sites here stay flag-free. """ import os @@ -7,6 +11,7 @@ import re import shutil import threading import time +import traceback import urllib.parse from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass @@ -14,6 +19,9 @@ from pathlib import Path from typing import List, Dict, Optional, Tuple, Any, TypeVar, Callable from datetime import datetime, timezone +from . import note_index +from .note_index import NoteRecord, extract_links_from_content + # ============================================================================ # Pagination Support @@ -112,14 +120,23 @@ def paginate( ) -# In-memory cache for parsed tags -# Format: {file_path: (mtime, tags)} -_tag_cache: Dict[str, Tuple[float, List[str]]] = {} - -# Notes tree scan cache (TTL). +# ============================================================================ +# In-memory caches # -# This avoids repeated full-directory walks when multiple endpoints (or the UI) -# request indexes in quick succession. +# Two layers: +# _tag_cache / _links_cache : per-file, mtime-keyed. Avoid re-parsing the +# same file on warm scans. Used by get_tags_and_links_cached. +# _SCAN_WALK_CACHE : per-scan TTL cache. Avoids repeated full- +# directory walks when several endpoints fire in quick succession. +# +# All three are invalidated on every mutation (save/delete/move). The +# note_index in note_index.py is the system of record for derived data +# (links, backlinks, tags-by-name, search) — these caches just exist to +# avoid double-reading files we already parsed. +# ============================================================================ + +_tag_cache: Dict[str, Tuple[float, List[str]]] = {} +_links_cache: Dict[str, Tuple[float, Dict[str, List[str]]]] = {} _SCAN_WALK_CACHE_LOCK = threading.Lock() _SCAN_WALK_CACHE_TTL_SECONDS = 1.0 @@ -145,6 +162,66 @@ def _scan_cache_set(key: Tuple[str, bool], value: Tuple[List[Dict], List[str]]) _SCAN_WALK_CACHE[key] = (time.monotonic(), value) +def _scan_cache_invalidate() -> None: + """Drop the TTL scan cache. Called from every mutation handler so callers + that hit the cache immediately after a save/delete/move see the new state + instead of stale data from the previous walk.""" + with _SCAN_WALK_CACHE_LOCK: + _SCAN_WALK_CACHE.clear() + + +def get_tags_and_links_cached( + file_path: Path, + rel_path: Optional[str] = None, +) -> Tuple[List[str], Dict[str, List[str]]]: + """Single-read fused extraction of tags + raw links, cached by mtime. + + Three-tier lookup, cheapest first: + 1. In-process per-file caches (warm scans on the live server) + 2. NoteIndex (warm after snapshot load — avoids file reads on cold start) + 3. Read the file from disk + + rel_path enables tier 2. scan_notes_fast_walk passes it; ad-hoc callers + can omit it and just get tiers 1 + 3. + """ + try: + mtime = file_path.stat().st_mtime + file_key = str(file_path) + + tag_cached = _tag_cache.get(file_key) + link_cached = _links_cache.get(file_key) + tags_ok = tag_cached is not None and tag_cached[0] == mtime + links_ok = link_cached is not None and link_cached[0] == mtime + + if tags_ok and links_ok: + return tag_cached[1], link_cached[1] + + # Try the NoteIndex before reading the file. On a snapshot-warm + # startup, this turns a 10K-file cold scan into a no-I/O walk. + if rel_path is not None: + indexed = note_index.try_get_extraction(rel_path, mtime) + if indexed is not None: + idx_tags, idx_links = indexed + _tag_cache[file_key] = (mtime, idx_tags) + _links_cache[file_key] = (mtime, idx_links) + return idx_tags, idx_links + + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + tags = tag_cached[1] if tags_ok else parse_tags(content) + links = link_cached[1] if links_ok else extract_links_from_content(content) + + if not tags_ok: + _tag_cache[file_key] = (mtime, tags) + if not links_ok: + _links_cache[file_key] = (mtime, links) + + return tags, links + except Exception: + return [], {"wikilinks": [], "mdlinks": []} + + def validate_path_security(notes_dir: str, path: Path) -> bool: """ Validate that a path is within the notes directory (security check). @@ -176,15 +253,12 @@ def ensure_directories(config: dict): def create_folder(notes_dir: str, folder_path: str) -> bool: - """Create a new folder in the notes directory""" + """Create a new folder in the notes directory.""" full_path = Path(notes_dir) / folder_path - - # Security check if not validate_path_security(notes_dir, full_path): return False - full_path.mkdir(parents=True, exist_ok=True) - + _scan_cache_invalidate() return True @@ -219,113 +293,99 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: _scan_cache_set(cache_key, normalized_value) return normalized_value - # TODO: remove this flag and the legacy branch below once the parallel - # path has been validated in production with large vaults. To revert to - # the pre-parallel behavior, flip _USE_PARALLEL_TAG_SCAN to False. - _USE_PARALLEL_TAG_SCAN = True - + # Walk the vault once. Tag/link extraction is deferred out of the walk + # (the expensive part is the file I/O) and parallelized below across files. notes: List[Dict] = [] folders_set = set() + md_to_extract: List[Tuple[int, Path, str]] = [] # (note_index, full_path, rel_path) - if not _USE_PARALLEL_TAG_SCAN: - # ===== LEGACY: sequential tag extraction inline with the walk ===== - for root, dirnames, filenames in os.walk(notes_path): - # Skip descending into dot-dirs - dirnames[:] = [d for d in dirnames if not d.startswith('.')] + for root, dirnames, filenames in os.walk(notes_path): + dirnames[:] = [d for d in dirnames if not d.startswith('.')] - root_path = Path(root) - rel_folder = root_path.relative_to(notes_path).as_posix() - if rel_folder != "." and not rel_folder.startswith('.'): - folders_set.add(rel_folder) + root_path = Path(root) + rel_folder = root_path.relative_to(notes_path).as_posix() + if rel_folder != "." and not rel_folder.startswith('.'): + folders_set.add(rel_folder) - for filename in filenames: - if filename.startswith('.'): - continue + for filename in filenames: + if filename.startswith('.'): + continue - full_path = root_path / filename - try: - st = full_path.stat() - except OSError: - continue + full_path = root_path / filename + try: + st = full_path.stat() + except OSError: + continue - relative_path = full_path.relative_to(notes_path) - media_type = get_media_type(filename) if include_media else None - is_markdown = full_path.suffix.lower() == '.md' - should_include = is_markdown or (include_media and media_type is not None) + relative_path = full_path.relative_to(notes_path) + media_type = get_media_type(filename) if include_media else None + is_markdown = full_path.suffix.lower() == '.md' + should_include = is_markdown or (include_media and media_type is not None) + if not should_include: + continue - if not should_include: - continue + folder = relative_path.parent.as_posix() + rel_str = relative_path.as_posix() + notes.append({ + "name": full_path.stem, + "path": rel_str, + "folder": "" if folder == "." else folder, + "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), + "size": st.st_size, + "type": media_type if media_type else "note", + "tags": [], + "_mtime": st.st_mtime, # internal — popped before returning + }) + if is_markdown: + md_to_extract.append((len(notes) - 1, full_path, rel_str)) - folder = relative_path.parent.as_posix() - tags = get_tags_cached(full_path) if is_markdown else [] - notes.append({ - "name": full_path.stem, - "path": relative_path.as_posix(), - "folder": "" if folder == "." else folder, - "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), - "size": st.st_size, - "type": media_type if media_type else "note", - "tags": tags, - }) - else: - # ===== NEW: tag extraction deferred out of the walk and parallelized ===== - # Indices of markdown entries in `notes` plus their full paths, so we can - # defer tag extraction (the expensive part) out of the walk and parallelize - # it across files. Tags are filled in after the walk completes. - md_to_tag: List[Tuple[int, Path]] = [] + # Fused tag + raw-link extraction. mtime-keyed cache makes warm scans + # mostly free; cold scans win from parallel I/O. + # + # We don't read full content here — that would double the parsing cost + # for every scan, and the search index is the only thing that needs it. + # Search is built lazily on the first search request (see + # note_index.ensure_search_index_built), which keeps startup fast and + # only pays the cost when a user actually searches. + sources_raw: Dict[str, Dict[str, List[str]]] = {} + if md_to_extract: + # Pass rel_path so each call can hit the NoteIndex before falling + # back to a file read — critical for fast startup after a snapshot + # load on large vaults. + path_pairs = [(full, rel) for (_, full, rel) in md_to_extract] + if len(md_to_extract) >= 50: + workers = min(8, (os.cpu_count() or 4)) + with ThreadPoolExecutor(max_workers=workers) as ex: + extraction = list(ex.map( + lambda pair: get_tags_and_links_cached(pair[0], pair[1]), + path_pairs, + )) + else: + extraction = [get_tags_and_links_cached(full, rel) for (full, rel) in path_pairs] - for root, dirnames, filenames in os.walk(notes_path): - dirnames[:] = [d for d in dirnames if not d.startswith('.')] + for (idx, _full, rel_str), (tags, links) in zip(md_to_extract, extraction): + notes[idx]["tags"] = tags + sources_raw[rel_str] = links - root_path = Path(root) - rel_folder = root_path.relative_to(notes_path).as_posix() - if rel_folder != "." and not rel_folder.startswith('.'): - folders_set.add(rel_folder) + # Populate the unified index. Cheap when the fingerprint matches (warm + # scan, nothing changed) — short-circuits internally. No-op when off. + notes_meta = [ + NoteRecord( + path=n["path"], + name=n["name"], + folder=n["folder"], + modified=n["modified"], + size=n["size"], + type=n["type"], + mtime=n["_mtime"], + tags=tuple(n["tags"]), + ) + for n in notes + ] + note_index.populate_from_scan(notes_meta, folders_set, sources_raw) - for filename in filenames: - if filename.startswith('.'): - continue - - full_path = root_path / filename - try: - st = full_path.stat() - except OSError: - continue - - relative_path = full_path.relative_to(notes_path) - media_type = get_media_type(filename) if include_media else None - is_markdown = full_path.suffix.lower() == '.md' - should_include = is_markdown or (include_media and media_type is not None) - - if not should_include: - continue - - folder = relative_path.parent.as_posix() - notes.append({ - "name": full_path.stem, - "path": relative_path.as_posix(), - "folder": "" if folder == "." else folder, - "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), - "size": st.st_size, - "type": media_type if media_type else "note", - "tags": [], - }) - if is_markdown: - md_to_tag.append((len(notes) - 1, full_path)) - - # Parallelize for large vaults (sequential branch avoids thread-spawn - # overhead for small vaults). get_tags_cached is mtime-keyed so this is - # mostly free on warm cache; the win is the cold first call. - if md_to_tag: - if len(md_to_tag) >= 50: - workers = min(8, (os.cpu_count() or 4)) - paths = [p for _, p in md_to_tag] - with ThreadPoolExecutor(max_workers=workers) as ex: - for (idx, _), tags in zip(md_to_tag, ex.map(get_tags_cached, paths)): - notes[idx]["tags"] = tags - else: - for idx, p in md_to_tag: - notes[idx]["tags"] = get_tags_cached(p) + for n in notes: + n.pop("_mtime", None) value = (sorted(notes, key=lambda x: x.get('modified', ''), reverse=True), sorted(folders_set)) if use_cache: @@ -333,132 +393,106 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: return value def move_note(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]: - """Move a note to a different location - - Returns: - Tuple of (success: bool, error_message: str) - """ + """Move a note. Returns (success, error_message).""" old_full_path = Path(notes_dir) / old_path new_full_path = Path(notes_dir) / new_path - - # Security checks + if not validate_path_security(notes_dir, old_full_path): return False, "Invalid source path" if not validate_path_security(notes_dir, new_full_path): return False, "Invalid destination path" - if not old_full_path.exists(): return False, f"Source note does not exist: {old_path}" - - # Check if target already exists (prevent overwriting) if new_full_path.exists(): return False, f"A note already exists at: {new_path}" - - # Invalidate cache for old path - old_key = str(old_full_path) - if old_key in _tag_cache: - del _tag_cache[old_key] - + + _drop_path_caches(old_full_path) + try: - # Create parent directory if needed new_full_path.parent.mkdir(parents=True, exist_ok=True) - - # Move the file old_full_path.rename(new_full_path) except Exception as e: return False, f"Failed to move file: {str(e)}" - - # Note: We don't automatically delete empty folders to preserve user's folder structure - + + note_index.on_note_renamed(notes_dir, old_full_path, new_full_path) + _scan_cache_invalidate() return True, "" def move_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]: - """Move a folder to a different location - - Returns: - Tuple of (success: bool, error_message: str) - """ - import shutil - + """Move a folder. Returns (success, error_message).""" old_full_path = Path(notes_dir) / old_path new_full_path = Path(notes_dir) / new_path - - # Security checks + if not validate_path_security(notes_dir, old_full_path): return False, "Invalid source path" if not validate_path_security(notes_dir, new_full_path): return False, "Invalid destination path" - if not old_full_path.exists() or not old_full_path.is_dir(): return False, f"Source folder does not exist: {old_path}" - - # Check if target already exists if new_full_path.exists(): return False, f"A folder already exists at: {new_path}" - - # Invalidate cache for all notes in this folder - global _tag_cache - old_path_str = str(old_full_path) - keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(old_path_str)] - for key in keys_to_delete: - del _tag_cache[key] - + + _drop_prefix_caches(old_full_path) + try: - # Create parent directory if needed new_full_path.parent.mkdir(parents=True, exist_ok=True) - - # Move the folder shutil.move(str(old_full_path), str(new_full_path)) except Exception as e: return False, f"Failed to move folder: {str(e)}" - - # Note: We don't automatically delete empty folders to preserve user's folder structure - + + note_index.on_folder_renamed(notes_dir, old_full_path, new_full_path) + _scan_cache_invalidate() return True, "" def rename_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]: - """Rename a folder (same as move but for clarity)""" + """Rename a folder (same as move, named for clarity).""" return move_folder(notes_dir, old_path, new_path) def delete_folder(notes_dir: str, folder_path: str) -> bool: - """Delete a folder and all its contents""" + """Delete a folder and all its contents.""" try: full_path = Path(notes_dir) / folder_path - - # Security check: ensure the path is within notes_dir + if not validate_path_security(notes_dir, full_path): print(f"Security: Path is outside notes directory: {full_path}") return False - if not full_path.exists(): print(f"Folder does not exist: {full_path}") return False - if not full_path.is_dir(): print(f"Path is not a directory: {full_path}") return False - - # Invalidate cache for all notes in this folder - global _tag_cache - folder_path_str = str(full_path) - keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(folder_path_str)] - for key in keys_to_delete: - del _tag_cache[key] - - # Delete the folder and all its contents + + _drop_prefix_caches(full_path) shutil.rmtree(full_path) - print(f"Successfully deleted folder: {full_path}") + note_index.on_folder_deleted(notes_dir, full_path) + _scan_cache_invalidate() return True except Exception as e: print(f"Error deleting folder '{folder_path}': {e}") - import traceback traceback.print_exc() return False +def _drop_path_caches(full_path: Path) -> None: + """Evict the per-file mtime caches for a single note.""" + key = str(full_path) + _tag_cache.pop(key, None) + _links_cache.pop(key, None) + + +def _drop_prefix_caches(folder_full_path: Path) -> None: + """Evict per-file mtime caches for every entry under a folder.""" + prefix = str(folder_full_path) + for k in [k for k in _tag_cache if k.startswith(prefix)]: + _tag_cache.pop(k, None) + for k in [k for k in _links_cache if k.startswith(prefix)]: + _links_cache.pop(k, None) + + def get_note_content(notes_dir: str, note_path: str) -> Optional[str]: @@ -477,114 +511,115 @@ def get_note_content(notes_dir: str, note_path: str) -> Optional[str]: def save_note(notes_dir: str, note_path: str, content: str) -> bool: - """Save or update a note""" + """Save or update a note.""" full_path = Path(notes_dir) / note_path - - # Ensure .md extension if not note_path.endswith('.md'): full_path = full_path.with_suffix('.md') - - # Security check + if not validate_path_security(notes_dir, full_path): return False - - # Create parent directories if needed + full_path.parent.mkdir(parents=True, exist_ok=True) - with open(full_path, 'w', encoding='utf-8') as f: f.write(content) - + + # Refresh the per-file mtime caches with what we just wrote, so the next + # scan doesn't re-parse this file. extract_links + parse_tags + save are + # tiny relative to the file write. + try: + mtime = full_path.stat().st_mtime + file_key = str(full_path) + _tag_cache[file_key] = (mtime, parse_tags(content)) + _links_cache[file_key] = (mtime, extract_links_from_content(content)) + except Exception: + pass # caches are best-effort + + note_index.on_note_saved(notes_dir, full_path, content) + _scan_cache_invalidate() return True def delete_note(notes_dir: str, note_path: str) -> bool: - """Delete a note""" + """Delete a note.""" full_path = Path(notes_dir) / note_path - if not full_path.exists(): return False - - # Security check if not validate_path_security(notes_dir, full_path): return False - - # Invalidate cache for this note - file_key = str(full_path) - if file_key in _tag_cache: - del _tag_cache[file_key] - + + _drop_path_caches(full_path) full_path.unlink() - - # Note: We don't automatically delete empty folders to preserve user's folder structure - + note_index.on_note_deleted(notes_dir, full_path) + _scan_cache_invalidate() return True def search_notes(notes_dir: str, query: str) -> List[Dict]: - """ - Full-text search through note contents only. - Does NOT search in file names, folder names, or paths - only note content. - Uses character-based context extraction with highlighted matches. + """Full-text search through note contents. + + Only searches inside note content (not filenames or paths). The index, when + available, narrows down the candidate files via a token-AND match on its + inverted index — then the existing per-line snippet extraction runs on the + narrow set. Output is byte-identical to a full scan. """ from html import escape - results = [] + results: List[Dict] = [] notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + # Build the search index lazily on the first search request. Subsequent + # searches hit the warm index; the first one pays the cost (read every + # file once) and is roughly as slow as the legacy full scan would be. + note_index.ensure_search_index(notes_dir) + candidates = note_index.try_search_candidates(query) + for note in notes: - md_file = Path(notes_dir) / note["path"] + path = note["path"] + if candidates is not None and path not in candidates: + continue + md_file = Path(notes_dir) / path try: with open(md_file, 'r', encoding='utf-8') as f: content = f.read() - - # Find all matches using regex (case-insensitive) + matches = list(re.finditer(re.escape(query), content, re.IGNORECASE)) - - if matches: - matched_lines = [] - - for match in matches[:3]: # Limit to 3 matches per file - start_index = match.start() - end_index = match.end() - matched_text = match.group() # Preserve original case - - # Create slice window: ±15 characters around match - context_start = max(0, start_index - 15) - context_end = min(len(content), end_index + 15) - - # Extract and clean parts (newlines → spaces) - before = escape(content[context_start:start_index].replace('\n', ' ')) - after = escape(content[end_index:context_end].replace('\n', ' ')) - matched_clean = escape(matched_text.replace('\n', ' ')) - - # Build snippet with highlight (styled via CSS) - snippet = f'{before}{matched_clean}{after}' - - # Add ellipsis if truncated at start - if context_start > 0: - snippet = '...' + snippet - - # Add ellipsis if truncated at end - if context_end < len(content): - snippet = snippet + '...' - - # Calculate line number by counting newlines up to match start - line_number = content.count('\n', 0, start_index) + 1 - - matched_lines.append({ - "line_number": line_number, - "context": snippet - }) - - relative_path = Path(note["path"]) - results.append({ - "name": md_file.stem, - "path": str(relative_path.as_posix()), - "folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "", - "matches": matched_lines + if not matches: + continue + + matched_lines = [] + for match in matches[:3]: + start_index = match.start() + end_index = match.end() + matched_text = match.group() + + context_start = max(0, start_index - 15) + context_end = min(len(content), end_index + 15) + + before = escape(content[context_start:start_index].replace('\n', ' ')) + after = escape(content[end_index:context_end].replace('\n', ' ')) + matched_clean = escape(matched_text.replace('\n', ' ')) + + snippet = f'{before}{matched_clean}{after}' + if context_start > 0: + snippet = '...' + snippet + if context_end < len(content): + snippet = snippet + '...' + + line_number = content.count('\n', 0, start_index) + 1 + matched_lines.append({ + "line_number": line_number, + "context": snippet, }) + + relative_path = Path(path) + results.append({ + "name": md_file.stem, + "path": str(relative_path.as_posix()), + "folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "", + "matches": matched_lines, + }) except Exception: continue - + return results @@ -852,105 +887,66 @@ def parse_tags(content: str) -> List[str]: return [] -def get_tags_cached(file_path: Path) -> List[str]: - """ - Get tags for a file with caching based on modification time. - - Args: - file_path: Path to the markdown file - - Returns: - List of tags from the file (cached if mtime unchanged) - """ - global _tag_cache - - try: - # Get current modification time - mtime = file_path.stat().st_mtime - file_key = str(file_path) - - # Check cache - if file_key in _tag_cache: - cached_mtime, cached_tags = _tag_cache[file_key] - if cached_mtime == mtime: - # Cache hit! Return cached tags - return cached_tags - - # Cache miss or stale - parse tags - with open(file_path, 'r', encoding='utf-8') as f: - content = f.read() - tags = parse_tags(content) - - # Update cache - _tag_cache[file_key] = (mtime, tags) - return tags - - except Exception: - # If anything fails, return empty list - return [] - - -def clear_tag_cache(): - """Clear the tag cache (useful for testing or manual cache invalidation)""" - global _tag_cache - _tag_cache.clear() - - def get_all_tags(notes_dir: str) -> Dict[str, int]: - """ - Get all tags used across all notes with their count (cached). - - Args: - notes_dir: Directory containing notes - - Returns: - Dictionary mapping tag names to note counts - """ - tag_counts = {} - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + """All tags in the vault with note counts. Index-backed when available.""" + # Fast path FIRST — when the index is ready it knows every tag count, + # so we skip the scan entirely (no disk walk, no JSON of 10K notes). + indexed = note_index.try_all_tags() + if indexed is not None: + return indexed + # Legacy fallback: scan the vault and aggregate. + notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + tag_counts: Dict[str, int] = {} for note in notes: - md_file = Path(notes_dir) / note["path"] - # Get tags using cache - tags = get_tags_cached(md_file) - - for tag in tags: + for tag in note.get("tags", []): tag_counts[tag] = tag_counts.get(tag, 0) + 1 - return dict(sorted(tag_counts.items())) def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]: - """ - Get all notes that have a specific tag (cached). - - Args: - notes_dir: Directory containing notes - tag: Tag to filter by (case-insensitive) - - Returns: - List of note dictionaries matching the tag - """ - matching_notes = [] + """All notes carrying `tag` (case-insensitive). Index-backed when available.""" tag_lower = tag.lower() - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + # Fast path FIRST: every field we need lives in the NoteRecord. Skip + # the vault scan when the index can answer directly. + indexed_paths = note_index.try_notes_by_tag(tag_lower) + if indexed_paths is not None: + idx = note_index.get_index() + records = [idx.get_note_record(p) for p in indexed_paths] + matching = [ + { + "name": r.name, + "path": r.path, + "folder": r.folder, + "modified": r.modified, + "size": r.size, + "tags": list(r.tags), + } + for r in records + if r is not None and r.type == "note" + ] + # Match legacy sort order (scan returns notes sorted by modified desc). + matching.sort(key=lambda n: n["modified"], reverse=True) + return matching + + # Legacy fallback. + notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + matching: List[Dict] = [] for note in notes: - md_file = Path(notes_dir) / note["path"] - # Get tags using cache - tags = get_tags_cached(md_file) - + if note.get("type") != "note": + continue + tags = note.get("tags", []) if tag_lower in tags: - matching_notes.append({ + matching.append({ "name": note["name"], "path": note["path"], "folder": note["folder"], "modified": note["modified"], "size": note["size"], - "tags": tags + "tags": tags, }) - - return matching_notes + return matching # ============================================================================ @@ -1107,140 +1103,138 @@ def apply_template_placeholders(content: str, note_path: str) -> str: return result +def _extract_backlink_references( + notes_dir: str, + source_path: str, + target_path: str, + target_path_lower: str, + target_path_no_ext_lower: str, + wikilink_refs: set, +) -> List[Dict]: + """Read one source file and pull line-level references that point to target. + + Same regex + resolution rules as the legacy get_backlinks loop body; just + factored out so both the index-driven and the legacy code paths use the + exact same context-extraction logic. + """ + source_folder = str(Path(source_path).parent).replace('\\', '/') + if source_folder == '.': + source_folder = '' + + full_path = Path(notes_dir) / source_path + try: + with open(full_path, 'r', encoding='utf-8') as f: + content = f.read() + except Exception: + return [] + + lines = content.split('\n') + found_links: List[Dict] = [] + + for line_num, line in enumerate(lines, 1): + for match in re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line): + link_target = match.group(1).strip().lower() + link_target_no_ext = link_target.replace('.md', '') + if link_target in wikilink_refs or link_target_no_ext in wikilink_refs: + start = max(0, match.start() - 30) + end = min(len(line), match.end() + 30) + context = line[start:end] + if start > 0: + context = '...' + context + if end < len(line): + context = context + '...' + found_links.append({ + "line_number": line_num, + "context": context, + "type": "wikilink", + }) + + for match in re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line): + link_path = match.group(2).split('#')[0] + if not link_path: + continue + link_path = urllib.parse.unquote(link_path) + if link_path.startswith('./'): + link_path = link_path[2:] + link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md' + + resolved_path = None + if source_folder and not link_path.startswith('/'): + relative_path = f"{source_folder}/{link_path_with_md}" + if relative_path.lower() == target_path_lower: + resolved_path = target_path + elif f"{source_folder}/{link_path}".lower() == target_path_no_ext_lower: + resolved_path = target_path + if not resolved_path: + if link_path_with_md.lower() == target_path_lower: + resolved_path = target_path + elif link_path.lower() == target_path_no_ext_lower: + resolved_path = target_path + + if resolved_path: + start = max(0, match.start() - 30) + end = min(len(line), match.end() + 30) + context = line[start:end] + if start > 0: + context = '...' + context + if end < len(line): + context = context + '...' + found_links.append({ + "line_number": line_num, + "context": context, + "type": "markdown", + }) + + return found_links + + def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]: - """ - Find all notes that link TO the specified note (reverse links / backlinks). + """Find all notes that link TO the specified note. - Args: - notes_dir: Base directory containing notes - target_note_path: Path of the note to find backlinks for - - Returns: - List of backlink objects with path, context, and line_number + Index-backed when available: the index hands us a superset of candidate + source files (O(1)), we read only those, then the per-line matcher + filters down to true matches with line-level context. Output is + byte-identical to a full vault scan. """ - backlinks = [] - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) - - # Normalize target path for matching target_path = target_note_path target_path_lower = target_path.lower() - target_path_no_ext = target_path.replace('.md', '') - target_path_no_ext_lower = target_path_no_ext.lower() - target_name = Path(target_path).stem.lower() - - # For wikilinks: global name matching (find note anywhere by name) + target_path_no_ext_lower = target_path_lower.replace('.md', '') wikilink_refs = { target_path_lower, target_path_no_ext_lower, - target_name, + Path(target_path).stem.lower(), } - + + # scan_notes_fast_walk is TTL-cached, so calling it here is a dict-lookup + # in the warm case. We need it for the canonical ordering of `notes`. + notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + + candidates = note_index.try_backlink_candidates(target_path) + + backlinks: List[Dict] = [] for note in notes: if note.get('type') != 'note': continue - source_path = note['path'] - - # Skip self-references if source_path == target_path: continue - - # Get source folder for resolving relative markdown links - source_folder = str(Path(source_path).parent).replace('\\', '/') - if source_folder == '.': - source_folder = '' - - # Read note content - full_path = Path(notes_dir) / source_path - try: - with open(full_path, 'r', encoding='utf-8') as f: - content = f.read() - except Exception: + if candidates is not None and source_path not in candidates: continue - - lines = content.split('\n') - found_links = [] - - for line_num, line in enumerate(lines, 1): - # Find wikilinks: [[target]] or [[target|display]] - # Wikilinks use GLOBAL matching (find note anywhere by name) - wikilink_matches = re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line) - for match in wikilink_matches: - link_target = match.group(1).strip().lower() - link_target_no_ext = link_target.replace('.md', '') - - # Check if this wikilink points to our target (global match) - if link_target in wikilink_refs or link_target_no_ext in wikilink_refs: - start = max(0, match.start() - 30) - end = min(len(line), match.end() + 30) - context = line[start:end] - if start > 0: - context = '...' + context - if end < len(line): - context = context + '...' - - found_links.append({ - "line_number": line_num, - "context": context, - "type": "wikilink" - }) - - # Find markdown links: [text](path) - # Markdown links must RESOLVE as paths (relative to source or absolute) - markdown_matches = re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line) - for match in markdown_matches: - link_path = match.group(2).split('#')[0] # Remove anchor - if not link_path: - continue - - link_path = urllib.parse.unquote(link_path) - if link_path.startswith('./'): - link_path = link_path[2:] - - # Add .md if not present - link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md' - - # Resolve the link path to get the actual target - resolved_path = None - - # 1. Try resolving relative to source folder - if source_folder and not link_path.startswith('/'): - relative_path = f"{source_folder}/{link_path_with_md}" - if relative_path.lower() == target_path_lower: - resolved_path = target_path - elif f"{source_folder}/{link_path}".lower() == target_path_no_ext_lower: - resolved_path = target_path - - # 2. Try as absolute path from root - if not resolved_path: - if link_path_with_md.lower() == target_path_lower: - resolved_path = target_path - elif link_path.lower() == target_path_no_ext_lower: - resolved_path = target_path - - # Only add if the resolved path matches the target - if resolved_path: - start = max(0, match.start() - 30) - end = min(len(line), match.end() + 30) - context = line[start:end] - if start > 0: - context = '...' + context - if end < len(line): - context = context + '...' - - found_links.append({ - "line_number": line_num, - "context": context, - "type": "markdown" - }) - - # If we found links in this note, add it to backlinks - if found_links: + + refs = _extract_backlink_references( + notes_dir, + source_path, + target_path, + target_path_lower, + target_path_no_ext_lower, + wikilink_refs, + ) + if refs: backlinks.append({ "path": source_path, "name": note['name'].replace('.md', ''), - "references": found_links[:3] # Limit to 3 references per note + "references": refs[:3], }) - + return backlinks diff --git a/documentation/API.md b/documentation/API.md index ad21981..767b0a0 100644 --- a/documentation/API.md +++ b/documentation/API.md @@ -580,6 +580,59 @@ Returns application statistics at a glance. Designed for dashboard widgets (e.g. label: Version ``` +### Get Index Stats +```http +GET /api/index/stats +``` +Returns internal counters and sizes for the in-memory note index. Useful for confirming the index is built and observing its growth — handy when debugging slow endpoints or verifying that incremental updates are firing on every save/delete. + +The index is rebuilt on the first scan after each process start and updated incrementally on every save/delete/move. It lives in process memory only — no disk persistence. + +**Response:** +```json +{ + "enabled": true, + "built": true, + "search_built": false, + "notes": 142, + "folders": 12, + "tags": 37, + "links_forward_entries": 89, + "links_backward_entries": 76, + "wikilink_tokens": 134, + "search_terms": 0, + "counters": { + "build_count": 1, + "last_build_ms": 12.4, + "last_built_at": "2026-03-17T14:32:00+00:00", + "incremental_updates": 8, + "fingerprint_short_circuits": 3, + "search_build_count": 0, + "last_search_build_ms": 0.0 + } +} +``` + +| Field | Description | +|-------|-------------| +| `enabled` | Whether the index is active (always `true` in current builds) | +| `built` | `true` after the first vault scan completes | +| `search_built` | `true` after the first `/api/search` request (search index is built lazily) | +| `notes` | Number of note records currently held in the index | +| `folders` | Number of folder paths | +| `tags` | Number of unique tags | +| `links_forward_entries` | Number of notes that link out to at least one other note | +| `links_backward_entries` | Number of notes that have at least one incoming backlink | +| `wikilink_tokens` | Number of unique wikilink tokens (used for loose backlink matching by stem name) | +| `search_terms` | Number of unique terms in the full-text inverted search index (0 until first search) | +| `counters.build_count` | Total number of full index rebuilds since process start | +| `counters.last_build_ms` | Wall-clock time of the most recent full rebuild | +| `counters.last_built_at` | ISO timestamp of the most recent full rebuild | +| `counters.incremental_updates` | Number of single-note updates applied (save/delete/rename) | +| `counters.fingerprint_short_circuits` | Times a re-scan was skipped because the vault hash matched the previous scan | +| `counters.search_build_count` | Total number of search-index rebuilds | +| `counters.last_search_build_ms` | Wall-clock time of the most recent search-index rebuild | + ### Health Check ```http GET /health From 0cd5332a99ec87969b2909faaed592a9e3dd4331 Mon Sep 17 00:00:00 2001 From: Gamosoft Date: Tue, 30 Jun 2026 11:59:25 +0200 Subject: [PATCH 03/18] use index for note backlink --- backend/main.py | 188 +----------------- backend/note_index.py | 429 +++++++++--------------------------------- backend/utils.py | 190 +++++++------------ documentation/API.md | 2 - 4 files changed, 166 insertions(+), 643 deletions(-) diff --git a/backend/main.py b/backend/main.py index bf823ef..5de81c7 100644 --- a/backend/main.py +++ b/backend/main.py @@ -46,7 +46,6 @@ from .utils import ( get_backlinks, ) from . import note_index -from .note_index import USE_NOTE_INDEX from .plugins import PluginManager from .themes import get_available_themes, get_theme_css from .share import ( @@ -1486,186 +1485,15 @@ async def search( @api_router.get("/graph", tags=["Graph"]) async def get_graph(): - """Get graph data for note visualization with wikilink and markdown link detection.""" + """Graph data (nodes + resolved wikilink/markdown edges) for the visualizer.""" try: - import re - import urllib.parse - - # Fast path: index already holds resolved edges. Ensure a scan has - # populated it (first request after startup, or after an invalidate) - # then ask the facade for a snapshot. - if USE_NOTE_INDEX: - if not note_index.get_index().is_built(): - scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) - indexed = note_index.try_graph_data() - if indexed is not None: - nodes_paths, edges_tuples = indexed - return { - "nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths], - "edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples], - } - - notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) - nodes = [] - edges = [] - - # Build set of valid note names/paths for matching - note_paths = set() - note_paths_lower = {} # Map lowercase path -> actual path for case-insensitive matching - note_names = {} # Map name -> path for quick lookup - - for note in notes: - if note.get('type') == 'note': - note_paths.add(note['path']) - note_paths.add(note['path'].replace('.md', '')) - # Store lowercase path -> actual path mapping for case-insensitive matching - note_paths_lower[note['path'].lower()] = note['path'] - note_paths_lower[note['path'].replace('.md', '').lower()] = note['path'] - # Store name -> path mapping (without extension) - name = note['name'].replace('.md', '') - note_names[name.lower()] = note['path'] - note_names[note['name'].lower()] = note['path'] - - # Build graph structure with link detection - for note in notes: - if note.get('type') == 'note': - nodes.append({ - "id": note['path'], - "label": note['name'].replace('.md', '') - }) - - # Read note content to find links - content = get_note_content(config['storage']['notes_dir'], note['path']) - if content: - # Find wikilinks: [[target]] or [[target|display]] - wikilinks = re.findall(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', content) - - # Find standard markdown internal links: [text](path) - any local path (not http/https) - # Match links that don't start with http://, https://, mailto:, #, etc. - markdown_links = re.findall(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', content) - - # Get source note's folder for resolving relative links - # Use forward slashes consistently (note_paths uses forward slashes) - source_folder = str(Path(note['path']).parent).replace('\\', '/') - if source_folder == '.': - source_folder = '' - - # Process wikilinks - for target in wikilinks: - target = target.strip() - target_lower = target.lower() - - # Try to match target to an existing note - target_path = None - - # 1. Try resolving relative to source note's folder first - if source_folder and '/' not in target: - relative_path = f"{source_folder}/{target}" - relative_path_lower = relative_path.lower() - - if relative_path in note_paths: - target_path = relative_path if relative_path.endswith('.md') else relative_path + '.md' - elif relative_path + '.md' in note_paths: - target_path = relative_path + '.md' - elif relative_path_lower in note_paths_lower: - target_path = note_paths_lower[relative_path_lower] - elif relative_path_lower + '.md' in note_paths_lower: - target_path = note_paths_lower[relative_path_lower + '.md'] - - # 2. Exact path match (absolute or already has folder) - if not target_path: - if target in note_paths: - target_path = target if target.endswith('.md') else target + '.md' - elif target + '.md' in note_paths: - target_path = target + '.md' - # 3. Case-insensitive path match (e.g., [[Folder/Note]] -> folder/note.md) - elif target_lower in note_paths_lower: - target_path = note_paths_lower[target_lower] - elif target_lower + '.md' in note_paths_lower: - target_path = note_paths_lower[target_lower + '.md'] - # 4. Just note name (case-insensitive) - global match - elif target_lower in note_names: - target_path = note_names[target_lower] - - if target_path and target_path != note['path']: - edges.append({ - "source": note['path'], - "target": target_path, - "type": "wikilink" - }) - - # Process markdown links - for _, link_path in markdown_links: - # Skip anchor-only links and external protocols - if not link_path or link_path.startswith('#'): - continue - - # Remove anchor part if present (e.g., "note.md#section" -> "note.md") - link_path = link_path.split('#')[0] - if not link_path: - continue - - # Normalize path: remove ./ prefix, handle URL encoding - link_path = urllib.parse.unquote(link_path) - if link_path.startswith('./'): - link_path = link_path[2:] - - # Add .md extension if not present and doesn't have other extension - link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md' - - # Try to match target to an existing note - target_path = None - - # 1. First, try resolving relative to source note's folder - if source_folder and not link_path.startswith('/'): - relative_path = f"{source_folder}/{link_path}" - relative_path_with_md = f"{source_folder}/{link_path_with_md}" - relative_path_lower = relative_path.lower() - relative_path_with_md_lower = relative_path_with_md.lower() - - if relative_path in note_paths: - target_path = relative_path if relative_path.endswith('.md') else relative_path + '.md' - elif relative_path_with_md in note_paths: - target_path = relative_path_with_md - elif relative_path_lower in note_paths_lower: - target_path = note_paths_lower[relative_path_lower] - elif relative_path_with_md_lower in note_paths_lower: - target_path = note_paths_lower[relative_path_with_md_lower] - - # 2. Try exact path match from root (for absolute paths or notes at root) - if not target_path: - link_path_lower = link_path.lower() - link_path_with_md_lower = link_path_with_md.lower() - - if link_path in note_paths: - target_path = link_path if link_path.endswith('.md') else link_path + '.md' - elif link_path_with_md in note_paths: - target_path = link_path_with_md - # Case-insensitive path match - elif link_path_lower in note_paths_lower: - target_path = note_paths_lower[link_path_lower] - elif link_path_with_md_lower in note_paths_lower: - target_path = note_paths_lower[link_path_with_md_lower] - - # No global filename fallback for markdown links - they must resolve as paths - - if target_path and target_path != note['path']: - edges.append({ - "source": note['path'], - "target": target_path, - "type": "markdown" - }) - - # Remove duplicate edges - seen = set() - unique_edges = [] - for edge in edges: - key = (edge['source'], edge['target']) - if key not in seen: - seen.add(key) - unique_edges.append(edge) - - return {"nodes": nodes, "edges": unique_edges} + if not note_index.get_index().is_built(): + scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) + nodes_paths, edges_tuples = note_index.get_graph_data() + return { + "nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths], + "edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples], + } except Exception as e: raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to generate graph data")) diff --git a/backend/note_index.py b/backend/note_index.py index 5a0d7eb..42767c5 100644 --- a/backend/note_index.py +++ b/backend/note_index.py @@ -1,77 +1,12 @@ """ -NoteIndex — unified in-memory index of everything that's expensive to recompute -from disk: links, tags, full-text search, and per-note metadata. +NoteIndex — in-memory index of vault state: notes, folders, tags, links, search. - +---------------------------------------------------------------+ - | ONE module, ONE singleton, ONE lock, ONE rollback switch. | - | All call sites in utils.py / main.py are one-line shims. | - +---------------------------------------------------------------+ +Backs /api/notes, /api/tags, /api/backlinks, /api/graph, /api/search. Keeps +everything thread-safe under one RLock. Process-memory only — rebuilds on the +first /api/notes request after each process start (~1-3s on a 10K-note vault). -Why this exists ---------------- -Without an index, every backlink lookup, every graph render, and every text -search re-walks the entire vault and re-reads every file. That's O(N) per -request and scales linearly with vault size — 4-second note loads on a 10K -vault, 5-10s graph renders, multi-second searches. With an index, those -endpoints become O(matches) — milliseconds regardless of vault size. - -What's indexed --------------- - * Note metadata (path -> name, folder, mtime, size, type, tags) - * Folders (set of folder paths) - * Tags forward (path -> sorted tags) - * Tags backward (tag -> set of paths) - * Links forward (source_path -> {target_path: "wikilink"|"markdown"}) - * Links backward (target_path -> set of source paths) strict - * Wikilink tokens (lowercased token -> set of source paths) loose - * Search inverted (lowercased term -> set of paths) - -What's NOT cached ------------------ -File content. Reading file content for line-level context (backlinks, search -snippets) is done only for the small set of matched files, never for the -whole vault. - -Threading model ---------------- -Everything mutating goes through one RLock. Reads return snapshot copies, so -callers can iterate freely after they release the lock. The index is -designed to be called from FastAPI request handlers concurrently. - -Lifetime --------- -Process-memory only. No persistence to disk — keeping the app lightweight -matters more than the ~500ms saved on cold restart for a 10K-note vault. -The index rebuilds on the first `/api/notes` request after every restart -(typically 1-3 seconds, dominated by reading every file once for tag/link -extraction). F5 / browser reload doesn't touch this — only Python process -restart does. - -Rollback switch ---------------- -USE_NOTE_INDEX (below) is the single, global on/off. Flip it to False and -every facade function becomes a no-op or returns None, and every call site -in utils.py / main.py falls through to the legacy file-scanning behavior. - - ROLLBACK RECIPE (decide the index isn't worth it) - ------------------------------------------------- - Set USE_NOTE_INDEX = False below. That's it. Every try_* facade returns - None, every on_* facade becomes a no-op, callers fall through to the - legacy file-scanning paths that are preserved as the function bodies. - - COMMIT RECIPE (you're happy, drop the legacy paths) - --------------------------------------------------- - 1. Here: delete USE_NOTE_INDEX and inline `if not USE_NOTE_INDEX` to True - in every try_/on_ facade. - 2. In backend/utils.py: - - get_backlinks: drop the `candidates is None` branch — the - index always provides them. - - search_notes: drop the `candidates is None` fallback. - - get_all_tags / get_notes_by_tag: drop the legacy aggregation - tail block. - 3. In backend/main.py: - - /api/graph: drop the legacy fallback block at the bottom of - the endpoint, keep only `try_graph_data`. +Updated incrementally on every save/delete/move/rename through the on_* +facades at the bottom of this file. """ from __future__ import annotations @@ -87,40 +22,20 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Set, Tuple -# ============================================================================ -# Configuration -# ============================================================================ - -# Single global rollback switch. Flip to False to disable the index and every -# call site in utils.py / main.py falls back to the legacy code paths. -USE_NOTE_INDEX: bool = True - -# Parallelism cutoff for full rescans. Below this many markdown files, threads -# add more overhead than they save (small vaults stay sequential). +# Below this many markdown files, parallel tag extraction adds more overhead +# than it saves. _PARALLEL_CUTOFF = 50 _PARALLEL_WORKERS = min(8, (os.cpu_count() or 4)) -# Search tokenization. Lowercase, split on anything that's not a word char. -# Min length 2 keeps single-letter noise out of the index without losing -# common short queries like "go", "ai". +# Search tokenization. Min length 2 keeps single-letter noise out without +# losing common short queries like "go", "ai". _SEARCH_TOKEN_RE = re.compile(r"[A-Za-z0-9_\-]{2,}") _SEARCH_MIN_QUERY_LEN = 2 - -# ============================================================================ -# Shared regexes (same as legacy get_backlinks / /api/graph). Kept here so the -# index and the legacy paths are guaranteed to extract the same tokens. -# ============================================================================ - WIKILINK_RE = re.compile(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]') MDLINK_RE = re.compile(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)') -# ============================================================================ -# Public extraction helpers — used by the index AND by lifecycle hooks in -# utils.py (so a single save can update the index without re-reading the file). -# ============================================================================ - def extract_links_from_content(content: str) -> Dict[str, List[str]]: """Pull raw wikilink targets and markdown-link paths out of note content.""" wikilinks = [m.strip() for m in WIKILINK_RE.findall(content)] @@ -129,68 +44,50 @@ def extract_links_from_content(content: str) -> Dict[str, List[str]]: def extract_search_terms(content: str) -> Set[str]: - """Tokenize content for the inverted search index. Lowercased, deduped.""" + """Tokenize content for the inverted search index.""" return {m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(content)} -# ============================================================================ -# Data record for one note's metadata snapshot -# ============================================================================ - @dataclass class NoteRecord: - """Snapshot of one note's metadata. Kept tiny — no content, no resolved - links (those live in the inverted indexes).""" + """One note's metadata. No content, no resolved links.""" path: str # vault-relative POSIX name: str # stem (no extension) folder: str # vault-relative POSIX, "" for root modified: str # ISO timestamp size: int # bytes type: str # "note" | "image" | "audio" | ... - mtime: float # raw stat mtime, for staleness checks - tags: Tuple[str, ...] = field(default_factory=tuple) # sorted, deduped + mtime: float # raw stat mtime + tags: Tuple[str, ...] = field(default_factory=tuple) -# ============================================================================ -# The index itself -# ============================================================================ - class NoteIndex: - """Thread-safe in-memory index of vault state. Every read returns a - snapshot copy so callers don't have to hold the lock.""" - - # ------------------------------------------------------------------ - # Construction - # ------------------------------------------------------------------ + """Thread-safe in-memory index of vault state. Reads return copies so + callers don't have to hold the lock.""" def __init__(self) -> None: self._lock = threading.RLock() - # Note metadata - self._notes: Dict[str, NoteRecord] = {} # path -> record + self._notes: Dict[str, NoteRecord] = {} # path -> record self._folders: Set[str] = set() - # Tag indexes self._tags_forward: Dict[str, Tuple[str, ...]] = {} # path -> tags self._tags_backward: Dict[str, Set[str]] = {} # tag -> paths - # Link indexes - self._raw_links: Dict[str, Dict[str, List[str]]] = {} # path -> {"wikilinks":[], "mdlinks":[]} - self._links_forward: Dict[str, Dict[str, str]] = {} # src -> {tgt: type} - self._links_backward: Dict[str, Set[str]] = {} # tgt -> {srcs} - self._wikilink_tokens: Dict[str, Set[str]] = {} # token -> {srcs} (loose) + self._raw_links: Dict[str, Dict[str, List[str]]] = {} # path -> {"wikilinks":[], "mdlinks":[]} + self._links_forward: Dict[str, Dict[str, str]] = {} # src -> {tgt: type} + self._links_backward: Dict[str, Set[str]] = {} # tgt -> {srcs} + self._wikilink_tokens: Dict[str, Set[str]] = {} # lower token -> {srcs} (loose match) - # Full-text inverted index - self._search_terms: Dict[str, Set[str]] = {} # term -> {paths} + self._search_terms: Dict[str, Set[str]] = {} # term -> {paths} - # Build state. _built tracks the cheap part (notes/tags/links). - # _search_built tracks the expensive search index separately — that - # one is built lazily on first search request to keep startup fast. + # _search_built tracked separately — the search index is built lazily + # on the first /api/search call, after the cheaper notes/tags/links + # part is already built. self._built = False self._search_built = False self._raw_fingerprint: Optional[int] = None # short-circuits no-op rebuilds - # Observability counters (lock-free reads OK; rough is fine) self._stats = { "build_count": 0, "last_build_ms": 0.0, @@ -201,17 +98,12 @@ class NoteIndex: "last_search_build_ms": 0.0, } - # ------------------------------------------------------------------ - # Status / lifecycle gates - # ------------------------------------------------------------------ - def is_built(self) -> bool: with self._lock: return self._built def invalidate(self) -> None: - """Mark as needing rebuild on next scan. Used as a safety net when - an operation's incremental update is too fiddly to track precisely.""" + """Mark as needing rebuild on next scan.""" with self._lock: self._built = False self._raw_fingerprint = None @@ -236,25 +128,14 @@ class NoteIndex: return self._search_built def ensure_search_index_built(self, notes_dir: str) -> bool: - """Build the full-text search index by reading every markdown file. - - Cheap if already built (instant return). Called lazily by search_notes - on the first search request so app startup stays fast. Subsequent - searches reuse the built index. - - Concurrency: while one thread is building, others can still read the - (still-cold) index — they'll just take the legacy path until the - build completes. Returns True if the index is built (now or already). - """ - # Cheap pre-check without lock. + """Build the search index on demand. Returns True when ready, False + only if the main index isn't built yet. File I/O happens OUTSIDE the + lock so other reads aren't blocked.""" if self._search_built: return True - # Slow path: do the read+tokenize OUTSIDE the lock so other reads - # aren't blocked. Snapshot the paths we need to read under the lock, - # release, do I/O, then re-acquire to install the result. with self._lock: - if self._search_built: # raced with another builder + if self._search_built: return True if not self._built: return False @@ -265,16 +146,11 @@ class NoteIndex: terms_per_path: Dict[str, Set[str]] = {} for rel in paths_to_read: try: - full = base / rel - with open(full, "r", encoding="utf-8") as f: + with open(base / rel, "r", encoding="utf-8") as f: terms_per_path[rel] = extract_search_terms(f.read()) except Exception: terms_per_path[rel] = set() - # Re-acquire and install. If state changed under us (vault was - # heavily mutated during the build), still install — the index - # will be slightly stale until the next note save, which is the - # same liveness guarantee as the per-save update path. with self._lock: if self._search_built: return True @@ -287,24 +163,15 @@ class NoteIndex: self._stats["last_search_build_ms"] = (time.perf_counter() - t0) * 1000 return True - # ------------------------------------------------------------------ - # Bulk population — called from a full scan_notes_fast_walk - # ------------------------------------------------------------------ - def bulk_set( self, notes_meta: List[NoteRecord], folders: Iterable[str], sources_raw: Dict[str, Dict[str, List[str]]], ) -> None: - """Replace the entire index atomically. Builds notes/tags/links - (cheap on top of an already-completed scan). The search index is - NOT touched here — it's built lazily on the first search request - (see ensure_search_index_built) so app startup stays fast. - - Fast-paths when the input fingerprints to the same state we already - hold (typical warm scan where nothing changed in the vault). - """ + """Replace the entire index from a fresh scan. Short-circuits when + the input fingerprints to the same state we already hold (warm + scan, nothing changed).""" new_fp = _fingerprint(notes_meta, sources_raw) with self._lock: if self._built and self._raw_fingerprint == new_fp: @@ -319,8 +186,6 @@ class NoteIndex: self._rebuild_tags_unlocked() self._rebuild_links_unlocked() - # Search index: if it was built and still references known notes, - # prune stale entries; full rebuild is deferred until a search. if self._search_built: self._prune_search_unlocked() @@ -331,19 +196,13 @@ class NoteIndex: self._stats["last_build_ms"] = (time.perf_counter() - t0) * 1000 self._stats["last_built_at"] = datetime.now(tz=timezone.utc).isoformat() - # ------------------------------------------------------------------ - # Incremental updates — called from save/delete/rename handlers - # ------------------------------------------------------------------ - def update_note( self, record: NoteRecord, raw_links: Dict[str, List[str]], content: Optional[str] = None, ) -> None: - """A single note's content changed (or was just created). Patches the - index in place: re-extracts its links, updates the tag indexes, and - if `content` is provided, refreshes its search-term entries.""" + """Patch the index in place after a note is created or saved.""" with self._lock: old_record = self._notes.get(record.path) old_tags = old_record.tags if old_record else () @@ -352,7 +211,6 @@ class NoteIndex: if record.folder: self._folders.add(record.folder) - # Tags: diff old vs new and patch the backward index. new_tags = record.tags if old_tags != new_tags: for t in set(old_tags) - set(new_tags): @@ -365,13 +223,11 @@ class NoteIndex: self._tags_backward.setdefault(t, set()).add(record.path) self._tags_forward[record.path] = new_tags - # Links: re-resolve this one source against the current vault. self._raw_links[record.path] = raw_links self._resolve_single_source_unlocked(record.path) - # Search: only patch the search index if it's already been built. - # Otherwise we'd be doing work for an index nobody is using yet - # (the first /api/search call will build it from disk). + # Search index only gets patched if it's already been built — + # otherwise the first /api/search will build it from scratch. if content is not None and self._search_built: self._update_search_for_note_unlocked(record.path, content) @@ -437,11 +293,9 @@ class NoteIndex: self._stats["incremental_updates"] += 1 def rename_note(self, old_path: str, new_path: str) -> None: - """A note was renamed/moved. Move all references from old_path to - new_path in every index. Because the new path can change the source's - own relative-folder resolution AND can change how other notes resolve - their wikilinks to it (different parent folder, different stem), - the simplest correct path is to migrate raw state then re-resolve.""" + """Move all references from old_path to new_path. The path change can + affect how other notes' wikilinks resolve, so we wipe the resolved + link indexes for this note and rely on the next bulk_set to rebuild.""" if old_path == new_path: return with self._lock: @@ -465,7 +319,6 @@ class NoteIndex: if new_record.folder: self._folders.add(new_record.folder) - # Tags forward + backward — swap the key. if old_path in self._tags_forward: tags = self._tags_forward.pop(old_path) self._tags_forward[new_path] = tags @@ -475,24 +328,17 @@ class NoteIndex: bucket.discard(old_path) bucket.add(new_path) - # Search: swap path in every term bucket. Bounded by terms-per-note. for paths in self._search_terms.values(): if old_path in paths: paths.discard(old_path) paths.add(new_path) - # Raw links: migrate. if old_path in self._raw_links: self._raw_links[new_path] = self._raw_links.pop(old_path) - # Strict link indexes: drop both old src and old target everywhere, - # then re-resolve. Other sources that linked TO old_path by name - # may now resolve to new_path (or not), so we need a re-resolve. - # - # Implementation: drop forward/backward for old_path, drop old - # entries in other sources' forwards that pointed to old_path, - # invalidate the index. The next scan rebuilds (which we trigger - # implicitly because the caller flips _built = False below). + # Other sources that linked to old_path by stem name may now + # resolve to new_path (or not), so we wipe both forward & backward + # for old_path and let the next bulk_set re-resolve from scratch. self._links_forward.pop(old_path, None) for bucket in self._links_backward.values(): bucket.discard(old_path) @@ -504,7 +350,6 @@ class NoteIndex: if not fwd: del self._links_forward[src] - # Loose wikilink index: replace path values. empty_keys = [] for key, sources in self._wikilink_tokens.items(): if old_path in sources: @@ -516,17 +361,12 @@ class NoteIndex: del self._wikilink_tokens[k] self._raw_fingerprint = None - self._built = False # force re-resolve on next bulk_set/scan + self._built = False # force re-resolve on next bulk_set self._stats["incremental_updates"] += 1 def rename_folder_prefix(self, old_prefix: str, new_prefix: str) -> None: - """A folder was renamed/moved. Migrate every entry whose path starts - with `old_prefix/` to `new_prefix/`. Much cheaper than a full rebuild: - for a 1000-note folder rename, this is microseconds of key swaps - vs ~400ms of disk re-scan. - - Both prefixes are normalized to forward slashes, no trailing slash. - """ + """Migrate every entry under `old_prefix/` to `new_prefix/`. Much + cheaper than a full rebuild — microseconds of key swaps.""" old_prefix = old_prefix.rstrip("/") new_prefix = new_prefix.rstrip("/") if old_prefix == new_prefix: @@ -534,14 +374,9 @@ class NoteIndex: with self._lock: affected_paths = [p for p in self._notes if p == old_prefix or p.startswith(old_prefix + "/")] for old_path in affected_paths: - suffix = old_path[len(old_prefix):] # includes leading "/" or nothing - new_path = new_prefix + suffix - # Reuse rename_note's heavy lifting — already correct, just - # called many times. Acceptable for folder operations. - # (Inlining would be a perf gain but multiplies bug surface.) - self._rename_note_unlocked(old_path, new_path) + suffix = old_path[len(old_prefix):] + self._rename_note_unlocked(old_path, new_prefix + suffix) - # Migrate the folder set too. folders_to_rename = [ f for f in self._folders if f == old_prefix or f.startswith(old_prefix + "/") ] @@ -554,7 +389,7 @@ class NoteIndex: self._built = False def remove_folder_prefix(self, prefix: str) -> None: - """A folder was deleted. Drop everything under it.""" + """Drop every entry under `prefix/`.""" prefix = prefix.rstrip("/") with self._lock: affected = [p for p in self._notes if p == prefix or p.startswith(prefix + "/")] @@ -597,7 +432,7 @@ class NoteIndex: return nodes, edges def get_all_tags(self) -> Dict[str, int]: - """Snapshot: {tag: count}, sorted by tag name.""" + """Snapshot {tag: count}, sorted by tag name.""" with self._lock: return {tag: len(paths) for tag, paths in sorted(self._tags_backward.items())} @@ -610,23 +445,19 @@ class NoteIndex: with self._lock: return self._notes.get(path) + def all_note_records(self) -> List[Tuple[str, NoteRecord]]: + """Snapshot list of (path, record) for every indexed markdown note.""" + with self._lock: + return [(p, r) for p, r in self._notes.items() if r.type == "note"] + def try_get_extraction( self, rel_path: str, mtime: float, ) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: - """Return (tags, raw_links) from the index when fresh. - - The caller passes the file's current mtime; we hand back the cached - extraction only when the recorded mtime matches exactly. Lets - scan_notes_fast_walk skip the per-file read on a snapshot-warm - startup — the bulk of cold-load latency on large vaults. - - Returns None when: - - The note isn't in the index (new file) - - mtime differs (file changed since snapshot) - - We've somehow lost the raw_links entry (defensive) - """ + """Return (tags, raw_links) from the index only if the recorded mtime + matches `mtime` exactly. Lets scan_notes_fast_walk skip the per-file + read on a warm scan.""" with self._lock: rec = self._notes.get(rel_path) if rec is None or rec.mtime != mtime: @@ -643,34 +474,18 @@ class NoteIndex: ) def get_search_candidates(self, query: str) -> Optional[Set[str]]: - # Index can only narrow candidates if it's been built. + """Superset of paths whose content COULD contain `query`. Caller + still runs the substring match per candidate for confirmation + + snippet extraction. Returns None when the query is too short or + tokenizes to nothing — caller should iterate every note instead.""" if not self._search_built: return None - """Return the set of paths that COULD contain `query` as a substring, - based on the inverted term index. - - Returns None when the query is too short to use the index (caller - should fall through to the legacy full-scan). When the query - tokenizes to nothing useful (all stopword-like noise), returns the - set of every indexed path so the caller still does a substring - check on each (no false negatives). - - IMPORTANT: this is a SUPERSET — the caller must still run the - substring match per candidate to confirm and extract context. - Token-AND can include docs that have both tokens but not as the - adjacent substring the user searched for. - """ if len(query) < _SEARCH_MIN_QUERY_LEN: return None tokens = [m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(query)] if not tokens: - # Query has no tokenizable parts (pure punctuation, single char, - # etc.). Index can't help; fall through. return None with self._lock: - # Intersection of all token buckets. Empty set means no doc - # contains ALL of the tokens (and therefore none contain the - # query as substring either). candidate = self._search_terms.get(tokens[0]) if candidate is None: return set() @@ -685,11 +500,9 @@ class NoteIndex: return result def stats(self) -> Dict[str, Any]: - """Snapshot of internal counters + size metrics. Cheap to call — - no traversal beyond `len()` on the top-level dicts.""" + """Snapshot of counters + size metrics.""" with self._lock: return { - "enabled": USE_NOTE_INDEX, "built": self._built, "search_built": self._search_built, "notes": len(self._notes), @@ -720,11 +533,8 @@ class NoteIndex: self._tags_backward.setdefault(tag, set()).add(path) def _rebuild_links_unlocked(self) -> None: - """Full link re-resolution. Builds a shared Resolver lookup once - (O(N)) and reuses it for every source (O(K) each) — total O(N+K*L) - rather than the O(N*K) you'd get by building a fresh Resolver per - source. - """ + """Full link re-resolution. Reuses one _Resolver across all sources + (O(N+K*L) instead of O(N*K)).""" self._links_forward.clear() self._links_backward.clear() self._wikilink_tokens.clear() @@ -734,9 +544,7 @@ class NoteIndex: self._resolve_single_source_unlocked(source_path, resolver=resolver, skip_cleanup=True) def _prune_search_unlocked(self) -> None: - """Drop search-term entries for paths no longer in the index. Cheap - compared to a full rebuild — only touches terms whose bucket still - references a now-deleted path.""" + """Drop search-term entries for paths no longer in the index.""" live_paths = set(self._notes.keys()) empty_terms = [] for term, paths in self._search_terms.items(): @@ -750,7 +558,6 @@ class NoteIndex: def _update_search_for_note_unlocked(self, path: str, content: str) -> None: """Replace one note's terms in the inverted index.""" - # Drop old entries for this path. empty_terms = [] for term, paths in self._search_terms.items(): if path in paths: @@ -759,7 +566,6 @@ class NoteIndex: empty_terms.append(term) for term in empty_terms: del self._search_terms[term] - # Add new entries. for term in extract_search_terms(content): self._search_terms.setdefault(term, set()).add(path) @@ -799,16 +605,13 @@ class NoteIndex: targets: Dict[str, str] = {} - # Wikilinks first so they win the "first wins" dedup that legacy - # /api/graph also implements. + # Wikilinks first (they win the "first wins" dedup that /api/graph uses). for target in raw.get("wikilinks", []): resolved = resolver.resolve_wikilink(target, source_folder) if resolved and resolved != source_path and resolved not in targets: targets[resolved] = "wikilink" - # Loose wikilink reverse index — populated regardless of strict - # resolution success, because the legacy get_backlinks uses - # token-stem matching independent of where the link actually - # navigates. + # Loose token index — populated even when strict resolution fails, + # because backlink matching uses stem comparison independently. t_lower = target.strip().lower() if t_lower: self._wikilink_tokens.setdefault(t_lower, set()).add(source_path) @@ -827,8 +630,8 @@ class NoteIndex: self._links_backward.setdefault(t, set()).add(source_path) def _rename_note_unlocked(self, old_path: str, new_path: str) -> None: - """Same as rename_note() but assumes the caller already holds the lock. - Used by folder-prefix rename to avoid re-acquiring the lock per file.""" + """rename_note() body without acquiring the lock — used by + rename_folder_prefix to avoid re-locking per file.""" if old_path == new_path: return old_record = self._notes.pop(old_path, None) @@ -936,9 +739,8 @@ class NoteIndex: # ============================================================================ class _Resolver: - """Build the lookup tables once per resolution batch, then call - resolve_* repeatedly. Reused across sources within a single rebuild so - we don't pay O(N) to construct it on every source.""" + """Link-target lookup tables. Build once per resolution batch, then call + resolve_* repeatedly across many sources.""" def __init__(self, all_notes: Set[str]) -> None: self.note_paths: Set[str] = set(all_notes) @@ -1029,8 +831,7 @@ def _fingerprint( notes_meta: List[NoteRecord], sources_raw: Dict[str, Dict[str, List[str]]], ) -> int: - """Cheap content hash of a scan result. Used to short-circuit bulk_set - when nothing has changed in the vault.""" + """Cheap hash of a scan result. Short-circuits bulk_set when unchanged.""" notes_fp = hash(frozenset((n.path, n.mtime) for n in notes_meta)) raw_items = ( (src, tuple(raw.get("wikilinks", [])), tuple(raw.get("mdlinks", []))) @@ -1040,11 +841,7 @@ def _fingerprint( # ============================================================================ -# Module-level singleton + facade -# -# Every utils.py / main.py call site uses these — never instantiates its own -# index. The facade functions are no-ops (or return None) when -# USE_NOTE_INDEX is False, so call sites don't have to repeat the flag check. +# Module singleton + facade. Callers in utils.py / main.py go through these. # ============================================================================ _index = NoteIndex() @@ -1054,12 +851,10 @@ def get_index() -> NoteIndex: return _index -# --- Lifecycle facade (one-line calls from utils.py mutators) ---------------- +# --- Lifecycle hooks (one-line calls from utils.py mutators) ---------------- def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None: - """A note was created or updated on disk. Patch the index in place.""" - if not USE_NOTE_INDEX: - return + """A note was created or updated on disk.""" try: rel_path = full_path.relative_to(Path(notes_dir)).as_posix() st = full_path.stat() @@ -1074,15 +869,12 @@ def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None: mtime=st.st_mtime, tags=tuple(_parse_tags_for_record(content)), ) - raw_links = extract_links_from_content(content) - _index.update_note(record, raw_links, content=content) + _index.update_note(record, extract_links_from_content(content), content=content) except Exception as e: print(f"note_index: on_note_saved failed for {full_path}: {e}") def on_note_deleted(notes_dir: str, full_path: Path) -> None: - if not USE_NOTE_INDEX: - return try: rel_path = full_path.relative_to(Path(notes_dir)).as_posix() _index.remove_note(rel_path) @@ -1091,8 +883,6 @@ def on_note_deleted(notes_dir: str, full_path: Path) -> None: def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: - if not USE_NOTE_INDEX: - return try: base = Path(notes_dir) _index.rename_note( @@ -1104,10 +894,7 @@ def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: - """A folder was moved/renamed. Re-keys every entry under it (cheap, - no disk reads) rather than invalidating the whole index.""" - if not USE_NOTE_INDEX: - return + """Re-key every entry under the folder. No disk reads.""" try: base = Path(notes_dir) _index.rename_folder_prefix( @@ -1116,12 +903,10 @@ def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) ) except Exception as e: print(f"note_index: on_folder_renamed failed: {e}") - _index.invalidate() # fail-safe + _index.invalidate() def on_folder_deleted(notes_dir: str, full_path: Path) -> None: - if not USE_NOTE_INDEX: - return try: rel_prefix = full_path.relative_to(Path(notes_dir)).as_posix() _index.remove_folder_prefix(rel_prefix) @@ -1135,9 +920,7 @@ def populate_from_scan( folders: Iterable[str], sources_raw: Dict[str, Dict[str, List[str]]], ) -> None: - """Bulk-replace the index from a fresh scan. No-op when off.""" - if not USE_NOTE_INDEX: - return + """Bulk-replace the index after a full scan_notes_fast_walk.""" try: _index.bulk_set(notes_meta, folders, sources_raw) except Exception as e: @@ -1145,44 +928,34 @@ def populate_from_scan( def ensure_search_index(notes_dir: str) -> bool: - """Lazily build the full-text search index on first /api/search request. - Cheap after the first call. Returns True if the index is now usable.""" - if not USE_NOTE_INDEX: - return False + """Lazy-build the search index on first /api/search. Returns False only + if the main index isn't built yet.""" if not _index.is_built(): return False return _index.ensure_search_index_built(notes_dir) -# --- Read facade (returns None when off — callers fall through to legacy) ---- +# --- Read facade ------------------------------------------------------------- -def try_backlink_candidates(target_path: str) -> Optional[Set[str]]: - if not USE_NOTE_INDEX or not _index.is_built(): - return None +def get_backlink_candidates(target_path: str) -> Set[str]: return _index.get_backlink_candidate_sources(target_path) -def try_graph_data() -> Optional[Tuple[List[str], List[Tuple[str, str, str]]]]: - if not USE_NOTE_INDEX or not _index.is_built(): - return None +def get_graph_data() -> Tuple[List[str], List[Tuple[str, str, str]]]: return _index.get_graph_data() -def try_search_candidates(query: str) -> Optional[Set[str]]: - if not USE_NOTE_INDEX or not _index.is_built(): - return None +def get_search_candidates(query: str) -> Optional[Set[str]]: + """Returns None when query is too short / untokenizable — caller iterates + every indexed note instead.""" return _index.get_search_candidates(query) -def try_all_tags() -> Optional[Dict[str, int]]: - if not USE_NOTE_INDEX or not _index.is_built(): - return None +def get_all_tags() -> Dict[str, int]: return _index.get_all_tags() -def try_notes_by_tag(tag: str) -> Optional[Set[str]]: - if not USE_NOTE_INDEX or not _index.is_built(): - return None +def get_paths_for_tag(tag: str) -> Set[str]: return _index.get_paths_for_tag(tag) @@ -1190,30 +963,16 @@ def try_get_extraction( rel_path: str, mtime: float, ) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: - """Serve (tags, raw_links) from the index for a single file, when fresh. - - Used by scan_notes_fast_walk to skip the cold per-file read after a - snapshot load. Returns None when the index can't help (off, not built, - file unknown, mtime mismatch) — caller falls back to reading the file. - """ - if not USE_NOTE_INDEX or not _index.is_built(): - return None + """(tags, raw_links) from the index when mtime matches, else None so the + caller reads the file. Used by scan_notes_fast_walk.""" return _index.try_get_extraction(rel_path, mtime) -# --- Observability ----------------------------------------------------------- - def stats() -> Dict[str, Any]: return _index.stats() -# ============================================================================ -# Late import to avoid a circular dependency with utils.parse_tags. -# We need tag parsing inside on_note_saved but utils.py imports this module. -# ============================================================================ - +# Late import — utils imports this module, so we delay the reverse direction. def _parse_tags_for_record(content: str) -> List[str]: - """Thin shim to utils.parse_tags. Late-bound so this module imports cleanly - before utils.py is loaded.""" from .utils import parse_tags return parse_tags(content) diff --git a/backend/utils.py b/backend/utils.py index f3902a5..5061f9d 100644 --- a/backend/utils.py +++ b/backend/utils.py @@ -2,8 +2,7 @@ Utility functions for file operations, search, and markdown processing. Heavy work (backlinks, graph, full-text search, tag aggregation) is delegated -to the in-memory index in note_index.py — every facade function there is a -no-op when USE_NOTE_INDEX is False, so call sites here stay flag-free. +to the in-memory index in note_index.py. """ import os @@ -163,27 +162,25 @@ def _scan_cache_set(key: Tuple[str, bool], value: Tuple[List[Dict], List[str]]) def _scan_cache_invalidate() -> None: - """Drop the TTL scan cache. Called from every mutation handler so callers - that hit the cache immediately after a save/delete/move see the new state - instead of stale data from the previous walk.""" + """Drop the TTL scan cache. Called from every mutation handler.""" with _SCAN_WALK_CACHE_LOCK: _SCAN_WALK_CACHE.clear() +def _ensure_index_built(notes_dir: str) -> None: + """Trigger a fresh scan when the NoteIndex hasn't been populated yet. + Idempotent: the scan short-circuits when nothing changed in the vault.""" + if not note_index.get_index().is_built(): + scan_notes_fast_walk(notes_dir, use_cache=False, include_media=False) + + def get_tags_and_links_cached( file_path: Path, rel_path: Optional[str] = None, ) -> Tuple[List[str], Dict[str, List[str]]]: - """Single-read fused extraction of tags + raw links, cached by mtime. - - Three-tier lookup, cheapest first: - 1. In-process per-file caches (warm scans on the live server) - 2. NoteIndex (warm after snapshot load — avoids file reads on cold start) - 3. Read the file from disk - - rel_path enables tier 2. scan_notes_fast_walk passes it; ad-hoc callers - can omit it and just get tiers 1 + 3. - """ + """Fused tags + raw-links extraction, mtime-cached. Tries in-process + per-file caches first, then the NoteIndex (when rel_path is provided), + then reads the file.""" try: mtime = file_path.stat().st_mtime file_key = str(file_path) @@ -196,8 +193,6 @@ def get_tags_and_links_cached( if tags_ok and links_ok: return tag_cached[1], link_cached[1] - # Try the NoteIndex before reading the file. On a snapshot-warm - # startup, this turns a 10K-file cold scan into a no-I/O walk. if rel_path is not None: indexed = note_index.try_get_extraction(rel_path, mtime) if indexed is not None: @@ -293,11 +288,11 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: _scan_cache_set(cache_key, normalized_value) return normalized_value - # Walk the vault once. Tag/link extraction is deferred out of the walk - # (the expensive part is the file I/O) and parallelized below across files. + # Walk the vault once. Tag/link extraction is parallelized below across + # the markdown files we collected here. notes: List[Dict] = [] folders_set = set() - md_to_extract: List[Tuple[int, Path, str]] = [] # (note_index, full_path, rel_path) + md_to_extract: List[Tuple[int, Path, str]] = [] # (index, full_path, rel_path) for root, dirnames, filenames in os.walk(notes_path): dirnames[:] = [d for d in dirnames if not d.startswith('.')] @@ -339,19 +334,10 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: if is_markdown: md_to_extract.append((len(notes) - 1, full_path, rel_str)) - # Fused tag + raw-link extraction. mtime-keyed cache makes warm scans - # mostly free; cold scans win from parallel I/O. - # - # We don't read full content here — that would double the parsing cost - # for every scan, and the search index is the only thing that needs it. - # Search is built lazily on the first search request (see - # note_index.ensure_search_index_built), which keeps startup fast and - # only pays the cost when a user actually searches. + # Tag + raw-link extraction in parallel. Search-index terms are built + # later, lazily, on the first /api/search call. sources_raw: Dict[str, Dict[str, List[str]]] = {} if md_to_extract: - # Pass rel_path so each call can hit the NoteIndex before falling - # back to a file read — critical for fast startup after a snapshot - # load on large vaults. path_pairs = [(full, rel) for (_, full, rel) in md_to_extract] if len(md_to_extract) >= 50: workers = min(8, (os.cpu_count() or 4)) @@ -367,8 +353,7 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: notes[idx]["tags"] = tags sources_raw[rel_str] = links - # Populate the unified index. Cheap when the fingerprint matches (warm - # scan, nothing changed) — short-circuits internally. No-op when off. + # Push the result into the NoteIndex. Short-circuits on warm scans. notes_meta = [ NoteRecord( path=n["path"], @@ -555,27 +540,28 @@ def delete_note(notes_dir: str, note_path: str) -> bool: def search_notes(notes_dir: str, query: str) -> List[Dict]: - """Full-text search through note contents. - - Only searches inside note content (not filenames or paths). The index, when - available, narrows down the candidate files via a token-AND match on its - inverted index — then the existing per-line snippet extraction runs on the - narrow set. Output is byte-identical to a full scan. - """ + """Full-text search through note contents. Narrow the candidate set via + the inverted index, then run the snippet extractor on each candidate.""" from html import escape results: List[Dict] = [] - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) - # Build the search index lazily on the first search request. Subsequent - # searches hit the warm index; the first one pays the cost (read every - # file once) and is roughly as slow as the legacy full scan would be. + _ensure_index_built(notes_dir) note_index.ensure_search_index(notes_dir) - candidates = note_index.try_search_candidates(query) + candidates = note_index.get_search_candidates(query) - for note in notes: + idx = note_index.get_index() + if candidates is None: + # Query too short to tokenize — iterate every indexed note instead. + candidate_records = idx.all_note_records() + else: + candidate_records = [(p, idx.get_note_record(p)) for p in candidates] + candidate_records = [(p, r) for (p, r) in candidate_records if r is not None and r.type == "note"] + + candidate_records.sort(key=lambda pr: pr[1].mtime, reverse=True) + iterable = [{"path": p, "name": r.name} for (p, r) in candidate_records] + + for note in iterable: path = note["path"] - if candidates is not None and path not in candidates: - continue md_file = Path(notes_dir) / path try: with open(md_file, 'r', encoding='utf-8') as f: @@ -888,64 +874,29 @@ def parse_tags(content: str) -> List[str]: def get_all_tags(notes_dir: str) -> Dict[str, int]: - """All tags in the vault with note counts. Index-backed when available.""" - # Fast path FIRST — when the index is ready it knows every tag count, - # so we skip the scan entirely (no disk walk, no JSON of 10K notes). - indexed = note_index.try_all_tags() - if indexed is not None: - return indexed - - # Legacy fallback: scan the vault and aggregate. - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) - tag_counts: Dict[str, int] = {} - for note in notes: - for tag in note.get("tags", []): - tag_counts[tag] = tag_counts.get(tag, 0) + 1 - return dict(sorted(tag_counts.items())) + """All tags in the vault with note counts.""" + _ensure_index_built(notes_dir) + return note_index.get_all_tags() def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]: - """All notes carrying `tag` (case-insensitive). Index-backed when available.""" - tag_lower = tag.lower() - - # Fast path FIRST: every field we need lives in the NoteRecord. Skip - # the vault scan when the index can answer directly. - indexed_paths = note_index.try_notes_by_tag(tag_lower) - if indexed_paths is not None: - idx = note_index.get_index() - records = [idx.get_note_record(p) for p in indexed_paths] - matching = [ - { - "name": r.name, - "path": r.path, - "folder": r.folder, - "modified": r.modified, - "size": r.size, - "tags": list(r.tags), - } - for r in records - if r is not None and r.type == "note" - ] - # Match legacy sort order (scan returns notes sorted by modified desc). - matching.sort(key=lambda n: n["modified"], reverse=True) - return matching - - # Legacy fallback. - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) - matching: List[Dict] = [] - for note in notes: - if note.get("type") != "note": - continue - tags = note.get("tags", []) - if tag_lower in tags: - matching.append({ - "name": note["name"], - "path": note["path"], - "folder": note["folder"], - "modified": note["modified"], - "size": note["size"], - "tags": tags, - }) + """All notes carrying `tag` (case-insensitive).""" + _ensure_index_built(notes_dir) + idx = note_index.get_index() + records = [idx.get_note_record(p) for p in note_index.get_paths_for_tag(tag.lower())] + matching = [ + { + "name": r.name, + "path": r.path, + "folder": r.folder, + "modified": r.modified, + "size": r.size, + "tags": list(r.tags), + } + for r in records + if r is not None and r.type == "note" + ] + matching.sort(key=lambda n: n["modified"], reverse=True) return matching @@ -1189,13 +1140,8 @@ def _extract_backlink_references( def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]: - """Find all notes that link TO the specified note. - - Index-backed when available: the index hands us a superset of candidate - source files (O(1)), we read only those, then the per-line matcher - filters down to true matches with line-level context. Output is - byte-identical to a full vault scan. - """ + """All notes that link TO `target_note_path`. The index narrows the + candidate set; we only read the candidate files for line context.""" target_path = target_note_path target_path_lower = target_path.lower() target_path_no_ext_lower = target_path_lower.replace('.md', '') @@ -1205,22 +1151,15 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]: Path(target_path).stem.lower(), } - # scan_notes_fast_walk is TTL-cached, so calling it here is a dict-lookup - # in the warm case. We need it for the canonical ordering of `notes`. - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) - - candidates = note_index.try_backlink_candidates(target_path) + _ensure_index_built(notes_dir) + idx = note_index.get_index() + candidates = note_index.get_backlink_candidates(target_path) + records = [(p, idx.get_note_record(p)) for p in candidates] + records = [(p, r) for (p, r) in records if r is not None and r.type == "note" and p != target_path] + records.sort(key=lambda pr: pr[1].mtime, reverse=True) backlinks: List[Dict] = [] - for note in notes: - if note.get('type') != 'note': - continue - source_path = note['path'] - if source_path == target_path: - continue - if candidates is not None and source_path not in candidates: - continue - + for source_path, record in records: refs = _extract_backlink_references( notes_dir, source_path, @@ -1232,9 +1171,8 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]: if refs: backlinks.append({ "path": source_path, - "name": note['name'].replace('.md', ''), + "name": record.name.replace('.md', ''), "references": refs[:3], }) - return backlinks diff --git a/documentation/API.md b/documentation/API.md index 767b0a0..270af9c 100644 --- a/documentation/API.md +++ b/documentation/API.md @@ -591,7 +591,6 @@ The index is rebuilt on the first scan after each process start and updated incr **Response:** ```json { - "enabled": true, "built": true, "search_built": false, "notes": 142, @@ -615,7 +614,6 @@ The index is rebuilt on the first scan after each process start and updated incr | Field | Description | |-------|-------------| -| `enabled` | Whether the index is active (always `true` in current builds) | | `built` | `true` after the first vault scan completes | | `search_built` | `true` after the first `/api/search` request (search index is built lazily) | | `notes` | Number of note records currently held in the index | From c21d44a16991c0a4632c91299e6bfd93e3de7506 Mon Sep 17 00:00:00 2001 From: Gamosoft Date: Tue, 30 Jun 2026 12:26:32 +0200 Subject: [PATCH 04/18] added loading indicator for large vaults --- frontend/app.js | 21 +++++++++++++++++++++ frontend/index.html | 44 +++++++++++++++++++++++++++++++++++++++----- locales/de-DE.json | 1 + locales/en-GB.json | 1 + locales/en-US.json | 1 + locales/es-ES.json | 1 + locales/fr-FR.json | 1 + locales/hu-HU.json | 1 + locales/it-IT.json | 1 + locales/ja-JP.json | 1 + locales/ru-RU.json | 1 + locales/sl-SI.json | 1 + locales/zh-CN.json | 1 + 13 files changed, 71 insertions(+), 5 deletions(-) diff --git a/frontend/app.js b/frontend/app.js index 5ff7d7d..fcab43f 100644 --- a/frontend/app.js +++ b/frontend/app.js @@ -262,6 +262,13 @@ function noteApp() { alreadyDonated: false, autosaveDelayMs: CONFIG.AUTOSAVE_DELAY, // hydrated from /api/config in loadConfig() notes: [], + + // True while /api/notes is in flight. Drives the "Loading your vault…" + // placeholder + the delayed overlay (notesLoadingShowOverlay). + notesLoading: true, + notesLoadingShowOverlay: false, + _notesLoadingOverlayTimer: null, + currentNote: '', currentNoteName: '', noteContent: '', @@ -1524,6 +1531,16 @@ function noteApp() { // Load all notes async loadNotes() { + this.notesLoading = true; + // Show the overlay only if we're still loading after 800ms AND + // the tree is currently empty (first load). Refreshes after a + // save typically resolve in <100ms and never trip this. + clearTimeout(this._notesLoadingOverlayTimer); + this._notesLoadingOverlayTimer = setTimeout(() => { + if (this.notesLoading && this.notes.length === 0 && this.allFolders.length === 0) { + this.notesLoadingShowOverlay = true; + } + }, 800); try { const response = await fetch('/api/notes'); const data = await response.json(); @@ -1534,6 +1551,10 @@ function noteApp() { await this.loadTags(); // Load tags after notes are loaded } catch (error) { ErrorHandler.handle('load notes', error); + } finally { + clearTimeout(this._notesLoadingOverlayTimer); + this.notesLoading = false; + this.notesLoadingShowOverlay = false; } }, diff --git a/frontend/index.html b/frontend/index.html index a167237..228be3a 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1397,7 +1397,30 @@ - + + +
+
+ + + + + +
+
+