global in memory index for large vaults

backend/main.py

@@ -45,6 +45,8 @@ from .utils import (
     paginate,
     get_backlinks,
 )
+from . import note_index
+from .note_index import USE_NOTE_INDEX
 from .plugins import PluginManager
 from .themes import get_available_themes, get_theme_css
 from .share import (
@@ -265,6 +267,7 @@ plugin_manager = PluginManager(config['storage']['plugins_dir'])
 # Run app startup hooks
 plugin_manager.run_hook('on_app_startup')
 
+
 # Mount static files
 static_path = Path(__file__).parent.parent / "frontend"
 app.mount("/static", StaticFiles(directory=static_path), name="static")
@@ -1483,10 +1486,25 @@ async def search(
 
 @api_router.get("/graph", tags=["Graph"])
 async def get_graph():
-    """Get graph data for note visualization with wikilink and markdown link detection"""
+    """Get graph data for note visualization with wikilink and markdown link detection."""
     try:
         import re
         import urllib.parse
+
+        # Fast path: index already holds resolved edges. Ensure a scan has
+        # populated it (first request after startup, or after an invalidate)
+        # then ask the facade for a snapshot.
+        if USE_NOTE_INDEX:
+            if not note_index.get_index().is_built():
+                scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
+            indexed = note_index.try_graph_data()
+            if indexed is not None:
+                nodes_paths, edges_tuples = indexed
+                return {
+                    "nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths],
+                    "edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples],
+                }
+
         notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
         nodes = []
         edges = []
@@ -1692,6 +1710,19 @@ async def toggle_plugin(request: Request, plugin_name: str, enabled: dict):
         raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to toggle plugin"))
 
 
+# ============================================================================
+# Index observability — internal counters & sizes for debugging
+#
+# Useful for confirming the index is doing what we expect (build_count >0,
+# search_terms not empty, fingerprint_short_circuits incrementing on idle
+# warm scans). Not rate-limited — cheap, no I/O, dict lookups only.
+# ============================================================================
+
+@api_router.get("/index/stats", tags=["System"])
+async def get_index_stats():
+    return note_index.stats()
+
+
 # ============================================================================
 # Stats Endpoint (for dashboards)
 # ============================================================================

backend/utils.py

@@ -1,5 +1,9 @@
 """
-Utility functions for file operations, search, and markdown processing
+Utility functions for file operations, search, and markdown processing.
+
+Heavy work (backlinks, graph, full-text search, tag aggregation) is delegated
+to the in-memory index in note_index.py — every facade function there is a
+no-op when USE_NOTE_INDEX is False, so call sites here stay flag-free.
 """
 
 import os
@@ -7,6 +11,7 @@ import re
 import shutil
 import threading
 import time
+import traceback
 import urllib.parse
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
@@ -14,6 +19,9 @@ from pathlib import Path
 from typing import List, Dict, Optional, Tuple, Any, TypeVar, Callable
 from datetime import datetime, timezone
 
+from . import note_index
+from .note_index import NoteRecord, extract_links_from_content
+
 
 # ============================================================================
 # Pagination Support
@@ -112,14 +120,23 @@ def paginate(
     )
 
 
-# In-memory cache for parsed tags
+# ============================================================================
-# Format: {file_path: (mtime, tags)}
+# In-memory caches
-_tag_cache: Dict[str, Tuple[float, List[str]]] = {}
-
-# Notes tree scan cache (TTL).
 #
-# This avoids repeated full-directory walks when multiple endpoints (or the UI)
+# Two layers:
-# request indexes in quick succession.
+#   _tag_cache / _links_cache : per-file, mtime-keyed. Avoid re-parsing the
+#     same file on warm scans. Used by get_tags_and_links_cached.
+#   _SCAN_WALK_CACHE          : per-scan TTL cache. Avoids repeated full-
+#     directory walks when several endpoints fire in quick succession.
+#
+# All three are invalidated on every mutation (save/delete/move). The
+# note_index in note_index.py is the system of record for derived data
+# (links, backlinks, tags-by-name, search) — these caches just exist to
+# avoid double-reading files we already parsed.
+# ============================================================================
+
+_tag_cache: Dict[str, Tuple[float, List[str]]] = {}
+_links_cache: Dict[str, Tuple[float, Dict[str, List[str]]]] = {}
 
 _SCAN_WALK_CACHE_LOCK = threading.Lock()
 _SCAN_WALK_CACHE_TTL_SECONDS = 1.0
@@ -145,6 +162,66 @@ def _scan_cache_set(key: Tuple[str, bool], value: Tuple[List[Dict], List[str]])
         _SCAN_WALK_CACHE[key] = (time.monotonic(), value)
 
 
+def _scan_cache_invalidate() -> None:
+    """Drop the TTL scan cache. Called from every mutation handler so callers
+    that hit the cache immediately after a save/delete/move see the new state
+    instead of stale data from the previous walk."""
+    with _SCAN_WALK_CACHE_LOCK:
+        _SCAN_WALK_CACHE.clear()
+
+
+def get_tags_and_links_cached(
+    file_path: Path,
+    rel_path: Optional[str] = None,
+) -> Tuple[List[str], Dict[str, List[str]]]:
+    """Single-read fused extraction of tags + raw links, cached by mtime.
+
+    Three-tier lookup, cheapest first:
+      1. In-process per-file caches (warm scans on the live server)
+      2. NoteIndex (warm after snapshot load — avoids file reads on cold start)
+      3. Read the file from disk
+
+    rel_path enables tier 2. scan_notes_fast_walk passes it; ad-hoc callers
+    can omit it and just get tiers 1 + 3.
+    """
+    try:
+        mtime = file_path.stat().st_mtime
+        file_key = str(file_path)
+
+        tag_cached = _tag_cache.get(file_key)
+        link_cached = _links_cache.get(file_key)
+        tags_ok = tag_cached is not None and tag_cached[0] == mtime
+        links_ok = link_cached is not None and link_cached[0] == mtime
+
+        if tags_ok and links_ok:
+            return tag_cached[1], link_cached[1]
+
+        # Try the NoteIndex before reading the file. On a snapshot-warm
+        # startup, this turns a 10K-file cold scan into a no-I/O walk.
+        if rel_path is not None:
+            indexed = note_index.try_get_extraction(rel_path, mtime)
+            if indexed is not None:
+                idx_tags, idx_links = indexed
+                _tag_cache[file_key] = (mtime, idx_tags)
+                _links_cache[file_key] = (mtime, idx_links)
+                return idx_tags, idx_links
+
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+
+        tags = tag_cached[1] if tags_ok else parse_tags(content)
+        links = link_cached[1] if links_ok else extract_links_from_content(content)
+
+        if not tags_ok:
+            _tag_cache[file_key] = (mtime, tags)
+        if not links_ok:
+            _links_cache[file_key] = (mtime, links)
+
+        return tags, links
+    except Exception:
+        return [], {"wikilinks": [], "mdlinks": []}
+
+
 def validate_path_security(notes_dir: str, path: Path) -> bool:
     """
     Validate that a path is within the notes directory (security check).
@@ -176,15 +253,12 @@ def ensure_directories(config: dict):
 
 
 def create_folder(notes_dir: str, folder_path: str) -> bool:
-    """Create a new folder in the notes directory"""
+    """Create a new folder in the notes directory."""
     full_path = Path(notes_dir) / folder_path
-    
-    # Security check
     if not validate_path_security(notes_dir, full_path):
         return False
-    
     full_path.mkdir(parents=True, exist_ok=True)
-    
+    _scan_cache_invalidate()
     return True
 
 
@@ -219,60 +293,11 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
                 _scan_cache_set(cache_key, normalized_value)
                 return normalized_value
 
-    # TODO: remove this flag and the legacy branch below once the parallel
+    # Walk the vault once. Tag/link extraction is deferred out of the walk
-    # path has been validated in production with large vaults. To revert to
+    # (the expensive part is the file I/O) and parallelized below across files.
-    # the pre-parallel behavior, flip _USE_PARALLEL_TAG_SCAN to False.
-    _USE_PARALLEL_TAG_SCAN = True
-
     notes: List[Dict] = []
     folders_set = set()
-
+    md_to_extract: List[Tuple[int, Path, str]] = []  # (note_index, full_path, rel_path)
-    if not _USE_PARALLEL_TAG_SCAN:
-        # ===== LEGACY: sequential tag extraction inline with the walk =====
-        for root, dirnames, filenames in os.walk(notes_path):
-            # Skip descending into dot-dirs
-            dirnames[:] = [d for d in dirnames if not d.startswith('.')]
-
-            root_path = Path(root)
-            rel_folder = root_path.relative_to(notes_path).as_posix()
-            if rel_folder != "." and not rel_folder.startswith('.'):
-                folders_set.add(rel_folder)
-
-            for filename in filenames:
-                if filename.startswith('.'):
-                    continue
-
-                full_path = root_path / filename
-                try:
-                    st = full_path.stat()
-                except OSError:
-                    continue
-
-                relative_path = full_path.relative_to(notes_path)
-                media_type = get_media_type(filename) if include_media else None
-                is_markdown = full_path.suffix.lower() == '.md'
-                should_include = is_markdown or (include_media and media_type is not None)
-
-                if not should_include:
-                    continue
-
-                folder = relative_path.parent.as_posix()
-                tags = get_tags_cached(full_path) if is_markdown else []
-                notes.append({
-                    "name": full_path.stem,
-                    "path": relative_path.as_posix(),
-                    "folder": "" if folder == "." else folder,
-                    "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
-                    "size": st.st_size,
-                    "type": media_type if media_type else "note",
-                    "tags": tags,
-                })
-    else:
-        # ===== NEW: tag extraction deferred out of the walk and parallelized =====
-        # Indices of markdown entries in `notes` plus their full paths, so we can
-        # defer tag extraction (the expensive part) out of the walk and parallelize
-        # it across files. Tags are filled in after the walk completes.
-        md_to_tag: List[Tuple[int, Path]] = []
 
     for root, dirnames, filenames in os.walk(notes_path):
         dirnames[:] = [d for d in dirnames if not d.startswith('.')]
@@ -296,36 +321,71 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
             media_type = get_media_type(filename) if include_media else None
             is_markdown = full_path.suffix.lower() == '.md'
             should_include = is_markdown or (include_media and media_type is not None)
-
             if not should_include:
                 continue
 
             folder = relative_path.parent.as_posix()
+            rel_str = relative_path.as_posix()
             notes.append({
                 "name": full_path.stem,
-                    "path": relative_path.as_posix(),
+                "path": rel_str,
                 "folder": "" if folder == "." else folder,
                 "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
                 "size": st.st_size,
                 "type": media_type if media_type else "note",
                 "tags": [],
+                "_mtime": st.st_mtime,  # internal — popped before returning
             })
             if is_markdown:
-                    md_to_tag.append((len(notes) - 1, full_path))
+                md_to_extract.append((len(notes) - 1, full_path, rel_str))
 
-        # Parallelize for large vaults (sequential branch avoids thread-spawn
+    # Fused tag + raw-link extraction. mtime-keyed cache makes warm scans
-        # overhead for small vaults). get_tags_cached is mtime-keyed so this is
+    # mostly free; cold scans win from parallel I/O.
-        # mostly free on warm cache; the win is the cold first call.
+    #
-        if md_to_tag:
+    # We don't read full content here — that would double the parsing cost
-            if len(md_to_tag) >= 50:
+    # for every scan, and the search index is the only thing that needs it.
+    # Search is built lazily on the first search request (see
+    # note_index.ensure_search_index_built), which keeps startup fast and
+    # only pays the cost when a user actually searches.
+    sources_raw: Dict[str, Dict[str, List[str]]] = {}
+    if md_to_extract:
+        # Pass rel_path so each call can hit the NoteIndex before falling
+        # back to a file read — critical for fast startup after a snapshot
+        # load on large vaults.
+        path_pairs = [(full, rel) for (_, full, rel) in md_to_extract]
+        if len(md_to_extract) >= 50:
             workers = min(8, (os.cpu_count() or 4))
-                paths = [p for _, p in md_to_tag]
             with ThreadPoolExecutor(max_workers=workers) as ex:
-                    for (idx, _), tags in zip(md_to_tag, ex.map(get_tags_cached, paths)):
+                extraction = list(ex.map(
-                        notes[idx]["tags"] = tags
+                    lambda pair: get_tags_and_links_cached(pair[0], pair[1]),
+                    path_pairs,
+                ))
         else:
-                for idx, p in md_to_tag:
+            extraction = [get_tags_and_links_cached(full, rel) for (full, rel) in path_pairs]
-                    notes[idx]["tags"] = get_tags_cached(p)
+
+        for (idx, _full, rel_str), (tags, links) in zip(md_to_extract, extraction):
+            notes[idx]["tags"] = tags
+            sources_raw[rel_str] = links
+
+    # Populate the unified index. Cheap when the fingerprint matches (warm
+    # scan, nothing changed) — short-circuits internally. No-op when off.
+    notes_meta = [
+        NoteRecord(
+            path=n["path"],
+            name=n["name"],
+            folder=n["folder"],
+            modified=n["modified"],
+            size=n["size"],
+            type=n["type"],
+            mtime=n["_mtime"],
+            tags=tuple(n["tags"]),
+        )
+        for n in notes
+    ]
+    note_index.populate_from_scan(notes_meta, folders_set, sources_raw)
+
+    for n in notes:
+        n.pop("_mtime", None)
 
     value = (sorted(notes, key=lambda x: x.get('modified', ''), reverse=True), sorted(folders_set))
     if use_cache:
@@ -333,132 +393,106 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
     return value
 
 def move_note(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]:
-    """Move a note to a different location
+    """Move a note. Returns (success, error_message)."""
-    
-    Returns:
-        Tuple of (success: bool, error_message: str)
-    """
     old_full_path = Path(notes_dir) / old_path
     new_full_path = Path(notes_dir) / new_path
 
-    # Security checks
     if not validate_path_security(notes_dir, old_full_path):
         return False, "Invalid source path"
     if not validate_path_security(notes_dir, new_full_path):
         return False, "Invalid destination path"
-    
     if not old_full_path.exists():
         return False, f"Source note does not exist: {old_path}"
-    
-    # Check if target already exists (prevent overwriting)
     if new_full_path.exists():
         return False, f"A note already exists at: {new_path}"
 
-    # Invalidate cache for old path
+    _drop_path_caches(old_full_path)
-    old_key = str(old_full_path)
-    if old_key in _tag_cache:
-        del _tag_cache[old_key]
 
     try:
-        # Create parent directory if needed
         new_full_path.parent.mkdir(parents=True, exist_ok=True)
-        
-        # Move the file
         old_full_path.rename(new_full_path)
     except Exception as e:
         return False, f"Failed to move file: {str(e)}"
 
-    # Note: We don't automatically delete empty folders to preserve user's folder structure
+    note_index.on_note_renamed(notes_dir, old_full_path, new_full_path)
-    
+    _scan_cache_invalidate()
     return True, ""
 
 
 def move_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]:
-    """Move a folder to a different location
+    """Move a folder. Returns (success, error_message)."""
-    
-    Returns:
-        Tuple of (success: bool, error_message: str)
-    """
-    import shutil
-    
     old_full_path = Path(notes_dir) / old_path
     new_full_path = Path(notes_dir) / new_path
 
-    # Security checks
     if not validate_path_security(notes_dir, old_full_path):
         return False, "Invalid source path"
     if not validate_path_security(notes_dir, new_full_path):
         return False, "Invalid destination path"
-    
     if not old_full_path.exists() or not old_full_path.is_dir():
         return False, f"Source folder does not exist: {old_path}"
-    
-    # Check if target already exists
     if new_full_path.exists():
         return False, f"A folder already exists at: {new_path}"
 
-    # Invalidate cache for all notes in this folder
+    _drop_prefix_caches(old_full_path)
-    global _tag_cache
-    old_path_str = str(old_full_path)
-    keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(old_path_str)]
-    for key in keys_to_delete:
-        del _tag_cache[key]
 
     try:
-        # Create parent directory if needed
         new_full_path.parent.mkdir(parents=True, exist_ok=True)
-        
-        # Move the folder
         shutil.move(str(old_full_path), str(new_full_path))
     except Exception as e:
         return False, f"Failed to move folder: {str(e)}"
 
-    # Note: We don't automatically delete empty folders to preserve user's folder structure
+    note_index.on_folder_renamed(notes_dir, old_full_path, new_full_path)
-    
+    _scan_cache_invalidate()
     return True, ""
 
 
 def rename_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]:
-    """Rename a folder (same as move but for clarity)"""
+    """Rename a folder (same as move, named for clarity)."""
     return move_folder(notes_dir, old_path, new_path)
 
 
 def delete_folder(notes_dir: str, folder_path: str) -> bool:
-    """Delete a folder and all its contents"""
+    """Delete a folder and all its contents."""
     try:
         full_path = Path(notes_dir) / folder_path
 
-        # Security check: ensure the path is within notes_dir
         if not validate_path_security(notes_dir, full_path):
             print(f"Security: Path is outside notes directory: {full_path}")
             return False
-        
         if not full_path.exists():
             print(f"Folder does not exist: {full_path}")
             return False
-            
         if not full_path.is_dir():
             print(f"Path is not a directory: {full_path}")
             return False
 
-        # Invalidate cache for all notes in this folder
+        _drop_prefix_caches(full_path)
-        global _tag_cache
-        folder_path_str = str(full_path)
-        keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(folder_path_str)]
-        for key in keys_to_delete:
-            del _tag_cache[key]
-        
-        # Delete the folder and all its contents
         shutil.rmtree(full_path)
-        print(f"Successfully deleted folder: {full_path}")
+        note_index.on_folder_deleted(notes_dir, full_path)
+        _scan_cache_invalidate()
         return True
     except Exception as e:
         print(f"Error deleting folder '{folder_path}': {e}")
-        import traceback
         traceback.print_exc()
         return False
 
 
+def _drop_path_caches(full_path: Path) -> None:
+    """Evict the per-file mtime caches for a single note."""
+    key = str(full_path)
+    _tag_cache.pop(key, None)
+    _links_cache.pop(key, None)
+
+
+def _drop_prefix_caches(folder_full_path: Path) -> None:
+    """Evict per-file mtime caches for every entry under a folder."""
+    prefix = str(folder_full_path)
+    for k in [k for k in _tag_cache if k.startswith(prefix)]:
+        _tag_cache.pop(k, None)
+    for k in [k for k in _links_cache if k.startswith(prefix)]:
+        _links_cache.pop(k, None)
+
+
 
 
 def get_note_content(notes_dir: str, note_path: str) -> Optional[str]:
@@ -477,110 +511,111 @@ def get_note_content(notes_dir: str, note_path: str) -> Optional[str]:
 
 
 def save_note(notes_dir: str, note_path: str, content: str) -> bool:
-    """Save or update a note"""
+    """Save or update a note."""
     full_path = Path(notes_dir) / note_path
-    
-    # Ensure .md extension
     if not note_path.endswith('.md'):
         full_path = full_path.with_suffix('.md')
 
-    # Security check
     if not validate_path_security(notes_dir, full_path):
         return False
 
-    # Create parent directories if needed
     full_path.parent.mkdir(parents=True, exist_ok=True)
-    
     with open(full_path, 'w', encoding='utf-8') as f:
         f.write(content)
 
+    # Refresh the per-file mtime caches with what we just wrote, so the next
+    # scan doesn't re-parse this file. extract_links + parse_tags + save are
+    # tiny relative to the file write.
+    try:
+        mtime = full_path.stat().st_mtime
+        file_key = str(full_path)
+        _tag_cache[file_key] = (mtime, parse_tags(content))
+        _links_cache[file_key] = (mtime, extract_links_from_content(content))
+    except Exception:
+        pass  # caches are best-effort
+
+    note_index.on_note_saved(notes_dir, full_path, content)
+    _scan_cache_invalidate()
     return True
 
 
 def delete_note(notes_dir: str, note_path: str) -> bool:
-    """Delete a note"""
+    """Delete a note."""
     full_path = Path(notes_dir) / note_path
-    
     if not full_path.exists():
         return False
-    
-    # Security check
     if not validate_path_security(notes_dir, full_path):
         return False
 
-    # Invalidate cache for this note
+    _drop_path_caches(full_path)
-    file_key = str(full_path)
-    if file_key in _tag_cache:
-        del _tag_cache[file_key]
-    
     full_path.unlink()
-    
+    note_index.on_note_deleted(notes_dir, full_path)
-    # Note: We don't automatically delete empty folders to preserve user's folder structure
+    _scan_cache_invalidate()
-    
     return True
 
 
 def search_notes(notes_dir: str, query: str) -> List[Dict]:
-    """
+    """Full-text search through note contents.
-    Full-text search through note contents only.
+
-    Does NOT search in file names, folder names, or paths - only note content.
+    Only searches inside note content (not filenames or paths). The index, when
-    Uses character-based context extraction with highlighted matches.
+    available, narrows down the candidate files via a token-AND match on its
+    inverted index — then the existing per-line snippet extraction runs on the
+    narrow set. Output is byte-identical to a full scan.
     """
     from html import escape
-    results = []
+    results: List[Dict] = []
     notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
 
+    # Build the search index lazily on the first search request. Subsequent
+    # searches hit the warm index; the first one pays the cost (read every
+    # file once) and is roughly as slow as the legacy full scan would be.
+    note_index.ensure_search_index(notes_dir)
+    candidates = note_index.try_search_candidates(query)
+
     for note in notes:
-        md_file = Path(notes_dir) / note["path"]
+        path = note["path"]
+        if candidates is not None and path not in candidates:
+            continue
+        md_file = Path(notes_dir) / path
         try:
             with open(md_file, 'r', encoding='utf-8') as f:
                 content = f.read()
 
-            # Find all matches using regex (case-insensitive)
             matches = list(re.finditer(re.escape(query), content, re.IGNORECASE))
+            if not matches:
+                continue
 
-            if matches:
             matched_lines = []
-                
+            for match in matches[:3]:
-                for match in matches[:3]:  # Limit to 3 matches per file
                 start_index = match.start()
                 end_index = match.end()
-                    matched_text = match.group()  # Preserve original case
+                matched_text = match.group()
 
-                    # Create slice window: ±15 characters around match
                 context_start = max(0, start_index - 15)
                 context_end = min(len(content), end_index + 15)
 
-                    # Extract and clean parts (newlines → spaces)
                 before = escape(content[context_start:start_index].replace('\n', ' '))
                 after = escape(content[end_index:context_end].replace('\n', ' '))
                 matched_clean = escape(matched_text.replace('\n', ' '))
 
-                    # Build snippet with <mark> highlight (styled via CSS)
                 snippet = f'{before}<mark class="search-highlight">{matched_clean}</mark>{after}'
-                    
-                    # Add ellipsis if truncated at start
                 if context_start > 0:
                     snippet = '...' + snippet
-                    
-                    # Add ellipsis if truncated at end
                 if context_end < len(content):
                     snippet = snippet + '...'
 
-                    # Calculate line number by counting newlines up to match start
                 line_number = content.count('\n', 0, start_index) + 1
-                    
                 matched_lines.append({
                     "line_number": line_number,
-                        "context": snippet
+                    "context": snippet,
                 })
 
-                relative_path = Path(note["path"])
+            relative_path = Path(path)
             results.append({
                 "name": md_file.stem,
                 "path": str(relative_path.as_posix()),
                 "folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "",
-                    "matches": matched_lines
+                "matches": matched_lines,
             })
         except Exception:
             continue
@@ -852,105 +887,66 @@ def parse_tags(content: str) -> List[str]:
         return []
 
 
-def get_tags_cached(file_path: Path) -> List[str]:
-    """
-    Get tags for a file with caching based on modification time.
-    
-    Args:
-        file_path: Path to the markdown file
-        
-    Returns:
-        List of tags from the file (cached if mtime unchanged)
-    """
-    global _tag_cache
-    
-    try:
-        # Get current modification time
-        mtime = file_path.stat().st_mtime
-        file_key = str(file_path)
-        
-        # Check cache
-        if file_key in _tag_cache:
-            cached_mtime, cached_tags = _tag_cache[file_key]
-            if cached_mtime == mtime:
-                # Cache hit! Return cached tags
-                return cached_tags
-        
-        # Cache miss or stale - parse tags
-        with open(file_path, 'r', encoding='utf-8') as f:
-            content = f.read()
-            tags = parse_tags(content)
-        
-        # Update cache
-        _tag_cache[file_key] = (mtime, tags)
-        return tags
-        
-    except Exception:
-        # If anything fails, return empty list
-        return []
-
-
-def clear_tag_cache():
-    """Clear the tag cache (useful for testing or manual cache invalidation)"""
-    global _tag_cache
-    _tag_cache.clear()
-
-
 def get_all_tags(notes_dir: str) -> Dict[str, int]:
-    """
+    """All tags in the vault with note counts. Index-backed when available."""
-    Get all tags used across all notes with their count (cached).
+    # Fast path FIRST — when the index is ready it knows every tag count,
+    # so we skip the scan entirely (no disk walk, no JSON of 10K notes).
+    indexed = note_index.try_all_tags()
+    if indexed is not None:
+        return indexed
 
-    Args:
+    # Legacy fallback: scan the vault and aggregate.
-        notes_dir: Directory containing notes
-        
-    Returns:
-        Dictionary mapping tag names to note counts
-    """
-    tag_counts = {}
     notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
-
+    tag_counts: Dict[str, int] = {}
     for note in notes:
-        md_file = Path(notes_dir) / note["path"]
+        for tag in note.get("tags", []):
-        # Get tags using cache
-        tags = get_tags_cached(md_file)
-        
-        for tag in tags:
             tag_counts[tag] = tag_counts.get(tag, 0) + 1
-    
     return dict(sorted(tag_counts.items()))
 
 
 def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]:
-    """
+    """All notes carrying `tag` (case-insensitive). Index-backed when available."""
-    Get all notes that have a specific tag (cached).
-    
-    Args:
-        notes_dir: Directory containing notes
-        tag: Tag to filter by (case-insensitive)
-        
-    Returns:
-        List of note dictionaries matching the tag
-    """
-    matching_notes = []
     tag_lower = tag.lower()
+
+    # Fast path FIRST: every field we need lives in the NoteRecord. Skip
+    # the vault scan when the index can answer directly.
+    indexed_paths = note_index.try_notes_by_tag(tag_lower)
+    if indexed_paths is not None:
+        idx = note_index.get_index()
+        records = [idx.get_note_record(p) for p in indexed_paths]
+        matching = [
+            {
+                "name": r.name,
+                "path": r.path,
+                "folder": r.folder,
+                "modified": r.modified,
+                "size": r.size,
+                "tags": list(r.tags),
+            }
+            for r in records
+            if r is not None and r.type == "note"
+        ]
+        # Match legacy sort order (scan returns notes sorted by modified desc).
+        matching.sort(key=lambda n: n["modified"], reverse=True)
+        return matching
+
+    # Legacy fallback.
     notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
-
+    matching: List[Dict] = []
     for note in notes:
-        md_file = Path(notes_dir) / note["path"]
+        if note.get("type") != "note":
-        # Get tags using cache
+            continue
-        tags = get_tags_cached(md_file)
+        tags = note.get("tags", [])
-        
         if tag_lower in tags:
-            matching_notes.append({
+            matching.append({
                 "name": note["name"],
                 "path": note["path"],
                 "folder": note["folder"],
                 "modified": note["modified"],
                 "size": note["size"],
-                "tags": tags
+                "tags": tags,
             })
-    
+    return matching
-    return matching_notes
 
 
 # ============================================================================
@@ -1107,69 +1103,38 @@ def apply_template_placeholders(content: str, note_path: str) -> str:
     return result
 
 
-def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
+def _extract_backlink_references(
+    notes_dir: str,
+    source_path: str,
+    target_path: str,
+    target_path_lower: str,
+    target_path_no_ext_lower: str,
+    wikilink_refs: set,
+) -> List[Dict]:
+    """Read one source file and pull line-level references that point to target.
+
+    Same regex + resolution rules as the legacy get_backlinks loop body; just
+    factored out so both the index-driven and the legacy code paths use the
+    exact same context-extraction logic.
     """
-    Find all notes that link TO the specified note (reverse links / backlinks).
-
-    Args:
-        notes_dir: Base directory containing notes
-        target_note_path: Path of the note to find backlinks for
-
-    Returns:
-        List of backlink objects with path, context, and line_number
-    """
-    backlinks = []
-    notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
-    
-    # Normalize target path for matching
-    target_path = target_note_path
-    target_path_lower = target_path.lower()
-    target_path_no_ext = target_path.replace('.md', '')
-    target_path_no_ext_lower = target_path_no_ext.lower()
-    target_name = Path(target_path).stem.lower()
-    
-    # For wikilinks: global name matching (find note anywhere by name)
-    wikilink_refs = {
-        target_path_lower,
-        target_path_no_ext_lower,
-        target_name,
-    }
-    
-    for note in notes:
-        if note.get('type') != 'note':
-            continue
-        
-        source_path = note['path']
-        
-        # Skip self-references
-        if source_path == target_path:
-            continue
-        
-        # Get source folder for resolving relative markdown links
     source_folder = str(Path(source_path).parent).replace('\\', '/')
     if source_folder == '.':
         source_folder = ''
 
-        # Read note content
     full_path = Path(notes_dir) / source_path
     try:
         with open(full_path, 'r', encoding='utf-8') as f:
             content = f.read()
     except Exception:
-            continue
+        return []
 
     lines = content.split('\n')
-        found_links = []
+    found_links: List[Dict] = []
 
     for line_num, line in enumerate(lines, 1):
-            # Find wikilinks: [[target]] or [[target|display]]
+        for match in re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line):
-            # Wikilinks use GLOBAL matching (find note anywhere by name)
-            wikilink_matches = re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line)
-            for match in wikilink_matches:
             link_target = match.group(1).strip().lower()
             link_target_no_ext = link_target.replace('.md', '')
-                
-                # Check if this wikilink points to our target (global match)
             if link_target in wikilink_refs or link_target_no_ext in wikilink_refs:
                 start = max(0, match.start() - 30)
                 end = min(len(line), match.end() + 30)
@@ -1178,47 +1143,34 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
                     context = '...' + context
                 if end < len(line):
                     context = context + '...'
-                    
                 found_links.append({
                     "line_number": line_num,
                     "context": context,
-                        "type": "wikilink"
+                    "type": "wikilink",
                 })
 
-            # Find markdown links: [text](path)
+        for match in re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line):
-            # Markdown links must RESOLVE as paths (relative to source or absolute)
+            link_path = match.group(2).split('#')[0]
-            markdown_matches = re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line)
-            for match in markdown_matches:
-                link_path = match.group(2).split('#')[0]  # Remove anchor
             if not link_path:
                 continue
-                
             link_path = urllib.parse.unquote(link_path)
             if link_path.startswith('./'):
                 link_path = link_path[2:]
-                
-                # Add .md if not present
             link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md'
 
-                # Resolve the link path to get the actual target
             resolved_path = None
-                
-                # 1. Try resolving relative to source folder
             if source_folder and not link_path.startswith('/'):
                 relative_path = f"{source_folder}/{link_path_with_md}"
                 if relative_path.lower() == target_path_lower:
                     resolved_path = target_path
                 elif f"{source_folder}/{link_path}".lower() == target_path_no_ext_lower:
                     resolved_path = target_path
-                
-                # 2. Try as absolute path from root
             if not resolved_path:
                 if link_path_with_md.lower() == target_path_lower:
                     resolved_path = target_path
                 elif link_path.lower() == target_path_no_ext_lower:
                     resolved_path = target_path
 
-                # Only add if the resolved path matches the target
             if resolved_path:
                 start = max(0, match.start() - 30)
                 end = min(len(line), match.end() + 30)
@@ -1227,19 +1179,61 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
                     context = '...' + context
                 if end < len(line):
                     context = context + '...'
-                    
                 found_links.append({
                     "line_number": line_num,
                     "context": context,
-                        "type": "markdown"
+                    "type": "markdown",
                 })
 
-        # If we found links in this note, add it to backlinks
+    return found_links
-        if found_links:
+
+
+def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
+    """Find all notes that link TO the specified note.
+
+    Index-backed when available: the index hands us a superset of candidate
+    source files (O(1)), we read only those, then the per-line matcher
+    filters down to true matches with line-level context. Output is
+    byte-identical to a full vault scan.
+    """
+    target_path = target_note_path
+    target_path_lower = target_path.lower()
+    target_path_no_ext_lower = target_path_lower.replace('.md', '')
+    wikilink_refs = {
+        target_path_lower,
+        target_path_no_ext_lower,
+        Path(target_path).stem.lower(),
+    }
+
+    # scan_notes_fast_walk is TTL-cached, so calling it here is a dict-lookup
+    # in the warm case. We need it for the canonical ordering of `notes`.
+    notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
+
+    candidates = note_index.try_backlink_candidates(target_path)
+
+    backlinks: List[Dict] = []
+    for note in notes:
+        if note.get('type') != 'note':
+            continue
+        source_path = note['path']
+        if source_path == target_path:
+            continue
+        if candidates is not None and source_path not in candidates:
+            continue
+
+        refs = _extract_backlink_references(
+            notes_dir,
+            source_path,
+            target_path,
+            target_path_lower,
+            target_path_no_ext_lower,
+            wikilink_refs,
+        )
+        if refs:
             backlinks.append({
                 "path": source_path,
                 "name": note['name'].replace('.md', ''),
-                "references": found_links[:3]  # Limit to 3 references per note
+                "references": refs[:3],
             })
 
     return backlinks

documentation/API.md

@@ -580,6 +580,59 @@ Returns application statistics at a glance. Designed for dashboard widgets (e.g.
           label: Version
 ```
 
+### Get Index Stats
+```http
+GET /api/index/stats
+```
+Returns internal counters and sizes for the in-memory note index. Useful for confirming the index is built and observing its growth — handy when debugging slow endpoints or verifying that incremental updates are firing on every save/delete.
+
+The index is rebuilt on the first scan after each process start and updated incrementally on every save/delete/move. It lives in process memory only — no disk persistence.
+
+**Response:**
+```json
+{
+  "enabled": true,
+  "built": true,
+  "search_built": false,
+  "notes": 142,
+  "folders": 12,
+  "tags": 37,
+  "links_forward_entries": 89,
+  "links_backward_entries": 76,
+  "wikilink_tokens": 134,
+  "search_terms": 0,
+  "counters": {
+    "build_count": 1,
+    "last_build_ms": 12.4,
+    "last_built_at": "2026-03-17T14:32:00+00:00",
+    "incremental_updates": 8,
+    "fingerprint_short_circuits": 3,
+    "search_build_count": 0,
+    "last_search_build_ms": 0.0
+  }
+}
+```
+
+| Field | Description |
+|-------|-------------|
+| `enabled` | Whether the index is active (always `true` in current builds) |
+| `built` | `true` after the first vault scan completes |
+| `search_built` | `true` after the first `/api/search` request (search index is built lazily) |
+| `notes` | Number of note records currently held in the index |
+| `folders` | Number of folder paths |
+| `tags` | Number of unique tags |
+| `links_forward_entries` | Number of notes that link out to at least one other note |
+| `links_backward_entries` | Number of notes that have at least one incoming backlink |
+| `wikilink_tokens` | Number of unique wikilink tokens (used for loose backlink matching by stem name) |
+| `search_terms` | Number of unique terms in the full-text inverted search index (0 until first search) |
+| `counters.build_count` | Total number of full index rebuilds since process start |
+| `counters.last_build_ms` | Wall-clock time of the most recent full rebuild |
+| `counters.last_built_at` | ISO timestamp of the most recent full rebuild |
+| `counters.incremental_updates` | Number of single-note updates applied (save/delete/rename) |
+| `counters.fingerprint_short_circuits` | Times a re-scan was skipped because the vault hash matched the previous scan |
+| `counters.search_build_count` | Total number of search-index rebuilds |
+| `counters.last_search_build_ms` | Wall-clock time of the most recent search-index rebuild |
+
 ### Health Check
 ```http
 GET /health