builderadmin
/
NoteDiscovery
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

use index for note backlink

This commit is contained in:
Gamosoft 2026-06-30 11:59:25 +02:00
parent 4b316b2cc7
commit 0cd5332a99
4 changed files with 166 additions and 643 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
backend/main.py
View File

@ -46,7 +46,6 @@ from .utils import (
get_backlinks, get_backlinks,
) )
from . import note_index from . import note_index
from .note_index import USE_NOTE_INDEX
from .plugins import PluginManager from .plugins import PluginManager
from .themes import get_available_themes, get_theme_css from .themes import get_available_themes, get_theme_css
from .share import ( from .share import (
@ -1486,186 +1485,15 @@ async def search(
@api_router.get("/graph", tags=["Graph"]) @api_router.get("/graph", tags=["Graph"])
async def get_graph(): async def get_graph():
"""Get graph data for note visualization with wikilink and markdown link detection.""" """Graph data (nodes + resolved wikilink/markdown edges) for the visualizer."""
try: try:
import re
import urllib.parse
# Fast path: index already holds resolved edges. Ensure a scan has
# populated it (first request after startup, or after an invalidate)
# then ask the facade for a snapshot.
if USE_NOTE_INDEX:
if not note_index.get_index().is_built(): if not note_index.get_index().is_built():
scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
indexed = note_index.try_graph_data() nodes_paths, edges_tuples = note_index.get_graph_data()
if indexed is not None:
nodes_paths, edges_tuples = indexed
return { return {
"nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths], "nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths],
"edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples], "edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples],
} }
notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
nodes = []
edges = []
# Build set of valid note names/paths for matching
note_paths = set()
note_paths_lower = {} # Map lowercase path -> actual path for case-insensitive matching
note_names = {} # Map name -> path for quick lookup
for note in notes:
if note.get('type') == 'note':
note_paths.add(note['path'])
note_paths.add(note['path'].replace('.md', ''))
# Store lowercase path -> actual path mapping for case-insensitive matching
note_paths_lower[note['path'].lower()] = note['path']
note_paths_lower[note['path'].replace('.md', '').lower()] = note['path']
# Store name -> path mapping (without extension)
name = note['name'].replace('.md', '')
note_names[name.lower()] = note['path']
note_names[note['name'].lower()] = note['path']
# Build graph structure with link detection
for note in notes:
if note.get('type') == 'note':
nodes.append({
"id": note['path'],
"label": note['name'].replace('.md', '')
})
# Read note content to find links
content = get_note_content(config['storage']['notes_dir'], note['path'])
if content:
# Find wikilinks: [[target]] or [[target|display]]
wikilinks = re.findall(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', content)
# Find standard markdown internal links: [text](path) - any local path (not http/https)
# Match links that don't start with http://, https://, mailto:, #, etc.
markdown_links = re.findall(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', content)
# Get source note's folder for resolving relative links
# Use forward slashes consistently (note_paths uses forward slashes)
source_folder = str(Path(note['path']).parent).replace('\\', '/')
if source_folder == '.':
source_folder = ''
# Process wikilinks
for target in wikilinks:
target = target.strip()
target_lower = target.lower()
# Try to match target to an existing note
target_path = None
# 1. Try resolving relative to source note's folder first
if source_folder and '/' not in target:
relative_path = f"{source_folder}/{target}"
relative_path_lower = relative_path.lower()
if relative_path in note_paths:
target_path = relative_path if relative_path.endswith('.md') else relative_path + '.md'
elif relative_path + '.md' in note_paths:
target_path = relative_path + '.md'
elif relative_path_lower in note_paths_lower:
target_path = note_paths_lower[relative_path_lower]
elif relative_path_lower + '.md' in note_paths_lower:
target_path = note_paths_lower[relative_path_lower + '.md']
# 2. Exact path match (absolute or already has folder)
if not target_path:
if target in note_paths:
target_path = target if target.endswith('.md') else target + '.md'
elif target + '.md' in note_paths:
target_path = target + '.md'
# 3. Case-insensitive path match (e.g., [[Folder/Note]] -> folder/note.md)
elif target_lower in note_paths_lower:
target_path = note_paths_lower[target_lower]
elif target_lower + '.md' in note_paths_lower:
target_path = note_paths_lower[target_lower + '.md']
# 4. Just note name (case-insensitive) - global match
elif target_lower in note_names:
target_path = note_names[target_lower]
if target_path and target_path != note['path']:
edges.append({
"source": note['path'],
"target": target_path,
"type": "wikilink"
})
# Process markdown links
for _, link_path in markdown_links:
# Skip anchor-only links and external protocols
if not link_path or link_path.startswith('#'):
continue
# Remove anchor part if present (e.g., "note.md#section" -> "note.md")
link_path = link_path.split('#')[0]
if not link_path:
continue
# Normalize path: remove ./ prefix, handle URL encoding
link_path = urllib.parse.unquote(link_path)
if link_path.startswith('./'):
link_path = link_path[2:]
# Add .md extension if not present and doesn't have other extension
link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md'
# Try to match target to an existing note
target_path = None
# 1. First, try resolving relative to source note's folder
if source_folder and not link_path.startswith('/'):
relative_path = f"{source_folder}/{link_path}"
relative_path_with_md = f"{source_folder}/{link_path_with_md}"
relative_path_lower = relative_path.lower()
relative_path_with_md_lower = relative_path_with_md.lower()
if relative_path in note_paths:
target_path = relative_path if relative_path.endswith('.md') else relative_path + '.md'
elif relative_path_with_md in note_paths:
target_path = relative_path_with_md
elif relative_path_lower in note_paths_lower:
target_path = note_paths_lower[relative_path_lower]
elif relative_path_with_md_lower in note_paths_lower:
target_path = note_paths_lower[relative_path_with_md_lower]
# 2. Try exact path match from root (for absolute paths or notes at root)
if not target_path:
link_path_lower = link_path.lower()
link_path_with_md_lower = link_path_with_md.lower()
if link_path in note_paths:
target_path = link_path if link_path.endswith('.md') else link_path + '.md'
elif link_path_with_md in note_paths:
target_path = link_path_with_md
# Case-insensitive path match
elif link_path_lower in note_paths_lower:
target_path = note_paths_lower[link_path_lower]
elif link_path_with_md_lower in note_paths_lower:
target_path = note_paths_lower[link_path_with_md_lower]
# No global filename fallback for markdown links - they must resolve as paths
if target_path and target_path != note['path']:
edges.append({
"source": note['path'],
"target": target_path,
"type": "markdown"
})
# Remove duplicate edges
seen = set()
unique_edges = []
for edge in edges:
key = (edge['source'], edge['target'])
if key not in seen:
seen.add(key)
unique_edges.append(edge)
return {"nodes": nodes, "edges": unique_edges}
except Exception as e: except Exception as e:
raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to generate graph data")) raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to generate graph data"))

419
backend/note_index.py
View File

@ -1,77 +1,12 @@
""" """
NoteIndex — unified in-memory index of everything that's expensive to recompute NoteIndex — in-memory index of vault state: notes, folders, tags, links, search.
from disk: links, tags, full-text search, and per-note metadata.
+---------------------------------------------------------------+ Backs /api/notes, /api/tags, /api/backlinks, /api/graph, /api/search. Keeps
| ONE module, ONE singleton, ONE lock, ONE rollback switch. | everything thread-safe under one RLock. Process-memory only — rebuilds on the
| All call sites in utils.py / main.py are one-line shims. | first /api/notes request after each process start (~1-3s on a 10K-note vault).
+---------------------------------------------------------------+
Why this exists Updated incrementally on every save/delete/move/rename through the on_*
--------------- facades at the bottom of this file.
Without an index, every backlink lookup, every graph render, and every text
search re-walks the entire vault and re-reads every file. That's O(N) per
request and scales linearly with vault size — 4-second note loads on a 10K
vault, 5-10s graph renders, multi-second searches. With an index, those
endpoints become O(matches) — milliseconds regardless of vault size.
What's indexed
--------------
* Note metadata (path -> name, folder, mtime, size, type, tags)
* Folders (set of folder paths)
* Tags forward (path -> sorted tags)
* Tags backward (tag -> set of paths)
* Links forward (source_path -> {target_path: "wikilink"|"markdown"})
* Links backward (target_path -> set of source paths) strict
* Wikilink tokens (lowercased token -> set of source paths) loose
* Search inverted (lowercased term -> set of paths)
What's NOT cached
-----------------
File content. Reading file content for line-level context (backlinks, search
snippets) is done only for the small set of matched files, never for the
whole vault.
Threading model
---------------
Everything mutating goes through one RLock. Reads return snapshot copies, so
callers can iterate freely after they release the lock. The index is
designed to be called from FastAPI request handlers concurrently.
Lifetime
--------
Process-memory only. No persistence to disk — keeping the app lightweight
matters more than the ~500ms saved on cold restart for a 10K-note vault.
The index rebuilds on the first `/api/notes` request after every restart
(typically 1-3 seconds, dominated by reading every file once for tag/link
extraction). F5 / browser reload doesn't touch this — only Python process
restart does.
Rollback switch
---------------
USE_NOTE_INDEX (below) is the single, global on/off. Flip it to False and
every facade function becomes a no-op or returns None, and every call site
in utils.py / main.py falls through to the legacy file-scanning behavior.
ROLLBACK RECIPE (decide the index isn't worth it)
-------------------------------------------------
Set USE_NOTE_INDEX = False below. That's it. Every try_* facade returns
None, every on_* facade becomes a no-op, callers fall through to the
legacy file-scanning paths that are preserved as the function bodies.
COMMIT RECIPE (you're happy, drop the legacy paths)
---------------------------------------------------
1. Here: delete USE_NOTE_INDEX and inline `if not USE_NOTE_INDEX` to True
in every try_/on_ facade.
2. In backend/utils.py:
- get_backlinks: drop the `candidates is None` branch — the
index always provides them.
- search_notes: drop the `candidates is None` fallback.
- get_all_tags / get_notes_by_tag: drop the legacy aggregation
tail block.
3. In backend/main.py:
- /api/graph: drop the legacy fallback block at the bottom of
the endpoint, keep only `try_graph_data`.
""" """
from __future__ import annotations from __future__ import annotations
@ -87,40 +22,20 @@ from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
# ============================================================================ # Below this many markdown files, parallel tag extraction adds more overhead
# Configuration # than it saves.
# ============================================================================
# Single global rollback switch. Flip to False to disable the index and every
# call site in utils.py / main.py falls back to the legacy code paths.
USE_NOTE_INDEX: bool = True
# Parallelism cutoff for full rescans. Below this many markdown files, threads
# add more overhead than they save (small vaults stay sequential).
_PARALLEL_CUTOFF = 50 _PARALLEL_CUTOFF = 50
_PARALLEL_WORKERS = min(8, (os.cpu_count() or 4)) _PARALLEL_WORKERS = min(8, (os.cpu_count() or 4))
# Search tokenization. Lowercase, split on anything that's not a word char. # Search tokenization. Min length 2 keeps single-letter noise out without
# Min length 2 keeps single-letter noise out of the index without losing # losing common short queries like "go", "ai".
# common short queries like "go", "ai".
_SEARCH_TOKEN_RE = re.compile(r"[A-Za-z0-9_\-]{2,}") _SEARCH_TOKEN_RE = re.compile(r"[A-Za-z0-9_\-]{2,}")
_SEARCH_MIN_QUERY_LEN = 2 _SEARCH_MIN_QUERY_LEN = 2
# ============================================================================
# Shared regexes (same as legacy get_backlinks / /api/graph). Kept here so the
# index and the legacy paths are guaranteed to extract the same tokens.
# ============================================================================
WIKILINK_RE = re.compile(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]') WIKILINK_RE = re.compile(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]')
MDLINK_RE = re.compile(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)') MDLINK_RE = re.compile(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)')
# ============================================================================
# Public extraction helpers — used by the index AND by lifecycle hooks in
# utils.py (so a single save can update the index without re-reading the file).
# ============================================================================
def extract_links_from_content(content: str) -> Dict[str, List[str]]: def extract_links_from_content(content: str) -> Dict[str, List[str]]:
"""Pull raw wikilink targets and markdown-link paths out of note content.""" """Pull raw wikilink targets and markdown-link paths out of note content."""
wikilinks = [m.strip() for m in WIKILINK_RE.findall(content)] wikilinks = [m.strip() for m in WIKILINK_RE.findall(content)]
@ -129,68 +44,50 @@ def extract_links_from_content(content: str) -> Dict[str, List[str]]:
def extract_search_terms(content: str) -> Set[str]: def extract_search_terms(content: str) -> Set[str]:
"""Tokenize content for the inverted search index. Lowercased, deduped.""" """Tokenize content for the inverted search index."""
return {m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(content)} return {m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(content)}
# ============================================================================
# Data record for one note's metadata snapshot
# ============================================================================
@dataclass @dataclass
class NoteRecord: class NoteRecord:
"""Snapshot of one note's metadata. Kept tiny — no content, no resolved """One note's metadata. No content, no resolved links."""
links (those live in the inverted indexes)."""
path: str # vault-relative POSIX path: str # vault-relative POSIX
name: str # stem (no extension) name: str # stem (no extension)
folder: str # vault-relative POSIX, "" for root folder: str # vault-relative POSIX, "" for root
modified: str # ISO timestamp modified: str # ISO timestamp
size: int # bytes size: int # bytes
type: str # "note" | "image" | "audio" | ... type: str # "note" | "image" | "audio" | ...
mtime: float # raw stat mtime, for staleness checks mtime: float # raw stat mtime
tags: Tuple[str, ...] = field(default_factory=tuple) # sorted, deduped tags: Tuple[str, ...] = field(default_factory=tuple)
# ============================================================================
# The index itself
# ============================================================================
class NoteIndex: class NoteIndex:
"""Thread-safe in-memory index of vault state. Every read returns a """Thread-safe in-memory index of vault state. Reads return copies so
snapshot copy so callers don't have to hold the lock.""" callers don't have to hold the lock."""
# ------------------------------------------------------------------
# Construction
# ------------------------------------------------------------------
def __init__(self) -> None: def __init__(self) -> None:
self._lock = threading.RLock() self._lock = threading.RLock()
# Note metadata
self._notes: Dict[str, NoteRecord] = {} # path -> record self._notes: Dict[str, NoteRecord] = {} # path -> record
self._folders: Set[str] = set() self._folders: Set[str] = set()
# Tag indexes
self._tags_forward: Dict[str, Tuple[str, ...]] = {} # path -> tags self._tags_forward: Dict[str, Tuple[str, ...]] = {} # path -> tags
self._tags_backward: Dict[str, Set[str]] = {} # tag -> paths self._tags_backward: Dict[str, Set[str]] = {} # tag -> paths
# Link indexes
self._raw_links: Dict[str, Dict[str, List[str]]] = {} # path -> {"wikilinks":[], "mdlinks":[]} self._raw_links: Dict[str, Dict[str, List[str]]] = {} # path -> {"wikilinks":[], "mdlinks":[]}
self._links_forward: Dict[str, Dict[str, str]] = {} # src -> {tgt: type} self._links_forward: Dict[str, Dict[str, str]] = {} # src -> {tgt: type}
self._links_backward: Dict[str, Set[str]] = {} # tgt -> {srcs} self._links_backward: Dict[str, Set[str]] = {} # tgt -> {srcs}
self._wikilink_tokens: Dict[str, Set[str]] = {} # token -> {srcs} (loose) self._wikilink_tokens: Dict[str, Set[str]] = {} # lower token -> {srcs} (loose match)
# Full-text inverted index
self._search_terms: Dict[str, Set[str]] = {} # term -> {paths} self._search_terms: Dict[str, Set[str]] = {} # term -> {paths}
# Build state. _built tracks the cheap part (notes/tags/links). # _search_built tracked separately — the search index is built lazily
# _search_built tracks the expensive search index separately — that # on the first /api/search call, after the cheaper notes/tags/links
# one is built lazily on first search request to keep startup fast. # part is already built.
self._built = False self._built = False
self._search_built = False self._search_built = False
self._raw_fingerprint: Optional[int] = None # short-circuits no-op rebuilds self._raw_fingerprint: Optional[int] = None # short-circuits no-op rebuilds
# Observability counters (lock-free reads OK; rough is fine)
self._stats = { self._stats = {
"build_count": 0, "build_count": 0,
"last_build_ms": 0.0, "last_build_ms": 0.0,
@ -201,17 +98,12 @@ class NoteIndex:
"last_search_build_ms": 0.0, "last_search_build_ms": 0.0,
} }
# ------------------------------------------------------------------
# Status / lifecycle gates
# ------------------------------------------------------------------
def is_built(self) -> bool: def is_built(self) -> bool:
with self._lock: with self._lock:
return self._built return self._built
def invalidate(self) -> None: def invalidate(self) -> None:
"""Mark as needing rebuild on next scan. Used as a safety net when """Mark as needing rebuild on next scan."""
an operation's incremental update is too fiddly to track precisely."""
with self._lock: with self._lock:
self._built = False self._built = False
self._raw_fingerprint = None self._raw_fingerprint = None
@ -236,25 +128,14 @@ class NoteIndex:
return self._search_built return self._search_built
def ensure_search_index_built(self, notes_dir: str) -> bool: def ensure_search_index_built(self, notes_dir: str) -> bool:
"""Build the full-text search index by reading every markdown file. """Build the search index on demand. Returns True when ready, False
only if the main index isn't built yet. File I/O happens OUTSIDE the
Cheap if already built (instant return). Called lazily by search_notes lock so other reads aren't blocked."""
on the first search request so app startup stays fast. Subsequent
searches reuse the built index.
Concurrency: while one thread is building, others can still read the
(still-cold) index — they'll just take the legacy path until the
build completes. Returns True if the index is built (now or already).
"""
# Cheap pre-check without lock.
if self._search_built: if self._search_built:
return True return True
# Slow path: do the read+tokenize OUTSIDE the lock so other reads
# aren't blocked. Snapshot the paths we need to read under the lock,
# release, do I/O, then re-acquire to install the result.
with self._lock: with self._lock:
if self._search_built: # raced with another builder if self._search_built:
return True return True
if not self._built: if not self._built:
return False return False
@ -265,16 +146,11 @@ class NoteIndex:
terms_per_path: Dict[str, Set[str]] = {} terms_per_path: Dict[str, Set[str]] = {}
for rel in paths_to_read: for rel in paths_to_read:
try: try:
full = base / rel with open(base / rel, "r", encoding="utf-8") as f:
with open(full, "r", encoding="utf-8") as f:
terms_per_path[rel] = extract_search_terms(f.read()) terms_per_path[rel] = extract_search_terms(f.read())
except Exception: except Exception:
terms_per_path[rel] = set() terms_per_path[rel] = set()
# Re-acquire and install. If state changed under us (vault was
# heavily mutated during the build), still install — the index
# will be slightly stale until the next note save, which is the
# same liveness guarantee as the per-save update path.
with self._lock: with self._lock:
if self._search_built: if self._search_built:
return True return True
@ -287,24 +163,15 @@ class NoteIndex:
self._stats["last_search_build_ms"] = (time.perf_counter() - t0) * 1000 self._stats["last_search_build_ms"] = (time.perf_counter() - t0) * 1000
return True return True
# ------------------------------------------------------------------
# Bulk population — called from a full scan_notes_fast_walk
# ------------------------------------------------------------------
def bulk_set( def bulk_set(
self, self,
notes_meta: List[NoteRecord], notes_meta: List[NoteRecord],
folders: Iterable[str], folders: Iterable[str],
sources_raw: Dict[str, Dict[str, List[str]]], sources_raw: Dict[str, Dict[str, List[str]]],
) -> None: ) -> None:
"""Replace the entire index atomically. Builds notes/tags/links """Replace the entire index from a fresh scan. Short-circuits when
(cheap on top of an already-completed scan). The search index is the input fingerprints to the same state we already hold (warm
NOT touched here — it's built lazily on the first search request scan, nothing changed)."""
(see ensure_search_index_built) so app startup stays fast.
Fast-paths when the input fingerprints to the same state we already
hold (typical warm scan where nothing changed in the vault).
"""
new_fp = _fingerprint(notes_meta, sources_raw) new_fp = _fingerprint(notes_meta, sources_raw)
with self._lock: with self._lock:
if self._built and self._raw_fingerprint == new_fp: if self._built and self._raw_fingerprint == new_fp:
@ -319,8 +186,6 @@ class NoteIndex:
self._rebuild_tags_unlocked() self._rebuild_tags_unlocked()
self._rebuild_links_unlocked() self._rebuild_links_unlocked()
# Search index: if it was built and still references known notes,
# prune stale entries; full rebuild is deferred until a search.
if self._search_built: if self._search_built:
self._prune_search_unlocked() self._prune_search_unlocked()
@ -331,19 +196,13 @@ class NoteIndex:
self._stats["last_build_ms"] = (time.perf_counter() - t0) * 1000 self._stats["last_build_ms"] = (time.perf_counter() - t0) * 1000
self._stats["last_built_at"] = datetime.now(tz=timezone.utc).isoformat() self._stats["last_built_at"] = datetime.now(tz=timezone.utc).isoformat()
# ------------------------------------------------------------------
# Incremental updates — called from save/delete/rename handlers
# ------------------------------------------------------------------
def update_note( def update_note(
self, self,
record: NoteRecord, record: NoteRecord,
raw_links: Dict[str, List[str]], raw_links: Dict[str, List[str]],
content: Optional[str] = None, content: Optional[str] = None,
) -> None: ) -> None:
"""A single note's content changed (or was just created). Patches the """Patch the index in place after a note is created or saved."""
index in place: re-extracts its links, updates the tag indexes, and
if `content` is provided, refreshes its search-term entries."""
with self._lock: with self._lock:
old_record = self._notes.get(record.path) old_record = self._notes.get(record.path)
old_tags = old_record.tags if old_record else () old_tags = old_record.tags if old_record else ()
@ -352,7 +211,6 @@ class NoteIndex:
if record.folder: if record.folder:
self._folders.add(record.folder) self._folders.add(record.folder)
# Tags: diff old vs new and patch the backward index.
new_tags = record.tags new_tags = record.tags
if old_tags != new_tags: if old_tags != new_tags:
for t in set(old_tags) - set(new_tags): for t in set(old_tags) - set(new_tags):
@ -365,13 +223,11 @@ class NoteIndex:
self._tags_backward.setdefault(t, set()).add(record.path) self._tags_backward.setdefault(t, set()).add(record.path)
self._tags_forward[record.path] = new_tags self._tags_forward[record.path] = new_tags
# Links: re-resolve this one source against the current vault.
self._raw_links[record.path] = raw_links self._raw_links[record.path] = raw_links
self._resolve_single_source_unlocked(record.path) self._resolve_single_source_unlocked(record.path)
# Search: only patch the search index if it's already been built. # Search index only gets patched if it's already been built —
# Otherwise we'd be doing work for an index nobody is using yet # otherwise the first /api/search will build it from scratch.
# (the first /api/search call will build it from disk).
if content is not None and self._search_built: if content is not None and self._search_built:
self._update_search_for_note_unlocked(record.path, content) self._update_search_for_note_unlocked(record.path, content)
@ -437,11 +293,9 @@ class NoteIndex:
self._stats["incremental_updates"] += 1 self._stats["incremental_updates"] += 1
def rename_note(self, old_path: str, new_path: str) -> None: def rename_note(self, old_path: str, new_path: str) -> None:
"""A note was renamed/moved. Move all references from old_path to """Move all references from old_path to new_path. The path change can
new_path in every index. Because the new path can change the source's affect how other notes' wikilinks resolve, so we wipe the resolved
own relative-folder resolution AND can change how other notes resolve link indexes for this note and rely on the next bulk_set to rebuild."""
their wikilinks to it (different parent folder, different stem),
the simplest correct path is to migrate raw state then re-resolve."""
if old_path == new_path: if old_path == new_path:
return return
with self._lock: with self._lock:
@ -465,7 +319,6 @@ class NoteIndex:
if new_record.folder: if new_record.folder:
self._folders.add(new_record.folder) self._folders.add(new_record.folder)
# Tags forward + backward — swap the key.
if old_path in self._tags_forward: if old_path in self._tags_forward:
tags = self._tags_forward.pop(old_path) tags = self._tags_forward.pop(old_path)
self._tags_forward[new_path] = tags self._tags_forward[new_path] = tags
@ -475,24 +328,17 @@ class NoteIndex:
bucket.discard(old_path) bucket.discard(old_path)
bucket.add(new_path) bucket.add(new_path)
# Search: swap path in every term bucket. Bounded by terms-per-note.
for paths in self._search_terms.values(): for paths in self._search_terms.values():
if old_path in paths: if old_path in paths:
paths.discard(old_path) paths.discard(old_path)
paths.add(new_path) paths.add(new_path)
# Raw links: migrate.
if old_path in self._raw_links: if old_path in self._raw_links:
self._raw_links[new_path] = self._raw_links.pop(old_path) self._raw_links[new_path] = self._raw_links.pop(old_path)
# Strict link indexes: drop both old src and old target everywhere, # Other sources that linked to old_path by stem name may now
# then re-resolve. Other sources that linked TO old_path by name # resolve to new_path (or not), so we wipe both forward & backward
# may now resolve to new_path (or not), so we need a re-resolve. # for old_path and let the next bulk_set re-resolve from scratch.
#
# Implementation: drop forward/backward for old_path, drop old
# entries in other sources' forwards that pointed to old_path,
# invalidate the index. The next scan rebuilds (which we trigger
# implicitly because the caller flips _built = False below).
self._links_forward.pop(old_path, None) self._links_forward.pop(old_path, None)
for bucket in self._links_backward.values(): for bucket in self._links_backward.values():
bucket.discard(old_path) bucket.discard(old_path)
@ -504,7 +350,6 @@ class NoteIndex:
if not fwd: if not fwd:
del self._links_forward[src] del self._links_forward[src]
# Loose wikilink index: replace path values.
empty_keys = [] empty_keys = []
for key, sources in self._wikilink_tokens.items(): for key, sources in self._wikilink_tokens.items():
if old_path in sources: if old_path in sources:
@ -516,17 +361,12 @@ class NoteIndex:
del self._wikilink_tokens[k] del self._wikilink_tokens[k]
self._raw_fingerprint = None self._raw_fingerprint = None
self._built = False # force re-resolve on next bulk_set/scan self._built = False # force re-resolve on next bulk_set
self._stats["incremental_updates"] += 1 self._stats["incremental_updates"] += 1
def rename_folder_prefix(self, old_prefix: str, new_prefix: str) -> None: def rename_folder_prefix(self, old_prefix: str, new_prefix: str) -> None:
"""A folder was renamed/moved. Migrate every entry whose path starts """Migrate every entry under `old_prefix/` to `new_prefix/`. Much
with `old_prefix/` to `new_prefix/`. Much cheaper than a full rebuild: cheaper than a full rebuild — microseconds of key swaps."""
for a 1000-note folder rename, this is microseconds of key swaps
vs ~400ms of disk re-scan.
Both prefixes are normalized to forward slashes, no trailing slash.
"""
old_prefix = old_prefix.rstrip("/") old_prefix = old_prefix.rstrip("/")
new_prefix = new_prefix.rstrip("/") new_prefix = new_prefix.rstrip("/")
if old_prefix == new_prefix: if old_prefix == new_prefix:
@ -534,14 +374,9 @@ class NoteIndex:
with self._lock: with self._lock:
affected_paths = [p for p in self._notes if p == old_prefix or p.startswith(old_prefix + "/")] affected_paths = [p for p in self._notes if p == old_prefix or p.startswith(old_prefix + "/")]
for old_path in affected_paths: for old_path in affected_paths:
suffix = old_path[len(old_prefix):] # includes leading "/" or nothing suffix = old_path[len(old_prefix):]
new_path = new_prefix + suffix self._rename_note_unlocked(old_path, new_prefix + suffix)
# Reuse rename_note's heavy lifting — already correct, just
# called many times. Acceptable for folder operations.
# (Inlining would be a perf gain but multiplies bug surface.)
self._rename_note_unlocked(old_path, new_path)
# Migrate the folder set too.
folders_to_rename = [ folders_to_rename = [
f for f in self._folders if f == old_prefix or f.startswith(old_prefix + "/") f for f in self._folders if f == old_prefix or f.startswith(old_prefix + "/")
] ]
@ -554,7 +389,7 @@ class NoteIndex:
self._built = False self._built = False
def remove_folder_prefix(self, prefix: str) -> None: def remove_folder_prefix(self, prefix: str) -> None:
"""A folder was deleted. Drop everything under it.""" """Drop every entry under `prefix/`."""
prefix = prefix.rstrip("/") prefix = prefix.rstrip("/")
with self._lock: with self._lock:
affected = [p for p in self._notes if p == prefix or p.startswith(prefix + "/")] affected = [p for p in self._notes if p == prefix or p.startswith(prefix + "/")]
@ -597,7 +432,7 @@ class NoteIndex:
return nodes, edges return nodes, edges
def get_all_tags(self) -> Dict[str, int]: def get_all_tags(self) -> Dict[str, int]:
"""Snapshot: {tag: count}, sorted by tag name.""" """Snapshot {tag: count}, sorted by tag name."""
with self._lock: with self._lock:
return {tag: len(paths) for tag, paths in sorted(self._tags_backward.items())} return {tag: len(paths) for tag, paths in sorted(self._tags_backward.items())}
@ -610,23 +445,19 @@ class NoteIndex:
with self._lock: with self._lock:
return self._notes.get(path) return self._notes.get(path)
def all_note_records(self) -> List[Tuple[str, NoteRecord]]:
"""Snapshot list of (path, record) for every indexed markdown note."""
with self._lock:
return [(p, r) for p, r in self._notes.items() if r.type == "note"]
def try_get_extraction( def try_get_extraction(
self, self,
rel_path: str, rel_path: str,
mtime: float, mtime: float,
) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: ) -> Optional[Tuple[List[str], Dict[str, List[str]]]]:
"""Return (tags, raw_links) from the index when fresh. """Return (tags, raw_links) from the index only if the recorded mtime
matches `mtime` exactly. Lets scan_notes_fast_walk skip the per-file
The caller passes the file's current mtime; we hand back the cached read on a warm scan."""
extraction only when the recorded mtime matches exactly. Lets
scan_notes_fast_walk skip the per-file read on a snapshot-warm
startup — the bulk of cold-load latency on large vaults.
Returns None when:
- The note isn't in the index (new file)
- mtime differs (file changed since snapshot)
- We've somehow lost the raw_links entry (defensive)
"""
with self._lock: with self._lock:
rec = self._notes.get(rel_path) rec = self._notes.get(rel_path)
if rec is None or rec.mtime != mtime: if rec is None or rec.mtime != mtime:
@ -643,34 +474,18 @@ class NoteIndex:
) )
def get_search_candidates(self, query: str) -> Optional[Set[str]]: def get_search_candidates(self, query: str) -> Optional[Set[str]]:
# Index can only narrow candidates if it's been built. """Superset of paths whose content COULD contain `query`. Caller
still runs the substring match per candidate for confirmation +
snippet extraction. Returns None when the query is too short or
tokenizes to nothing — caller should iterate every note instead."""
if not self._search_built: if not self._search_built:
return None return None
"""Return the set of paths that COULD contain `query` as a substring,
based on the inverted term index.
Returns None when the query is too short to use the index (caller
should fall through to the legacy full-scan). When the query
tokenizes to nothing useful (all stopword-like noise), returns the
set of every indexed path so the caller still does a substring
check on each (no false negatives).
IMPORTANT: this is a SUPERSET — the caller must still run the
substring match per candidate to confirm and extract context.
Token-AND can include docs that have both tokens but not as the
adjacent substring the user searched for.
"""
if len(query) < _SEARCH_MIN_QUERY_LEN: if len(query) < _SEARCH_MIN_QUERY_LEN:
return None return None
tokens = [m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(query)] tokens = [m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(query)]
if not tokens: if not tokens:
# Query has no tokenizable parts (pure punctuation, single char,
# etc.). Index can't help; fall through.
return None return None
with self._lock: with self._lock:
# Intersection of all token buckets. Empty set means no doc
# contains ALL of the tokens (and therefore none contain the
# query as substring either).
candidate = self._search_terms.get(tokens[0]) candidate = self._search_terms.get(tokens[0])
if candidate is None: if candidate is None:
return set() return set()
@ -685,11 +500,9 @@ class NoteIndex:
return result return result
def stats(self) -> Dict[str, Any]: def stats(self) -> Dict[str, Any]:
"""Snapshot of internal counters + size metrics. Cheap to call — """Snapshot of counters + size metrics."""
no traversal beyond `len()` on the top-level dicts."""
with self._lock: with self._lock:
return { return {
"enabled": USE_NOTE_INDEX,
"built": self._built, "built": self._built,
"search_built": self._search_built, "search_built": self._search_built,
"notes": len(self._notes), "notes": len(self._notes),
@ -720,11 +533,8 @@ class NoteIndex:
self._tags_backward.setdefault(tag, set()).add(path) self._tags_backward.setdefault(tag, set()).add(path)
def _rebuild_links_unlocked(self) -> None: def _rebuild_links_unlocked(self) -> None:
"""Full link re-resolution. Builds a shared Resolver lookup once """Full link re-resolution. Reuses one _Resolver across all sources
(O(N)) and reuses it for every source (O(K) each) — total O(N+K*L) (O(N+K*L) instead of O(N*K))."""
rather than the O(N*K) you'd get by building a fresh Resolver per
source.
"""
self._links_forward.clear() self._links_forward.clear()
self._links_backward.clear() self._links_backward.clear()
self._wikilink_tokens.clear() self._wikilink_tokens.clear()
@ -734,9 +544,7 @@ class NoteIndex:
self._resolve_single_source_unlocked(source_path, resolver=resolver, skip_cleanup=True) self._resolve_single_source_unlocked(source_path, resolver=resolver, skip_cleanup=True)
def _prune_search_unlocked(self) -> None: def _prune_search_unlocked(self) -> None:
"""Drop search-term entries for paths no longer in the index. Cheap """Drop search-term entries for paths no longer in the index."""
compared to a full rebuild — only touches terms whose bucket still
references a now-deleted path."""
live_paths = set(self._notes.keys()) live_paths = set(self._notes.keys())
empty_terms = [] empty_terms = []
for term, paths in self._search_terms.items(): for term, paths in self._search_terms.items():
@ -750,7 +558,6 @@ class NoteIndex:
def _update_search_for_note_unlocked(self, path: str, content: str) -> None: def _update_search_for_note_unlocked(self, path: str, content: str) -> None:
"""Replace one note's terms in the inverted index.""" """Replace one note's terms in the inverted index."""
# Drop old entries for this path.
empty_terms = [] empty_terms = []
for term, paths in self._search_terms.items(): for term, paths in self._search_terms.items():
if path in paths: if path in paths:
@ -759,7 +566,6 @@ class NoteIndex:
empty_terms.append(term) empty_terms.append(term)
for term in empty_terms: for term in empty_terms:
del self._search_terms[term] del self._search_terms[term]
# Add new entries.
for term in extract_search_terms(content): for term in extract_search_terms(content):
self._search_terms.setdefault(term, set()).add(path) self._search_terms.setdefault(term, set()).add(path)
@ -799,16 +605,13 @@ class NoteIndex:
targets: Dict[str, str] = {} targets: Dict[str, str] = {}
# Wikilinks first so they win the "first wins" dedup that legacy # Wikilinks first (they win the "first wins" dedup that /api/graph uses).
# /api/graph also implements.
for target in raw.get("wikilinks", []): for target in raw.get("wikilinks", []):
resolved = resolver.resolve_wikilink(target, source_folder) resolved = resolver.resolve_wikilink(target, source_folder)
if resolved and resolved != source_path and resolved not in targets: if resolved and resolved != source_path and resolved not in targets:
targets[resolved] = "wikilink" targets[resolved] = "wikilink"
# Loose wikilink reverse index — populated regardless of strict # Loose token index — populated even when strict resolution fails,
# resolution success, because the legacy get_backlinks uses # because backlink matching uses stem comparison independently.
# token-stem matching independent of where the link actually
# navigates.
t_lower = target.strip().lower() t_lower = target.strip().lower()
if t_lower: if t_lower:
self._wikilink_tokens.setdefault(t_lower, set()).add(source_path) self._wikilink_tokens.setdefault(t_lower, set()).add(source_path)
@ -827,8 +630,8 @@ class NoteIndex:
self._links_backward.setdefault(t, set()).add(source_path) self._links_backward.setdefault(t, set()).add(source_path)
def _rename_note_unlocked(self, old_path: str, new_path: str) -> None: def _rename_note_unlocked(self, old_path: str, new_path: str) -> None:
"""Same as rename_note() but assumes the caller already holds the lock. """rename_note() body without acquiring the lock — used by
Used by folder-prefix rename to avoid re-acquiring the lock per file.""" rename_folder_prefix to avoid re-locking per file."""
if old_path == new_path: if old_path == new_path:
return return
old_record = self._notes.pop(old_path, None) old_record = self._notes.pop(old_path, None)
@ -936,9 +739,8 @@ class NoteIndex:
# ============================================================================ # ============================================================================
class _Resolver: class _Resolver:
"""Build the lookup tables once per resolution batch, then call """Link-target lookup tables. Build once per resolution batch, then call
resolve_* repeatedly. Reused across sources within a single rebuild so resolve_* repeatedly across many sources."""
we don't pay O(N) to construct it on every source."""
def __init__(self, all_notes: Set[str]) -> None: def __init__(self, all_notes: Set[str]) -> None:
self.note_paths: Set[str] = set(all_notes) self.note_paths: Set[str] = set(all_notes)
@ -1029,8 +831,7 @@ def _fingerprint(
notes_meta: List[NoteRecord], notes_meta: List[NoteRecord],
sources_raw: Dict[str, Dict[str, List[str]]], sources_raw: Dict[str, Dict[str, List[str]]],
) -> int: ) -> int:
"""Cheap content hash of a scan result. Used to short-circuit bulk_set """Cheap hash of a scan result. Short-circuits bulk_set when unchanged."""
when nothing has changed in the vault."""
notes_fp = hash(frozenset((n.path, n.mtime) for n in notes_meta)) notes_fp = hash(frozenset((n.path, n.mtime) for n in notes_meta))
raw_items = ( raw_items = (
(src, tuple(raw.get("wikilinks", [])), tuple(raw.get("mdlinks", []))) (src, tuple(raw.get("wikilinks", [])), tuple(raw.get("mdlinks", [])))
@ -1040,11 +841,7 @@ def _fingerprint(
# ============================================================================ # ============================================================================
# Module-level singleton + facade # Module singleton + facade. Callers in utils.py / main.py go through these.
#
# Every utils.py / main.py call site uses these — never instantiates its own
# index. The facade functions are no-ops (or return None) when
# USE_NOTE_INDEX is False, so call sites don't have to repeat the flag check.
# ============================================================================ # ============================================================================
_index = NoteIndex() _index = NoteIndex()
@ -1054,12 +851,10 @@ def get_index() -> NoteIndex:
return _index return _index
# --- Lifecycle facade (one-line calls from utils.py mutators) ---------------- # --- Lifecycle hooks (one-line calls from utils.py mutators) ----------------
def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None: def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None:
"""A note was created or updated on disk. Patch the index in place.""" """A note was created or updated on disk."""
if not USE_NOTE_INDEX:
return
try: try:
rel_path = full_path.relative_to(Path(notes_dir)).as_posix() rel_path = full_path.relative_to(Path(notes_dir)).as_posix()
st = full_path.stat() st = full_path.stat()
@ -1074,15 +869,12 @@ def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None:
mtime=st.st_mtime, mtime=st.st_mtime,
tags=tuple(_parse_tags_for_record(content)), tags=tuple(_parse_tags_for_record(content)),
) )
raw_links = extract_links_from_content(content) _index.update_note(record, extract_links_from_content(content), content=content)
_index.update_note(record, raw_links, content=content)
except Exception as e: except Exception as e:
print(f"note_index: on_note_saved failed for {full_path}: {e}") print(f"note_index: on_note_saved failed for {full_path}: {e}")
def on_note_deleted(notes_dir: str, full_path: Path) -> None: def on_note_deleted(notes_dir: str, full_path: Path) -> None:
if not USE_NOTE_INDEX:
return
try: try:
rel_path = full_path.relative_to(Path(notes_dir)).as_posix() rel_path = full_path.relative_to(Path(notes_dir)).as_posix()
_index.remove_note(rel_path) _index.remove_note(rel_path)
@ -1091,8 +883,6 @@ def on_note_deleted(notes_dir: str, full_path: Path) -> None:
def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None:
if not USE_NOTE_INDEX:
return
try: try:
base = Path(notes_dir) base = Path(notes_dir)
_index.rename_note( _index.rename_note(
@ -1104,10 +894,7 @@ def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) ->
def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None:
"""A folder was moved/renamed. Re-keys every entry under it (cheap, """Re-key every entry under the folder. No disk reads."""
no disk reads) rather than invalidating the whole index."""
if not USE_NOTE_INDEX:
return
try: try:
base = Path(notes_dir) base = Path(notes_dir)
_index.rename_folder_prefix( _index.rename_folder_prefix(
@ -1116,12 +903,10 @@ def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path)
) )
except Exception as e: except Exception as e:
print(f"note_index: on_folder_renamed failed: {e}") print(f"note_index: on_folder_renamed failed: {e}")
_index.invalidate() # fail-safe _index.invalidate()
def on_folder_deleted(notes_dir: str, full_path: Path) -> None: def on_folder_deleted(notes_dir: str, full_path: Path) -> None:
if not USE_NOTE_INDEX:
return
try: try:
rel_prefix = full_path.relative_to(Path(notes_dir)).as_posix() rel_prefix = full_path.relative_to(Path(notes_dir)).as_posix()
_index.remove_folder_prefix(rel_prefix) _index.remove_folder_prefix(rel_prefix)
@ -1135,9 +920,7 @@ def populate_from_scan(
folders: Iterable[str], folders: Iterable[str],
sources_raw: Dict[str, Dict[str, List[str]]], sources_raw: Dict[str, Dict[str, List[str]]],
) -> None: ) -> None:
"""Bulk-replace the index from a fresh scan. No-op when off.""" """Bulk-replace the index after a full scan_notes_fast_walk."""
if not USE_NOTE_INDEX:
return
try: try:
_index.bulk_set(notes_meta, folders, sources_raw) _index.bulk_set(notes_meta, folders, sources_raw)
except Exception as e: except Exception as e:
@ -1145,44 +928,34 @@ def populate_from_scan(
def ensure_search_index(notes_dir: str) -> bool: def ensure_search_index(notes_dir: str) -> bool:
"""Lazily build the full-text search index on first /api/search request. """Lazy-build the search index on first /api/search. Returns False only
Cheap after the first call. Returns True if the index is now usable.""" if the main index isn't built yet."""
if not USE_NOTE_INDEX:
return False
if not _index.is_built(): if not _index.is_built():
return False return False
return _index.ensure_search_index_built(notes_dir) return _index.ensure_search_index_built(notes_dir)
# --- Read facade (returns None when off — callers fall through to legacy) ---- # --- Read facade -------------------------------------------------------------
def try_backlink_candidates(target_path: str) -> Optional[Set[str]]: def get_backlink_candidates(target_path: str) -> Set[str]:
if not USE_NOTE_INDEX or not _index.is_built():
return None
return _index.get_backlink_candidate_sources(target_path) return _index.get_backlink_candidate_sources(target_path)
def try_graph_data() -> Optional[Tuple[List[str], List[Tuple[str, str, str]]]]: def get_graph_data() -> Tuple[List[str], List[Tuple[str, str, str]]]:
if not USE_NOTE_INDEX or not _index.is_built():
return None
return _index.get_graph_data() return _index.get_graph_data()
def try_search_candidates(query: str) -> Optional[Set[str]]: def get_search_candidates(query: str) -> Optional[Set[str]]:
if not USE_NOTE_INDEX or not _index.is_built(): """Returns None when query is too short / untokenizable — caller iterates
return None every indexed note instead."""
return _index.get_search_candidates(query) return _index.get_search_candidates(query)
def try_all_tags() -> Optional[Dict[str, int]]: def get_all_tags() -> Dict[str, int]:
if not USE_NOTE_INDEX or not _index.is_built():
return None
return _index.get_all_tags() return _index.get_all_tags()
def try_notes_by_tag(tag: str) -> Optional[Set[str]]: def get_paths_for_tag(tag: str) -> Set[str]:
if not USE_NOTE_INDEX or not _index.is_built():
return None
return _index.get_paths_for_tag(tag) return _index.get_paths_for_tag(tag)
@ -1190,30 +963,16 @@ def try_get_extraction(
rel_path: str, rel_path: str,
mtime: float, mtime: float,
) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: ) -> Optional[Tuple[List[str], Dict[str, List[str]]]]:
"""Serve (tags, raw_links) from the index for a single file, when fresh. """(tags, raw_links) from the index when mtime matches, else None so the
caller reads the file. Used by scan_notes_fast_walk."""
Used by scan_notes_fast_walk to skip the cold per-file read after a
snapshot load. Returns None when the index can't help (off, not built,
file unknown, mtime mismatch) — caller falls back to reading the file.
"""
if not USE_NOTE_INDEX or not _index.is_built():
return None
return _index.try_get_extraction(rel_path, mtime) return _index.try_get_extraction(rel_path, mtime)
# --- Observability -----------------------------------------------------------
def stats() -> Dict[str, Any]: def stats() -> Dict[str, Any]:
return _index.stats() return _index.stats()
# ============================================================================ # Late import — utils imports this module, so we delay the reverse direction.
# Late import to avoid a circular dependency with utils.parse_tags.
# We need tag parsing inside on_note_saved but utils.py imports this module.
# ============================================================================
def _parse_tags_for_record(content: str) -> List[str]: def _parse_tags_for_record(content: str) -> List[str]:
"""Thin shim to utils.parse_tags. Late-bound so this module imports cleanly
before utils.py is loaded."""
from .utils import parse_tags from .utils import parse_tags
return parse_tags(content) return parse_tags(content)

162
backend/utils.py
View File

@ -2,8 +2,7 @@
Utility functions for file operations, search, and markdown processing. Utility functions for file operations, search, and markdown processing.
Heavy work (backlinks, graph, full-text search, tag aggregation) is delegated Heavy work (backlinks, graph, full-text search, tag aggregation) is delegated
to the in-memory index in note_index.py — every facade function there is a to the in-memory index in note_index.py.
no-op when USE_NOTE_INDEX is False, so call sites here stay flag-free.
""" """
import os import os
@ -163,27 +162,25 @@ def _scan_cache_set(key: Tuple[str, bool], value: Tuple[List[Dict], List[str]])
def _scan_cache_invalidate() -> None: def _scan_cache_invalidate() -> None:
"""Drop the TTL scan cache. Called from every mutation handler so callers """Drop the TTL scan cache. Called from every mutation handler."""
that hit the cache immediately after a save/delete/move see the new state
instead of stale data from the previous walk."""
with _SCAN_WALK_CACHE_LOCK: with _SCAN_WALK_CACHE_LOCK:
_SCAN_WALK_CACHE.clear() _SCAN_WALK_CACHE.clear()
def _ensure_index_built(notes_dir: str) -> None:
"""Trigger a fresh scan when the NoteIndex hasn't been populated yet.
Idempotent: the scan short-circuits when nothing changed in the vault."""
if not note_index.get_index().is_built():
scan_notes_fast_walk(notes_dir, use_cache=False, include_media=False)
def get_tags_and_links_cached( def get_tags_and_links_cached(
file_path: Path, file_path: Path,
rel_path: Optional[str] = None, rel_path: Optional[str] = None,
) -> Tuple[List[str], Dict[str, List[str]]]: ) -> Tuple[List[str], Dict[str, List[str]]]:
"""Single-read fused extraction of tags + raw links, cached by mtime. """Fused tags + raw-links extraction, mtime-cached. Tries in-process
per-file caches first, then the NoteIndex (when rel_path is provided),
Three-tier lookup, cheapest first: then reads the file."""
1. In-process per-file caches (warm scans on the live server)
2. NoteIndex (warm after snapshot load — avoids file reads on cold start)
3. Read the file from disk
rel_path enables tier 2. scan_notes_fast_walk passes it; ad-hoc callers
can omit it and just get tiers 1 + 3.
"""
try: try:
mtime = file_path.stat().st_mtime mtime = file_path.stat().st_mtime
file_key = str(file_path) file_key = str(file_path)
@ -196,8 +193,6 @@ def get_tags_and_links_cached(
if tags_ok and links_ok: if tags_ok and links_ok:
return tag_cached[1], link_cached[1] return tag_cached[1], link_cached[1]
# Try the NoteIndex before reading the file. On a snapshot-warm
# startup, this turns a 10K-file cold scan into a no-I/O walk.
if rel_path is not None: if rel_path is not None:
indexed = note_index.try_get_extraction(rel_path, mtime) indexed = note_index.try_get_extraction(rel_path, mtime)
if indexed is not None: if indexed is not None:
@ -293,11 +288,11 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
_scan_cache_set(cache_key, normalized_value) _scan_cache_set(cache_key, normalized_value)
return normalized_value return normalized_value
# Walk the vault once. Tag/link extraction is deferred out of the walk # Walk the vault once. Tag/link extraction is parallelized below across
# (the expensive part is the file I/O) and parallelized below across files. # the markdown files we collected here.
notes: List[Dict] = [] notes: List[Dict] = []
folders_set = set() folders_set = set()
md_to_extract: List[Tuple[int, Path, str]] = [] # (note_index, full_path, rel_path) md_to_extract: List[Tuple[int, Path, str]] = [] # (index, full_path, rel_path)
for root, dirnames, filenames in os.walk(notes_path): for root, dirnames, filenames in os.walk(notes_path):
dirnames[:] = [d for d in dirnames if not d.startswith('.')] dirnames[:] = [d for d in dirnames if not d.startswith('.')]
@ -339,19 +334,10 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
if is_markdown: if is_markdown:
md_to_extract.append((len(notes) - 1, full_path, rel_str)) md_to_extract.append((len(notes) - 1, full_path, rel_str))
# Fused tag + raw-link extraction. mtime-keyed cache makes warm scans # Tag + raw-link extraction in parallel. Search-index terms are built
# mostly free; cold scans win from parallel I/O. # later, lazily, on the first /api/search call.
#
# We don't read full content here — that would double the parsing cost
# for every scan, and the search index is the only thing that needs it.
# Search is built lazily on the first search request (see
# note_index.ensure_search_index_built), which keeps startup fast and
# only pays the cost when a user actually searches.
sources_raw: Dict[str, Dict[str, List[str]]] = {} sources_raw: Dict[str, Dict[str, List[str]]] = {}
if md_to_extract: if md_to_extract:
# Pass rel_path so each call can hit the NoteIndex before falling
# back to a file read — critical for fast startup after a snapshot
# load on large vaults.
path_pairs = [(full, rel) for (_, full, rel) in md_to_extract] path_pairs = [(full, rel) for (_, full, rel) in md_to_extract]
if len(md_to_extract) >= 50: if len(md_to_extract) >= 50:
workers = min(8, (os.cpu_count() or 4)) workers = min(8, (os.cpu_count() or 4))
@ -367,8 +353,7 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
notes[idx]["tags"] = tags notes[idx]["tags"] = tags
sources_raw[rel_str] = links sources_raw[rel_str] = links
# Populate the unified index. Cheap when the fingerprint matches (warm # Push the result into the NoteIndex. Short-circuits on warm scans.
# scan, nothing changed) — short-circuits internally. No-op when off.
notes_meta = [ notes_meta = [
NoteRecord( NoteRecord(
path=n["path"], path=n["path"],
@ -555,27 +540,28 @@ def delete_note(notes_dir: str, note_path: str) -> bool:
def search_notes(notes_dir: str, query: str) -> List[Dict]: def search_notes(notes_dir: str, query: str) -> List[Dict]:
"""Full-text search through note contents. """Full-text search through note contents. Narrow the candidate set via
the inverted index, then run the snippet extractor on each candidate."""
Only searches inside note content (not filenames or paths). The index, when
available, narrows down the candidate files via a token-AND match on its
inverted index — then the existing per-line snippet extraction runs on the
narrow set. Output is byte-identical to a full scan.
"""
from html import escape from html import escape
results: List[Dict] = [] results: List[Dict] = []
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
# Build the search index lazily on the first search request. Subsequent _ensure_index_built(notes_dir)
# searches hit the warm index; the first one pays the cost (read every
# file once) and is roughly as slow as the legacy full scan would be.
note_index.ensure_search_index(notes_dir) note_index.ensure_search_index(notes_dir)
candidates = note_index.try_search_candidates(query) candidates = note_index.get_search_candidates(query)
for note in notes: idx = note_index.get_index()
if candidates is None:
# Query too short to tokenize — iterate every indexed note instead.
candidate_records = idx.all_note_records()
else:
candidate_records = [(p, idx.get_note_record(p)) for p in candidates]
candidate_records = [(p, r) for (p, r) in candidate_records if r is not None and r.type == "note"]
candidate_records.sort(key=lambda pr: pr[1].mtime, reverse=True)
iterable = [{"path": p, "name": r.name} for (p, r) in candidate_records]
for note in iterable:
path = note["path"] path = note["path"]
if candidates is not None and path not in candidates:
continue
md_file = Path(notes_dir) / path md_file = Path(notes_dir) / path
try: try:
with open(md_file, 'r', encoding='utf-8') as f: with open(md_file, 'r', encoding='utf-8') as f:
@ -888,32 +874,16 @@ def parse_tags(content: str) -> List[str]:
def get_all_tags(notes_dir: str) -> Dict[str, int]: def get_all_tags(notes_dir: str) -> Dict[str, int]:
"""All tags in the vault with note counts. Index-backed when available.""" """All tags in the vault with note counts."""
# Fast path FIRST — when the index is ready it knows every tag count, _ensure_index_built(notes_dir)
# so we skip the scan entirely (no disk walk, no JSON of 10K notes). return note_index.get_all_tags()
indexed = note_index.try_all_tags()
if indexed is not None:
return indexed
# Legacy fallback: scan the vault and aggregate.
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
tag_counts: Dict[str, int] = {}
for note in notes:
for tag in note.get("tags", []):
tag_counts[tag] = tag_counts.get(tag, 0) + 1
return dict(sorted(tag_counts.items()))
def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]: def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]:
"""All notes carrying `tag` (case-insensitive). Index-backed when available.""" """All notes carrying `tag` (case-insensitive)."""
tag_lower = tag.lower() _ensure_index_built(notes_dir)
# Fast path FIRST: every field we need lives in the NoteRecord. Skip
# the vault scan when the index can answer directly.
indexed_paths = note_index.try_notes_by_tag(tag_lower)
if indexed_paths is not None:
idx = note_index.get_index() idx = note_index.get_index()
records = [idx.get_note_record(p) for p in indexed_paths] records = [idx.get_note_record(p) for p in note_index.get_paths_for_tag(tag.lower())]
matching = [ matching = [
{ {
"name": r.name, "name": r.name,
@ -926,28 +896,9 @@ def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]:
for r in records for r in records
if r is not None and r.type == "note" if r is not None and r.type == "note"
] ]
# Match legacy sort order (scan returns notes sorted by modified desc).
matching.sort(key=lambda n: n["modified"], reverse=True) matching.sort(key=lambda n: n["modified"], reverse=True)
return matching return matching
# Legacy fallback.
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
matching: List[Dict] = []
for note in notes:
if note.get("type") != "note":
continue
tags = note.get("tags", [])
if tag_lower in tags:
matching.append({
"name": note["name"],
"path": note["path"],
"folder": note["folder"],
"modified": note["modified"],
"size": note["size"],
"tags": tags,
})
return matching
# ============================================================================ # ============================================================================
# Template Functions # Template Functions
@ -1189,13 +1140,8 @@ def _extract_backlink_references(
def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]: def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
"""Find all notes that link TO the specified note. """All notes that link TO `target_note_path`. The index narrows the
candidate set; we only read the candidate files for line context."""
Index-backed when available: the index hands us a superset of candidate
source files (O(1)), we read only those, then the per-line matcher
filters down to true matches with line-level context. Output is
byte-identical to a full vault scan.
"""
target_path = target_note_path target_path = target_note_path
target_path_lower = target_path.lower() target_path_lower = target_path.lower()
target_path_no_ext_lower = target_path_lower.replace('.md', '') target_path_no_ext_lower = target_path_lower.replace('.md', '')
@ -1205,22 +1151,15 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
Path(target_path).stem.lower(), Path(target_path).stem.lower(),
} }
# scan_notes_fast_walk is TTL-cached, so calling it here is a dict-lookup _ensure_index_built(notes_dir)
# in the warm case. We need it for the canonical ordering of `notes`. idx = note_index.get_index()
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) candidates = note_index.get_backlink_candidates(target_path)
records = [(p, idx.get_note_record(p)) for p in candidates]
candidates = note_index.try_backlink_candidates(target_path) records = [(p, r) for (p, r) in records if r is not None and r.type == "note" and p != target_path]
records.sort(key=lambda pr: pr[1].mtime, reverse=True)
backlinks: List[Dict] = [] backlinks: List[Dict] = []
for note in notes: for source_path, record in records:
if note.get('type') != 'note':
continue
source_path = note['path']
if source_path == target_path:
continue
if candidates is not None and source_path not in candidates:
continue
refs = _extract_backlink_references( refs = _extract_backlink_references(
notes_dir, notes_dir,
source_path, source_path,
@ -1232,9 +1171,8 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
if refs: if refs:
backlinks.append({ backlinks.append({
"path": source_path, "path": source_path,
"name": note['name'].replace('.md', ''), "name": record.name.replace('.md', ''),
"references": refs[:3], "references": refs[:3],
}) })
return backlinks return backlinks

2
documentation/API.md
View File

@ -591,7 +591,6 @@ The index is rebuilt on the first scan after each process start and updated incr
**Response:** **Response:**
```json ```json
{ {
"enabled": true,
"built": true, "built": true,
"search_built": false, "search_built": false,
"notes": 142, "notes": 142,
@ -615,7 +614,6 @@ The index is rebuilt on the first scan after each process start and updated incr
| Field | Description | | Field | Description |
|-------|-------------| |-------|-------------|
| `enabled` | Whether the index is active (always `true` in current builds) |
| `built` | `true` after the first vault scan completes | | `built` | `true` after the first vault scan completes |
| `search_built` | `true` after the first `/api/search` request (search index is built lazily) | | `search_built` | `true` after the first `/api/search` request (search index is built lazily) |
| `notes` | Number of note records currently held in the index | | `notes` | Number of note records currently held in the index |