global in memory index for large vaults
This commit is contained in:
Gamosoft
parent
030565299a
commit
4b316b2cc7
|
|
@ -45,6 +45,8 @@ from .utils import (
|
|||
paginate,
|
||||
get_backlinks,
|
||||
)
|
||||
from . import note_index
|
||||
from .note_index import USE_NOTE_INDEX
|
||||
from .plugins import PluginManager
|
||||
from .themes import get_available_themes, get_theme_css
|
||||
from .share import (
|
||||
|
|
@ -265,6 +267,7 @@ plugin_manager = PluginManager(config['storage']['plugins_dir'])
|
|||
# Run app startup hooks
|
||||
plugin_manager.run_hook('on_app_startup')
|
||||
|
||||
|
||||
# Mount static files
|
||||
static_path = Path(__file__).parent.parent / "frontend"
|
||||
app.mount("/static", StaticFiles(directory=static_path), name="static")
|
||||
|
|
@ -1483,10 +1486,25 @@ async def search(
|
|||
|
||||
@api_router.get("/graph", tags=["Graph"])
|
||||
async def get_graph():
|
||||
"""Get graph data for note visualization with wikilink and markdown link detection"""
|
||||
"""Get graph data for note visualization with wikilink and markdown link detection."""
|
||||
try:
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
# Fast path: index already holds resolved edges. Ensure a scan has
|
||||
# populated it (first request after startup, or after an invalidate)
|
||||
# then ask the facade for a snapshot.
|
||||
if USE_NOTE_INDEX:
|
||||
if not note_index.get_index().is_built():
|
||||
scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
|
||||
indexed = note_index.try_graph_data()
|
||||
if indexed is not None:
|
||||
nodes_paths, edges_tuples = indexed
|
||||
return {
|
||||
"nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths],
|
||||
"edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples],
|
||||
}
|
||||
|
||||
notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
|
||||
nodes = []
|
||||
edges = []
|
||||
|
|
@ -1692,6 +1710,19 @@ async def toggle_plugin(request: Request, plugin_name: str, enabled: dict):
|
|||
raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to toggle plugin"))
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Index observability — internal counters & sizes for debugging
|
||||
#
|
||||
# Useful for confirming the index is doing what we expect (build_count >0,
|
||||
# search_terms not empty, fingerprint_short_circuits incrementing on idle
|
||||
# warm scans). Not rate-limited — cheap, no I/O, dict lookups only.
|
||||
# ============================================================================
|
||||
|
||||
@api_router.get("/index/stats", tags=["System"])
|
||||
async def get_index_stats():
|
||||
return note_index.stats()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Stats Endpoint (for dashboards)
|
||||
# ============================================================================
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
638
backend/utils.py
638
backend/utils.py
|
|
@ -1,5 +1,9 @@
|
|||
"""
|
||||
Utility functions for file operations, search, and markdown processing
|
||||
Utility functions for file operations, search, and markdown processing.
|
||||
|
||||
Heavy work (backlinks, graph, full-text search, tag aggregation) is delegated
|
||||
to the in-memory index in note_index.py — every facade function there is a
|
||||
no-op when USE_NOTE_INDEX is False, so call sites here stay flag-free.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
|
@ -7,6 +11,7 @@ import re
|
|||
import shutil
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -14,6 +19,9 @@ from pathlib import Path
|
|||
from typing import List, Dict, Optional, Tuple, Any, TypeVar, Callable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from . import note_index
|
||||
from .note_index import NoteRecord, extract_links_from_content
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Pagination Support
|
||||
|
|
@ -112,14 +120,23 @@ def paginate(
|
|||
)
|
||||
|
||||
|
||||
# In-memory cache for parsed tags
|
||||
# Format: {file_path: (mtime, tags)}
|
||||
_tag_cache: Dict[str, Tuple[float, List[str]]] = {}
|
||||
|
||||
# Notes tree scan cache (TTL).
|
||||
# ============================================================================
|
||||
# In-memory caches
|
||||
#
|
||||
# This avoids repeated full-directory walks when multiple endpoints (or the UI)
|
||||
# request indexes in quick succession.
|
||||
# Two layers:
|
||||
# _tag_cache / _links_cache : per-file, mtime-keyed. Avoid re-parsing the
|
||||
# same file on warm scans. Used by get_tags_and_links_cached.
|
||||
# _SCAN_WALK_CACHE : per-scan TTL cache. Avoids repeated full-
|
||||
# directory walks when several endpoints fire in quick succession.
|
||||
#
|
||||
# All three are invalidated on every mutation (save/delete/move). The
|
||||
# note_index in note_index.py is the system of record for derived data
|
||||
# (links, backlinks, tags-by-name, search) — these caches just exist to
|
||||
# avoid double-reading files we already parsed.
|
||||
# ============================================================================
|
||||
|
||||
_tag_cache: Dict[str, Tuple[float, List[str]]] = {}
|
||||
_links_cache: Dict[str, Tuple[float, Dict[str, List[str]]]] = {}
|
||||
|
||||
_SCAN_WALK_CACHE_LOCK = threading.Lock()
|
||||
_SCAN_WALK_CACHE_TTL_SECONDS = 1.0
|
||||
|
|
@ -145,6 +162,66 @@ def _scan_cache_set(key: Tuple[str, bool], value: Tuple[List[Dict], List[str]])
|
|||
_SCAN_WALK_CACHE[key] = (time.monotonic(), value)
|
||||
|
||||
|
||||
def _scan_cache_invalidate() -> None:
|
||||
"""Drop the TTL scan cache. Called from every mutation handler so callers
|
||||
that hit the cache immediately after a save/delete/move see the new state
|
||||
instead of stale data from the previous walk."""
|
||||
with _SCAN_WALK_CACHE_LOCK:
|
||||
_SCAN_WALK_CACHE.clear()
|
||||
|
||||
|
||||
def get_tags_and_links_cached(
|
||||
file_path: Path,
|
||||
rel_path: Optional[str] = None,
|
||||
) -> Tuple[List[str], Dict[str, List[str]]]:
|
||||
"""Single-read fused extraction of tags + raw links, cached by mtime.
|
||||
|
||||
Three-tier lookup, cheapest first:
|
||||
1. In-process per-file caches (warm scans on the live server)
|
||||
2. NoteIndex (warm after snapshot load — avoids file reads on cold start)
|
||||
3. Read the file from disk
|
||||
|
||||
rel_path enables tier 2. scan_notes_fast_walk passes it; ad-hoc callers
|
||||
can omit it and just get tiers 1 + 3.
|
||||
"""
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime
|
||||
file_key = str(file_path)
|
||||
|
||||
tag_cached = _tag_cache.get(file_key)
|
||||
link_cached = _links_cache.get(file_key)
|
||||
tags_ok = tag_cached is not None and tag_cached[0] == mtime
|
||||
links_ok = link_cached is not None and link_cached[0] == mtime
|
||||
|
||||
if tags_ok and links_ok:
|
||||
return tag_cached[1], link_cached[1]
|
||||
|
||||
# Try the NoteIndex before reading the file. On a snapshot-warm
|
||||
# startup, this turns a 10K-file cold scan into a no-I/O walk.
|
||||
if rel_path is not None:
|
||||
indexed = note_index.try_get_extraction(rel_path, mtime)
|
||||
if indexed is not None:
|
||||
idx_tags, idx_links = indexed
|
||||
_tag_cache[file_key] = (mtime, idx_tags)
|
||||
_links_cache[file_key] = (mtime, idx_links)
|
||||
return idx_tags, idx_links
|
||||
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
tags = tag_cached[1] if tags_ok else parse_tags(content)
|
||||
links = link_cached[1] if links_ok else extract_links_from_content(content)
|
||||
|
||||
if not tags_ok:
|
||||
_tag_cache[file_key] = (mtime, tags)
|
||||
if not links_ok:
|
||||
_links_cache[file_key] = (mtime, links)
|
||||
|
||||
return tags, links
|
||||
except Exception:
|
||||
return [], {"wikilinks": [], "mdlinks": []}
|
||||
|
||||
|
||||
def validate_path_security(notes_dir: str, path: Path) -> bool:
|
||||
"""
|
||||
Validate that a path is within the notes directory (security check).
|
||||
|
|
@ -176,15 +253,12 @@ def ensure_directories(config: dict):
|
|||
|
||||
|
||||
def create_folder(notes_dir: str, folder_path: str) -> bool:
|
||||
"""Create a new folder in the notes directory"""
|
||||
"""Create a new folder in the notes directory."""
|
||||
full_path = Path(notes_dir) / folder_path
|
||||
|
||||
# Security check
|
||||
if not validate_path_security(notes_dir, full_path):
|
||||
return False
|
||||
|
||||
full_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_scan_cache_invalidate()
|
||||
return True
|
||||
|
||||
|
||||
|
|
@ -219,60 +293,11 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
|
|||
_scan_cache_set(cache_key, normalized_value)
|
||||
return normalized_value
|
||||
|
||||
# TODO: remove this flag and the legacy branch below once the parallel
|
||||
# path has been validated in production with large vaults. To revert to
|
||||
# the pre-parallel behavior, flip _USE_PARALLEL_TAG_SCAN to False.
|
||||
_USE_PARALLEL_TAG_SCAN = True
|
||||
|
||||
# Walk the vault once. Tag/link extraction is deferred out of the walk
|
||||
# (the expensive part is the file I/O) and parallelized below across files.
|
||||
notes: List[Dict] = []
|
||||
folders_set = set()
|
||||
|
||||
if not _USE_PARALLEL_TAG_SCAN:
|
||||
# ===== LEGACY: sequential tag extraction inline with the walk =====
|
||||
for root, dirnames, filenames in os.walk(notes_path):
|
||||
# Skip descending into dot-dirs
|
||||
dirnames[:] = [d for d in dirnames if not d.startswith('.')]
|
||||
|
||||
root_path = Path(root)
|
||||
rel_folder = root_path.relative_to(notes_path).as_posix()
|
||||
if rel_folder != "." and not rel_folder.startswith('.'):
|
||||
folders_set.add(rel_folder)
|
||||
|
||||
for filename in filenames:
|
||||
if filename.startswith('.'):
|
||||
continue
|
||||
|
||||
full_path = root_path / filename
|
||||
try:
|
||||
st = full_path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
relative_path = full_path.relative_to(notes_path)
|
||||
media_type = get_media_type(filename) if include_media else None
|
||||
is_markdown = full_path.suffix.lower() == '.md'
|
||||
should_include = is_markdown or (include_media and media_type is not None)
|
||||
|
||||
if not should_include:
|
||||
continue
|
||||
|
||||
folder = relative_path.parent.as_posix()
|
||||
tags = get_tags_cached(full_path) if is_markdown else []
|
||||
notes.append({
|
||||
"name": full_path.stem,
|
||||
"path": relative_path.as_posix(),
|
||||
"folder": "" if folder == "." else folder,
|
||||
"modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
|
||||
"size": st.st_size,
|
||||
"type": media_type if media_type else "note",
|
||||
"tags": tags,
|
||||
})
|
||||
else:
|
||||
# ===== NEW: tag extraction deferred out of the walk and parallelized =====
|
||||
# Indices of markdown entries in `notes` plus their full paths, so we can
|
||||
# defer tag extraction (the expensive part) out of the walk and parallelize
|
||||
# it across files. Tags are filled in after the walk completes.
|
||||
md_to_tag: List[Tuple[int, Path]] = []
|
||||
md_to_extract: List[Tuple[int, Path, str]] = [] # (note_index, full_path, rel_path)
|
||||
|
||||
for root, dirnames, filenames in os.walk(notes_path):
|
||||
dirnames[:] = [d for d in dirnames if not d.startswith('.')]
|
||||
|
|
@ -296,36 +321,71 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
|
|||
media_type = get_media_type(filename) if include_media else None
|
||||
is_markdown = full_path.suffix.lower() == '.md'
|
||||
should_include = is_markdown or (include_media and media_type is not None)
|
||||
|
||||
if not should_include:
|
||||
continue
|
||||
|
||||
folder = relative_path.parent.as_posix()
|
||||
rel_str = relative_path.as_posix()
|
||||
notes.append({
|
||||
"name": full_path.stem,
|
||||
"path": relative_path.as_posix(),
|
||||
"path": rel_str,
|
||||
"folder": "" if folder == "." else folder,
|
||||
"modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
|
||||
"size": st.st_size,
|
||||
"type": media_type if media_type else "note",
|
||||
"tags": [],
|
||||
"_mtime": st.st_mtime, # internal — popped before returning
|
||||
})
|
||||
if is_markdown:
|
||||
md_to_tag.append((len(notes) - 1, full_path))
|
||||
md_to_extract.append((len(notes) - 1, full_path, rel_str))
|
||||
|
||||
# Parallelize for large vaults (sequential branch avoids thread-spawn
|
||||
# overhead for small vaults). get_tags_cached is mtime-keyed so this is
|
||||
# mostly free on warm cache; the win is the cold first call.
|
||||
if md_to_tag:
|
||||
if len(md_to_tag) >= 50:
|
||||
# Fused tag + raw-link extraction. mtime-keyed cache makes warm scans
|
||||
# mostly free; cold scans win from parallel I/O.
|
||||
#
|
||||
# We don't read full content here — that would double the parsing cost
|
||||
# for every scan, and the search index is the only thing that needs it.
|
||||
# Search is built lazily on the first search request (see
|
||||
# note_index.ensure_search_index_built), which keeps startup fast and
|
||||
# only pays the cost when a user actually searches.
|
||||
sources_raw: Dict[str, Dict[str, List[str]]] = {}
|
||||
if md_to_extract:
|
||||
# Pass rel_path so each call can hit the NoteIndex before falling
|
||||
# back to a file read — critical for fast startup after a snapshot
|
||||
# load on large vaults.
|
||||
path_pairs = [(full, rel) for (_, full, rel) in md_to_extract]
|
||||
if len(md_to_extract) >= 50:
|
||||
workers = min(8, (os.cpu_count() or 4))
|
||||
paths = [p for _, p in md_to_tag]
|
||||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
for (idx, _), tags in zip(md_to_tag, ex.map(get_tags_cached, paths)):
|
||||
notes[idx]["tags"] = tags
|
||||
extraction = list(ex.map(
|
||||
lambda pair: get_tags_and_links_cached(pair[0], pair[1]),
|
||||
path_pairs,
|
||||
))
|
||||
else:
|
||||
for idx, p in md_to_tag:
|
||||
notes[idx]["tags"] = get_tags_cached(p)
|
||||
extraction = [get_tags_and_links_cached(full, rel) for (full, rel) in path_pairs]
|
||||
|
||||
for (idx, _full, rel_str), (tags, links) in zip(md_to_extract, extraction):
|
||||
notes[idx]["tags"] = tags
|
||||
sources_raw[rel_str] = links
|
||||
|
||||
# Populate the unified index. Cheap when the fingerprint matches (warm
|
||||
# scan, nothing changed) — short-circuits internally. No-op when off.
|
||||
notes_meta = [
|
||||
NoteRecord(
|
||||
path=n["path"],
|
||||
name=n["name"],
|
||||
folder=n["folder"],
|
||||
modified=n["modified"],
|
||||
size=n["size"],
|
||||
type=n["type"],
|
||||
mtime=n["_mtime"],
|
||||
tags=tuple(n["tags"]),
|
||||
)
|
||||
for n in notes
|
||||
]
|
||||
note_index.populate_from_scan(notes_meta, folders_set, sources_raw)
|
||||
|
||||
for n in notes:
|
||||
n.pop("_mtime", None)
|
||||
|
||||
value = (sorted(notes, key=lambda x: x.get('modified', ''), reverse=True), sorted(folders_set))
|
||||
if use_cache:
|
||||
|
|
@ -333,132 +393,106 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media:
|
|||
return value
|
||||
|
||||
def move_note(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]:
|
||||
"""Move a note to a different location
|
||||
|
||||
Returns:
|
||||
Tuple of (success: bool, error_message: str)
|
||||
"""
|
||||
"""Move a note. Returns (success, error_message)."""
|
||||
old_full_path = Path(notes_dir) / old_path
|
||||
new_full_path = Path(notes_dir) / new_path
|
||||
|
||||
# Security checks
|
||||
if not validate_path_security(notes_dir, old_full_path):
|
||||
return False, "Invalid source path"
|
||||
if not validate_path_security(notes_dir, new_full_path):
|
||||
return False, "Invalid destination path"
|
||||
|
||||
if not old_full_path.exists():
|
||||
return False, f"Source note does not exist: {old_path}"
|
||||
|
||||
# Check if target already exists (prevent overwriting)
|
||||
if new_full_path.exists():
|
||||
return False, f"A note already exists at: {new_path}"
|
||||
|
||||
# Invalidate cache for old path
|
||||
old_key = str(old_full_path)
|
||||
if old_key in _tag_cache:
|
||||
del _tag_cache[old_key]
|
||||
_drop_path_caches(old_full_path)
|
||||
|
||||
try:
|
||||
# Create parent directory if needed
|
||||
new_full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Move the file
|
||||
old_full_path.rename(new_full_path)
|
||||
except Exception as e:
|
||||
return False, f"Failed to move file: {str(e)}"
|
||||
|
||||
# Note: We don't automatically delete empty folders to preserve user's folder structure
|
||||
|
||||
note_index.on_note_renamed(notes_dir, old_full_path, new_full_path)
|
||||
_scan_cache_invalidate()
|
||||
return True, ""
|
||||
|
||||
|
||||
def move_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]:
|
||||
"""Move a folder to a different location
|
||||
|
||||
Returns:
|
||||
Tuple of (success: bool, error_message: str)
|
||||
"""
|
||||
import shutil
|
||||
|
||||
"""Move a folder. Returns (success, error_message)."""
|
||||
old_full_path = Path(notes_dir) / old_path
|
||||
new_full_path = Path(notes_dir) / new_path
|
||||
|
||||
# Security checks
|
||||
if not validate_path_security(notes_dir, old_full_path):
|
||||
return False, "Invalid source path"
|
||||
if not validate_path_security(notes_dir, new_full_path):
|
||||
return False, "Invalid destination path"
|
||||
|
||||
if not old_full_path.exists() or not old_full_path.is_dir():
|
||||
return False, f"Source folder does not exist: {old_path}"
|
||||
|
||||
# Check if target already exists
|
||||
if new_full_path.exists():
|
||||
return False, f"A folder already exists at: {new_path}"
|
||||
|
||||
# Invalidate cache for all notes in this folder
|
||||
global _tag_cache
|
||||
old_path_str = str(old_full_path)
|
||||
keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(old_path_str)]
|
||||
for key in keys_to_delete:
|
||||
del _tag_cache[key]
|
||||
_drop_prefix_caches(old_full_path)
|
||||
|
||||
try:
|
||||
# Create parent directory if needed
|
||||
new_full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Move the folder
|
||||
shutil.move(str(old_full_path), str(new_full_path))
|
||||
except Exception as e:
|
||||
return False, f"Failed to move folder: {str(e)}"
|
||||
|
||||
# Note: We don't automatically delete empty folders to preserve user's folder structure
|
||||
|
||||
note_index.on_folder_renamed(notes_dir, old_full_path, new_full_path)
|
||||
_scan_cache_invalidate()
|
||||
return True, ""
|
||||
|
||||
|
||||
def rename_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]:
|
||||
"""Rename a folder (same as move but for clarity)"""
|
||||
"""Rename a folder (same as move, named for clarity)."""
|
||||
return move_folder(notes_dir, old_path, new_path)
|
||||
|
||||
|
||||
def delete_folder(notes_dir: str, folder_path: str) -> bool:
|
||||
"""Delete a folder and all its contents"""
|
||||
"""Delete a folder and all its contents."""
|
||||
try:
|
||||
full_path = Path(notes_dir) / folder_path
|
||||
|
||||
# Security check: ensure the path is within notes_dir
|
||||
if not validate_path_security(notes_dir, full_path):
|
||||
print(f"Security: Path is outside notes directory: {full_path}")
|
||||
return False
|
||||
|
||||
if not full_path.exists():
|
||||
print(f"Folder does not exist: {full_path}")
|
||||
return False
|
||||
|
||||
if not full_path.is_dir():
|
||||
print(f"Path is not a directory: {full_path}")
|
||||
return False
|
||||
|
||||
# Invalidate cache for all notes in this folder
|
||||
global _tag_cache
|
||||
folder_path_str = str(full_path)
|
||||
keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(folder_path_str)]
|
||||
for key in keys_to_delete:
|
||||
del _tag_cache[key]
|
||||
|
||||
# Delete the folder and all its contents
|
||||
_drop_prefix_caches(full_path)
|
||||
shutil.rmtree(full_path)
|
||||
print(f"Successfully deleted folder: {full_path}")
|
||||
note_index.on_folder_deleted(notes_dir, full_path)
|
||||
_scan_cache_invalidate()
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error deleting folder '{folder_path}': {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
|
||||
def _drop_path_caches(full_path: Path) -> None:
|
||||
"""Evict the per-file mtime caches for a single note."""
|
||||
key = str(full_path)
|
||||
_tag_cache.pop(key, None)
|
||||
_links_cache.pop(key, None)
|
||||
|
||||
|
||||
def _drop_prefix_caches(folder_full_path: Path) -> None:
|
||||
"""Evict per-file mtime caches for every entry under a folder."""
|
||||
prefix = str(folder_full_path)
|
||||
for k in [k for k in _tag_cache if k.startswith(prefix)]:
|
||||
_tag_cache.pop(k, None)
|
||||
for k in [k for k in _links_cache if k.startswith(prefix)]:
|
||||
_links_cache.pop(k, None)
|
||||
|
||||
|
||||
|
||||
|
||||
def get_note_content(notes_dir: str, note_path: str) -> Optional[str]:
|
||||
|
|
@ -477,110 +511,111 @@ def get_note_content(notes_dir: str, note_path: str) -> Optional[str]:
|
|||
|
||||
|
||||
def save_note(notes_dir: str, note_path: str, content: str) -> bool:
|
||||
"""Save or update a note"""
|
||||
"""Save or update a note."""
|
||||
full_path = Path(notes_dir) / note_path
|
||||
|
||||
# Ensure .md extension
|
||||
if not note_path.endswith('.md'):
|
||||
full_path = full_path.with_suffix('.md')
|
||||
|
||||
# Security check
|
||||
if not validate_path_security(notes_dir, full_path):
|
||||
return False
|
||||
|
||||
# Create parent directories if needed
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(full_path, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
|
||||
# Refresh the per-file mtime caches with what we just wrote, so the next
|
||||
# scan doesn't re-parse this file. extract_links + parse_tags + save are
|
||||
# tiny relative to the file write.
|
||||
try:
|
||||
mtime = full_path.stat().st_mtime
|
||||
file_key = str(full_path)
|
||||
_tag_cache[file_key] = (mtime, parse_tags(content))
|
||||
_links_cache[file_key] = (mtime, extract_links_from_content(content))
|
||||
except Exception:
|
||||
pass # caches are best-effort
|
||||
|
||||
note_index.on_note_saved(notes_dir, full_path, content)
|
||||
_scan_cache_invalidate()
|
||||
return True
|
||||
|
||||
|
||||
def delete_note(notes_dir: str, note_path: str) -> bool:
|
||||
"""Delete a note"""
|
||||
"""Delete a note."""
|
||||
full_path = Path(notes_dir) / note_path
|
||||
|
||||
if not full_path.exists():
|
||||
return False
|
||||
|
||||
# Security check
|
||||
if not validate_path_security(notes_dir, full_path):
|
||||
return False
|
||||
|
||||
# Invalidate cache for this note
|
||||
file_key = str(full_path)
|
||||
if file_key in _tag_cache:
|
||||
del _tag_cache[file_key]
|
||||
|
||||
_drop_path_caches(full_path)
|
||||
full_path.unlink()
|
||||
|
||||
# Note: We don't automatically delete empty folders to preserve user's folder structure
|
||||
|
||||
note_index.on_note_deleted(notes_dir, full_path)
|
||||
_scan_cache_invalidate()
|
||||
return True
|
||||
|
||||
|
||||
def search_notes(notes_dir: str, query: str) -> List[Dict]:
|
||||
"""
|
||||
Full-text search through note contents only.
|
||||
Does NOT search in file names, folder names, or paths - only note content.
|
||||
Uses character-based context extraction with highlighted matches.
|
||||
"""Full-text search through note contents.
|
||||
|
||||
Only searches inside note content (not filenames or paths). The index, when
|
||||
available, narrows down the candidate files via a token-AND match on its
|
||||
inverted index — then the existing per-line snippet extraction runs on the
|
||||
narrow set. Output is byte-identical to a full scan.
|
||||
"""
|
||||
from html import escape
|
||||
results = []
|
||||
results: List[Dict] = []
|
||||
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
|
||||
|
||||
# Build the search index lazily on the first search request. Subsequent
|
||||
# searches hit the warm index; the first one pays the cost (read every
|
||||
# file once) and is roughly as slow as the legacy full scan would be.
|
||||
note_index.ensure_search_index(notes_dir)
|
||||
candidates = note_index.try_search_candidates(query)
|
||||
|
||||
for note in notes:
|
||||
md_file = Path(notes_dir) / note["path"]
|
||||
path = note["path"]
|
||||
if candidates is not None and path not in candidates:
|
||||
continue
|
||||
md_file = Path(notes_dir) / path
|
||||
try:
|
||||
with open(md_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find all matches using regex (case-insensitive)
|
||||
matches = list(re.finditer(re.escape(query), content, re.IGNORECASE))
|
||||
if not matches:
|
||||
continue
|
||||
|
||||
if matches:
|
||||
matched_lines = []
|
||||
|
||||
for match in matches[:3]: # Limit to 3 matches per file
|
||||
for match in matches[:3]:
|
||||
start_index = match.start()
|
||||
end_index = match.end()
|
||||
matched_text = match.group() # Preserve original case
|
||||
matched_text = match.group()
|
||||
|
||||
# Create slice window: ±15 characters around match
|
||||
context_start = max(0, start_index - 15)
|
||||
context_end = min(len(content), end_index + 15)
|
||||
|
||||
# Extract and clean parts (newlines → spaces)
|
||||
before = escape(content[context_start:start_index].replace('\n', ' '))
|
||||
after = escape(content[end_index:context_end].replace('\n', ' '))
|
||||
matched_clean = escape(matched_text.replace('\n', ' '))
|
||||
|
||||
# Build snippet with <mark> highlight (styled via CSS)
|
||||
snippet = f'{before}<mark class="search-highlight">{matched_clean}</mark>{after}'
|
||||
|
||||
# Add ellipsis if truncated at start
|
||||
if context_start > 0:
|
||||
snippet = '...' + snippet
|
||||
|
||||
# Add ellipsis if truncated at end
|
||||
if context_end < len(content):
|
||||
snippet = snippet + '...'
|
||||
|
||||
# Calculate line number by counting newlines up to match start
|
||||
line_number = content.count('\n', 0, start_index) + 1
|
||||
|
||||
matched_lines.append({
|
||||
"line_number": line_number,
|
||||
"context": snippet
|
||||
"context": snippet,
|
||||
})
|
||||
|
||||
relative_path = Path(note["path"])
|
||||
relative_path = Path(path)
|
||||
results.append({
|
||||
"name": md_file.stem,
|
||||
"path": str(relative_path.as_posix()),
|
||||
"folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "",
|
||||
"matches": matched_lines
|
||||
"matches": matched_lines,
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
|
@ -852,105 +887,66 @@ def parse_tags(content: str) -> List[str]:
|
|||
return []
|
||||
|
||||
|
||||
def get_tags_cached(file_path: Path) -> List[str]:
|
||||
"""
|
||||
Get tags for a file with caching based on modification time.
|
||||
|
||||
Args:
|
||||
file_path: Path to the markdown file
|
||||
|
||||
Returns:
|
||||
List of tags from the file (cached if mtime unchanged)
|
||||
"""
|
||||
global _tag_cache
|
||||
|
||||
try:
|
||||
# Get current modification time
|
||||
mtime = file_path.stat().st_mtime
|
||||
file_key = str(file_path)
|
||||
|
||||
# Check cache
|
||||
if file_key in _tag_cache:
|
||||
cached_mtime, cached_tags = _tag_cache[file_key]
|
||||
if cached_mtime == mtime:
|
||||
# Cache hit! Return cached tags
|
||||
return cached_tags
|
||||
|
||||
# Cache miss or stale - parse tags
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
tags = parse_tags(content)
|
||||
|
||||
# Update cache
|
||||
_tag_cache[file_key] = (mtime, tags)
|
||||
return tags
|
||||
|
||||
except Exception:
|
||||
# If anything fails, return empty list
|
||||
return []
|
||||
|
||||
|
||||
def clear_tag_cache():
|
||||
"""Clear the tag cache (useful for testing or manual cache invalidation)"""
|
||||
global _tag_cache
|
||||
_tag_cache.clear()
|
||||
|
||||
|
||||
def get_all_tags(notes_dir: str) -> Dict[str, int]:
|
||||
"""
|
||||
Get all tags used across all notes with their count (cached).
|
||||
"""All tags in the vault with note counts. Index-backed when available."""
|
||||
# Fast path FIRST — when the index is ready it knows every tag count,
|
||||
# so we skip the scan entirely (no disk walk, no JSON of 10K notes).
|
||||
indexed = note_index.try_all_tags()
|
||||
if indexed is not None:
|
||||
return indexed
|
||||
|
||||
Args:
|
||||
notes_dir: Directory containing notes
|
||||
|
||||
Returns:
|
||||
Dictionary mapping tag names to note counts
|
||||
"""
|
||||
tag_counts = {}
|
||||
# Legacy fallback: scan the vault and aggregate.
|
||||
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
|
||||
|
||||
tag_counts: Dict[str, int] = {}
|
||||
for note in notes:
|
||||
md_file = Path(notes_dir) / note["path"]
|
||||
# Get tags using cache
|
||||
tags = get_tags_cached(md_file)
|
||||
|
||||
for tag in tags:
|
||||
for tag in note.get("tags", []):
|
||||
tag_counts[tag] = tag_counts.get(tag, 0) + 1
|
||||
|
||||
return dict(sorted(tag_counts.items()))
|
||||
|
||||
|
||||
def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]:
|
||||
"""
|
||||
Get all notes that have a specific tag (cached).
|
||||
|
||||
Args:
|
||||
notes_dir: Directory containing notes
|
||||
tag: Tag to filter by (case-insensitive)
|
||||
|
||||
Returns:
|
||||
List of note dictionaries matching the tag
|
||||
"""
|
||||
matching_notes = []
|
||||
"""All notes carrying `tag` (case-insensitive). Index-backed when available."""
|
||||
tag_lower = tag.lower()
|
||||
|
||||
# Fast path FIRST: every field we need lives in the NoteRecord. Skip
|
||||
# the vault scan when the index can answer directly.
|
||||
indexed_paths = note_index.try_notes_by_tag(tag_lower)
|
||||
if indexed_paths is not None:
|
||||
idx = note_index.get_index()
|
||||
records = [idx.get_note_record(p) for p in indexed_paths]
|
||||
matching = [
|
||||
{
|
||||
"name": r.name,
|
||||
"path": r.path,
|
||||
"folder": r.folder,
|
||||
"modified": r.modified,
|
||||
"size": r.size,
|
||||
"tags": list(r.tags),
|
||||
}
|
||||
for r in records
|
||||
if r is not None and r.type == "note"
|
||||
]
|
||||
# Match legacy sort order (scan returns notes sorted by modified desc).
|
||||
matching.sort(key=lambda n: n["modified"], reverse=True)
|
||||
return matching
|
||||
|
||||
# Legacy fallback.
|
||||
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
|
||||
|
||||
matching: List[Dict] = []
|
||||
for note in notes:
|
||||
md_file = Path(notes_dir) / note["path"]
|
||||
# Get tags using cache
|
||||
tags = get_tags_cached(md_file)
|
||||
|
||||
if note.get("type") != "note":
|
||||
continue
|
||||
tags = note.get("tags", [])
|
||||
if tag_lower in tags:
|
||||
matching_notes.append({
|
||||
matching.append({
|
||||
"name": note["name"],
|
||||
"path": note["path"],
|
||||
"folder": note["folder"],
|
||||
"modified": note["modified"],
|
||||
"size": note["size"],
|
||||
"tags": tags
|
||||
"tags": tags,
|
||||
})
|
||||
|
||||
return matching_notes
|
||||
return matching
|
||||
|
||||
|
||||
# ============================================================================
|
||||
|
|
@ -1107,69 +1103,38 @@ def apply_template_placeholders(content: str, note_path: str) -> str:
|
|||
return result
|
||||
|
||||
|
||||
def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
|
||||
def _extract_backlink_references(
|
||||
notes_dir: str,
|
||||
source_path: str,
|
||||
target_path: str,
|
||||
target_path_lower: str,
|
||||
target_path_no_ext_lower: str,
|
||||
wikilink_refs: set,
|
||||
) -> List[Dict]:
|
||||
"""Read one source file and pull line-level references that point to target.
|
||||
|
||||
Same regex + resolution rules as the legacy get_backlinks loop body; just
|
||||
factored out so both the index-driven and the legacy code paths use the
|
||||
exact same context-extraction logic.
|
||||
"""
|
||||
Find all notes that link TO the specified note (reverse links / backlinks).
|
||||
|
||||
Args:
|
||||
notes_dir: Base directory containing notes
|
||||
target_note_path: Path of the note to find backlinks for
|
||||
|
||||
Returns:
|
||||
List of backlink objects with path, context, and line_number
|
||||
"""
|
||||
backlinks = []
|
||||
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
|
||||
|
||||
# Normalize target path for matching
|
||||
target_path = target_note_path
|
||||
target_path_lower = target_path.lower()
|
||||
target_path_no_ext = target_path.replace('.md', '')
|
||||
target_path_no_ext_lower = target_path_no_ext.lower()
|
||||
target_name = Path(target_path).stem.lower()
|
||||
|
||||
# For wikilinks: global name matching (find note anywhere by name)
|
||||
wikilink_refs = {
|
||||
target_path_lower,
|
||||
target_path_no_ext_lower,
|
||||
target_name,
|
||||
}
|
||||
|
||||
for note in notes:
|
||||
if note.get('type') != 'note':
|
||||
continue
|
||||
|
||||
source_path = note['path']
|
||||
|
||||
# Skip self-references
|
||||
if source_path == target_path:
|
||||
continue
|
||||
|
||||
# Get source folder for resolving relative markdown links
|
||||
source_folder = str(Path(source_path).parent).replace('\\', '/')
|
||||
if source_folder == '.':
|
||||
source_folder = ''
|
||||
|
||||
# Read note content
|
||||
full_path = Path(notes_dir) / source_path
|
||||
try:
|
||||
with open(full_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
except Exception:
|
||||
continue
|
||||
return []
|
||||
|
||||
lines = content.split('\n')
|
||||
found_links = []
|
||||
found_links: List[Dict] = []
|
||||
|
||||
for line_num, line in enumerate(lines, 1):
|
||||
# Find wikilinks: [[target]] or [[target|display]]
|
||||
# Wikilinks use GLOBAL matching (find note anywhere by name)
|
||||
wikilink_matches = re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line)
|
||||
for match in wikilink_matches:
|
||||
for match in re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line):
|
||||
link_target = match.group(1).strip().lower()
|
||||
link_target_no_ext = link_target.replace('.md', '')
|
||||
|
||||
# Check if this wikilink points to our target (global match)
|
||||
if link_target in wikilink_refs or link_target_no_ext in wikilink_refs:
|
||||
start = max(0, match.start() - 30)
|
||||
end = min(len(line), match.end() + 30)
|
||||
|
|
@ -1178,47 +1143,34 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
|
|||
context = '...' + context
|
||||
if end < len(line):
|
||||
context = context + '...'
|
||||
|
||||
found_links.append({
|
||||
"line_number": line_num,
|
||||
"context": context,
|
||||
"type": "wikilink"
|
||||
"type": "wikilink",
|
||||
})
|
||||
|
||||
# Find markdown links: [text](path)
|
||||
# Markdown links must RESOLVE as paths (relative to source or absolute)
|
||||
markdown_matches = re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line)
|
||||
for match in markdown_matches:
|
||||
link_path = match.group(2).split('#')[0] # Remove anchor
|
||||
for match in re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line):
|
||||
link_path = match.group(2).split('#')[0]
|
||||
if not link_path:
|
||||
continue
|
||||
|
||||
link_path = urllib.parse.unquote(link_path)
|
||||
if link_path.startswith('./'):
|
||||
link_path = link_path[2:]
|
||||
|
||||
# Add .md if not present
|
||||
link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md'
|
||||
|
||||
# Resolve the link path to get the actual target
|
||||
resolved_path = None
|
||||
|
||||
# 1. Try resolving relative to source folder
|
||||
if source_folder and not link_path.startswith('/'):
|
||||
relative_path = f"{source_folder}/{link_path_with_md}"
|
||||
if relative_path.lower() == target_path_lower:
|
||||
resolved_path = target_path
|
||||
elif f"{source_folder}/{link_path}".lower() == target_path_no_ext_lower:
|
||||
resolved_path = target_path
|
||||
|
||||
# 2. Try as absolute path from root
|
||||
if not resolved_path:
|
||||
if link_path_with_md.lower() == target_path_lower:
|
||||
resolved_path = target_path
|
||||
elif link_path.lower() == target_path_no_ext_lower:
|
||||
resolved_path = target_path
|
||||
|
||||
# Only add if the resolved path matches the target
|
||||
if resolved_path:
|
||||
start = max(0, match.start() - 30)
|
||||
end = min(len(line), match.end() + 30)
|
||||
|
|
@ -1227,19 +1179,61 @@ def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
|
|||
context = '...' + context
|
||||
if end < len(line):
|
||||
context = context + '...'
|
||||
|
||||
found_links.append({
|
||||
"line_number": line_num,
|
||||
"context": context,
|
||||
"type": "markdown"
|
||||
"type": "markdown",
|
||||
})
|
||||
|
||||
# If we found links in this note, add it to backlinks
|
||||
if found_links:
|
||||
return found_links
|
||||
|
||||
|
||||
def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]:
|
||||
"""Find all notes that link TO the specified note.
|
||||
|
||||
Index-backed when available: the index hands us a superset of candidate
|
||||
source files (O(1)), we read only those, then the per-line matcher
|
||||
filters down to true matches with line-level context. Output is
|
||||
byte-identical to a full vault scan.
|
||||
"""
|
||||
target_path = target_note_path
|
||||
target_path_lower = target_path.lower()
|
||||
target_path_no_ext_lower = target_path_lower.replace('.md', '')
|
||||
wikilink_refs = {
|
||||
target_path_lower,
|
||||
target_path_no_ext_lower,
|
||||
Path(target_path).stem.lower(),
|
||||
}
|
||||
|
||||
# scan_notes_fast_walk is TTL-cached, so calling it here is a dict-lookup
|
||||
# in the warm case. We need it for the canonical ordering of `notes`.
|
||||
notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False)
|
||||
|
||||
candidates = note_index.try_backlink_candidates(target_path)
|
||||
|
||||
backlinks: List[Dict] = []
|
||||
for note in notes:
|
||||
if note.get('type') != 'note':
|
||||
continue
|
||||
source_path = note['path']
|
||||
if source_path == target_path:
|
||||
continue
|
||||
if candidates is not None and source_path not in candidates:
|
||||
continue
|
||||
|
||||
refs = _extract_backlink_references(
|
||||
notes_dir,
|
||||
source_path,
|
||||
target_path,
|
||||
target_path_lower,
|
||||
target_path_no_ext_lower,
|
||||
wikilink_refs,
|
||||
)
|
||||
if refs:
|
||||
backlinks.append({
|
||||
"path": source_path,
|
||||
"name": note['name'].replace('.md', ''),
|
||||
"references": found_links[:3] # Limit to 3 references per note
|
||||
"references": refs[:3],
|
||||
})
|
||||
|
||||
return backlinks
|
||||
|
|
|
|||
|
|
@ -580,6 +580,59 @@ Returns application statistics at a glance. Designed for dashboard widgets (e.g.
|
|||
label: Version
|
||||
```
|
||||
|
||||
### Get Index Stats
|
||||
```http
|
||||
GET /api/index/stats
|
||||
```
|
||||
Returns internal counters and sizes for the in-memory note index. Useful for confirming the index is built and observing its growth — handy when debugging slow endpoints or verifying that incremental updates are firing on every save/delete.
|
||||
|
||||
The index is rebuilt on the first scan after each process start and updated incrementally on every save/delete/move. It lives in process memory only — no disk persistence.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"enabled": true,
|
||||
"built": true,
|
||||
"search_built": false,
|
||||
"notes": 142,
|
||||
"folders": 12,
|
||||
"tags": 37,
|
||||
"links_forward_entries": 89,
|
||||
"links_backward_entries": 76,
|
||||
"wikilink_tokens": 134,
|
||||
"search_terms": 0,
|
||||
"counters": {
|
||||
"build_count": 1,
|
||||
"last_build_ms": 12.4,
|
||||
"last_built_at": "2026-03-17T14:32:00+00:00",
|
||||
"incremental_updates": 8,
|
||||
"fingerprint_short_circuits": 3,
|
||||
"search_build_count": 0,
|
||||
"last_search_build_ms": 0.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `enabled` | Whether the index is active (always `true` in current builds) |
|
||||
| `built` | `true` after the first vault scan completes |
|
||||
| `search_built` | `true` after the first `/api/search` request (search index is built lazily) |
|
||||
| `notes` | Number of note records currently held in the index |
|
||||
| `folders` | Number of folder paths |
|
||||
| `tags` | Number of unique tags |
|
||||
| `links_forward_entries` | Number of notes that link out to at least one other note |
|
||||
| `links_backward_entries` | Number of notes that have at least one incoming backlink |
|
||||
| `wikilink_tokens` | Number of unique wikilink tokens (used for loose backlink matching by stem name) |
|
||||
| `search_terms` | Number of unique terms in the full-text inverted search index (0 until first search) |
|
||||
| `counters.build_count` | Total number of full index rebuilds since process start |
|
||||
| `counters.last_build_ms` | Wall-clock time of the most recent full rebuild |
|
||||
| `counters.last_built_at` | ISO timestamp of the most recent full rebuild |
|
||||
| `counters.incremental_updates` | Number of single-note updates applied (save/delete/rename) |
|
||||
| `counters.fingerprint_short_circuits` | Times a re-scan was skipped because the vault hash matched the previous scan |
|
||||
| `counters.search_build_count` | Total number of search-index rebuilds |
|
||||
| `counters.last_search_build_ms` | Wall-clock time of the most recent search-index rebuild |
|
||||
|
||||
### Health Check
|
||||
```http
|
||||
GET /health
|
||||
|
|
|
|||
Loading…
Reference in New Issue