diff --git a/backend/export.py b/backend/export.py index cea3fd2..6bd2a28 100644 --- a/backend/export.py +++ b/backend/export.py @@ -8,6 +8,7 @@ replaced with placeholder HTML since they would make exports too large. """ import base64 +import logging import re from pathlib import Path from typing import Optional, Tuple @@ -16,6 +17,8 @@ import mimetypes # Import shared media type definitions and scanner from utils to avoid duplication from backend.utils import MEDIA_EXTENSIONS, get_media_type, scan_notes_fast_walk +logger = logging.getLogger("uvicorn.error") + def get_media_as_base64(media_path: Path) -> Optional[Tuple[str, str]]: """ @@ -41,7 +44,7 @@ def get_media_as_base64(media_path: Path) -> Optional[Tuple[str, str]]: base64_data = base64.b64encode(media_data).decode('utf-8') return (f"data:{mime_type};base64,{base64_data}", media_type) except Exception as e: - print(f"Failed to read media {media_path}: {e}") + logger.error("Failed to read media %s: %s", media_path, e) return None diff --git a/backend/main.py b/backend/main.py index 72781ff..8b12b56 100644 --- a/backend/main.py +++ b/backend/main.py @@ -12,6 +12,7 @@ from starlette.middleware.sessions import SessionMiddleware import os import yaml import json +import logging from pathlib import Path from typing import List, Optional import aiofiles @@ -22,8 +23,11 @@ from slowapi import Limiter, _rate_limit_exceeded_handler from slowapi.util import get_remote_address from slowapi.errors import RateLimitExceeded +logger = logging.getLogger("uvicorn.error") + from .utils import ( scan_notes_fast_walk, + ensure_index_built, get_note_content, save_note, delete_note, @@ -36,6 +40,7 @@ from .utils import ( rename_folder, delete_folder, save_uploaded_image, + _scan_cache_invalidate, validate_path_security, get_all_tags, get_notes_by_tag, @@ -45,6 +50,7 @@ from .utils import ( paginate, get_backlinks, ) +from . import note_index from .plugins import PluginManager from .themes import get_available_themes, get_theme_css from .share import ( @@ -77,9 +83,9 @@ with open(version_path, 'r', encoding='utf-8') as f: if 'AUTHENTICATION_ENABLED' in os.environ: auth_enabled = os.getenv('AUTHENTICATION_ENABLED', 'false').lower() in ('true', '1', 'yes') config['authentication']['enabled'] = auth_enabled - print(f"🔐 Authentication {'ENABLED' if auth_enabled else 'DISABLED'} (from AUTHENTICATION_ENABLED env var)") + logger.info("Authentication %s (from AUTHENTICATION_ENABLED env var)", 'ENABLED' if auth_enabled else 'DISABLED') else: - print(f"🔐 Authentication {'ENABLED' if config.get('authentication', {}).get('enabled', False) else 'DISABLED'} (from config.yaml)") + logger.info("Authentication %s (from config.yaml)", 'ENABLED' if config.get('authentication', {}).get('enabled', False) else 'DISABLED') # Password configuration priority: # 1. AUTHENTICATION_PASSWORD env var (hashed at startup) @@ -92,9 +98,9 @@ if 'AUTHENTICATION_PASSWORD' in os.environ: plain_password.encode('utf-8'), bcrypt.gensalt() ).decode('utf-8') - print("🔑 Password loaded from AUTHENTICATION_PASSWORD env var") + logger.info("Password loaded from AUTHENTICATION_PASSWORD env var") else: - print("⚠️ WARNING: AUTHENTICATION_PASSWORD env var is empty - ignoring") + logger.warning("AUTHENTICATION_PASSWORD env var is empty - ignoring") elif config.get('authentication', {}).get('password', '').strip(): plain_password = config['authentication']['password'].strip() config['authentication']['password_hash'] = bcrypt.hashpw( @@ -102,12 +108,12 @@ elif config.get('authentication', {}).get('password', '').strip(): bcrypt.gensalt() ).decode('utf-8') del config['authentication']['password'] - print("🔑 Password loaded from config.yaml") + logger.info("Password loaded from config.yaml") # Allow secret key to be set via environment variable (for session security) if 'AUTHENTICATION_SECRET_KEY' in os.environ: config['authentication']['secret_key'] = os.getenv('AUTHENTICATION_SECRET_KEY') - print("🔐 Secret key loaded from AUTHENTICATION_SECRET_KEY env var") + logger.info("Secret key loaded from AUTHENTICATION_SECRET_KEY env var") # API key configuration for external integrations (MCP servers, scripts, etc.) # Priority: AUTHENTICATION_API_KEY env var > authentication.api_key in config.yaml @@ -115,11 +121,11 @@ if 'AUTHENTICATION_API_KEY' in os.environ: api_key_value = os.getenv('AUTHENTICATION_API_KEY', '').strip() if api_key_value: config['authentication']['api_key'] = api_key_value - print("🔑 API key loaded from AUTHENTICATION_API_KEY env var") + logger.info("API key loaded from AUTHENTICATION_API_KEY env var") else: config['authentication']['api_key'] = '' elif config.get('authentication', {}).get('api_key', '').strip(): - print("🔑 API key loaded from config.yaml") + logger.info("API key loaded from config.yaml") else: config['authentication']['api_key'] = '' @@ -131,15 +137,29 @@ if config.get('authentication', {}).get('enabled', False): _is_default_secret = _secret_key in ('', 'change_this_to_a_random_secret_key_in_production') if not _has_password and not _has_api_key: - print("🚨 CRITICAL: Authentication enabled but NO auth methods configured - ALL access will be denied!") + logger.critical("Authentication enabled but NO auth methods configured - ALL access will be denied!") else: if not _has_password: - print("⚠️ WARNING: No password configured - web UI login will not work") + logger.warning("No password configured - web UI login will not work") if not _has_api_key: - print("⚠️ WARNING: No API key configured - external integrations will require session cookies") + logger.warning("No API key configured - external integrations will require session cookies") if _is_default_secret: - print("🚨 SECURITY WARNING: Using default secret_key - sessions can be forged! Change it in config.yaml") + logger.critical("Using default secret_key - sessions can be forged! Change it in config.yaml") + +# Storage paths: env vars override config.yaml. Logged either way so the +# resolved location is visible at startup. +_notes_source = "config.yaml" +if 'NOTES_DIR' in os.environ: + config['storage']['notes_dir'] = os.getenv('NOTES_DIR') + _notes_source = "NOTES_DIR env var" +logger.info("Notes directory: %s (from %s)", config['storage']['notes_dir'], _notes_source) + +_plugins_source = "config.yaml" +if 'PLUGINS_DIR' in os.environ: + config['storage']['plugins_dir'] = os.getenv('PLUGINS_DIR') + _plugins_source = "PLUGINS_DIR env var" +logger.info("Plugins directory: %s (from %s)", config['storage']['plugins_dir'], _plugins_source) # OpenAPI tag metadata for grouping endpoints in Swagger UI tags_metadata = [ @@ -176,7 +196,7 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) -print(f"🌐 CORS allowed origins: {allowed_origins}") +logger.info("CORS allowed origins: %s", allowed_origins) # =========================================================== # ================= @@ -199,7 +219,7 @@ def safe_error_message(error: Exception, user_message: str = "An error occurred" error_details = f"{type(error).__name__}: {str(error)}" # Always log the full error server-side - print(f"⚠️ [ERROR] {error_details}") + logger.error(error_details) # In debug mode, return detailed error to help with development if config.get('server', {}).get('debug', False): @@ -245,7 +265,7 @@ if DEMO_MODE: limiter = Limiter(key_func=get_remote_address, default_limits=["200/hour"]) app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) - print("🎭 DEMO MODE enabled - Rate limiting active") + logger.info("DEMO MODE enabled - Rate limiting active") else: # Production/self-hosted mode - no restrictions # Create a dummy limiter that doesn't actually limit @@ -265,6 +285,7 @@ plugin_manager = PluginManager(config['storage']['plugins_dir']) # Run app startup hooks plugin_manager.run_hook('on_app_startup') + # Mount static files static_path = Path(__file__).parent.parent / "frontend" app.mount("/static", StaticFiles(directory=static_path), name="static") @@ -413,7 +434,7 @@ def verify_password(password: str) -> bool: try: return bcrypt.checkpw(password.encode('utf-8'), password_hash.encode('utf-8')) except Exception as e: - print(f"Password verification error: {e}") + logger.error("Password verification error: %s", e) return False @@ -841,7 +862,8 @@ async def move_media_endpoint(request: Request, data: dict): # Move the file import shutil shutil.move(str(old_full_path), str(new_full_path)) - + _scan_cache_invalidate() + return {"success": True, "message": "Media moved successfully", "newPath": new_path} except HTTPException: @@ -1483,171 +1505,15 @@ async def search( @api_router.get("/graph", tags=["Graph"]) async def get_graph(): - """Get graph data for note visualization with wikilink and markdown link detection""" + """Graph data (nodes + resolved wikilink/markdown edges) for the visualizer.""" try: - import re - import urllib.parse - notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) - nodes = [] - edges = [] - - # Build set of valid note names/paths for matching - note_paths = set() - note_paths_lower = {} # Map lowercase path -> actual path for case-insensitive matching - note_names = {} # Map name -> path for quick lookup - - for note in notes: - if note.get('type') == 'note': - note_paths.add(note['path']) - note_paths.add(note['path'].replace('.md', '')) - # Store lowercase path -> actual path mapping for case-insensitive matching - note_paths_lower[note['path'].lower()] = note['path'] - note_paths_lower[note['path'].replace('.md', '').lower()] = note['path'] - # Store name -> path mapping (without extension) - name = note['name'].replace('.md', '') - note_names[name.lower()] = note['path'] - note_names[note['name'].lower()] = note['path'] - - # Build graph structure with link detection - for note in notes: - if note.get('type') == 'note': - nodes.append({ - "id": note['path'], - "label": note['name'].replace('.md', '') - }) - - # Read note content to find links - content = get_note_content(config['storage']['notes_dir'], note['path']) - if content: - # Find wikilinks: [[target]] or [[target|display]] - wikilinks = re.findall(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', content) - - # Find standard markdown internal links: [text](path) - any local path (not http/https) - # Match links that don't start with http://, https://, mailto:, #, etc. - markdown_links = re.findall(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', content) - - # Get source note's folder for resolving relative links - # Use forward slashes consistently (note_paths uses forward slashes) - source_folder = str(Path(note['path']).parent).replace('\\', '/') - if source_folder == '.': - source_folder = '' - - # Process wikilinks - for target in wikilinks: - target = target.strip() - target_lower = target.lower() - - # Try to match target to an existing note - target_path = None - - # 1. Try resolving relative to source note's folder first - if source_folder and '/' not in target: - relative_path = f"{source_folder}/{target}" - relative_path_lower = relative_path.lower() - - if relative_path in note_paths: - target_path = relative_path if relative_path.endswith('.md') else relative_path + '.md' - elif relative_path + '.md' in note_paths: - target_path = relative_path + '.md' - elif relative_path_lower in note_paths_lower: - target_path = note_paths_lower[relative_path_lower] - elif relative_path_lower + '.md' in note_paths_lower: - target_path = note_paths_lower[relative_path_lower + '.md'] - - # 2. Exact path match (absolute or already has folder) - if not target_path: - if target in note_paths: - target_path = target if target.endswith('.md') else target + '.md' - elif target + '.md' in note_paths: - target_path = target + '.md' - # 3. Case-insensitive path match (e.g., [[Folder/Note]] -> folder/note.md) - elif target_lower in note_paths_lower: - target_path = note_paths_lower[target_lower] - elif target_lower + '.md' in note_paths_lower: - target_path = note_paths_lower[target_lower + '.md'] - # 4. Just note name (case-insensitive) - global match - elif target_lower in note_names: - target_path = note_names[target_lower] - - if target_path and target_path != note['path']: - edges.append({ - "source": note['path'], - "target": target_path, - "type": "wikilink" - }) - - # Process markdown links - for _, link_path in markdown_links: - # Skip anchor-only links and external protocols - if not link_path or link_path.startswith('#'): - continue - - # Remove anchor part if present (e.g., "note.md#section" -> "note.md") - link_path = link_path.split('#')[0] - if not link_path: - continue - - # Normalize path: remove ./ prefix, handle URL encoding - link_path = urllib.parse.unquote(link_path) - if link_path.startswith('./'): - link_path = link_path[2:] - - # Add .md extension if not present and doesn't have other extension - link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md' - - # Try to match target to an existing note - target_path = None - - # 1. First, try resolving relative to source note's folder - if source_folder and not link_path.startswith('/'): - relative_path = f"{source_folder}/{link_path}" - relative_path_with_md = f"{source_folder}/{link_path_with_md}" - relative_path_lower = relative_path.lower() - relative_path_with_md_lower = relative_path_with_md.lower() - - if relative_path in note_paths: - target_path = relative_path if relative_path.endswith('.md') else relative_path + '.md' - elif relative_path_with_md in note_paths: - target_path = relative_path_with_md - elif relative_path_lower in note_paths_lower: - target_path = note_paths_lower[relative_path_lower] - elif relative_path_with_md_lower in note_paths_lower: - target_path = note_paths_lower[relative_path_with_md_lower] - - # 2. Try exact path match from root (for absolute paths or notes at root) - if not target_path: - link_path_lower = link_path.lower() - link_path_with_md_lower = link_path_with_md.lower() - - if link_path in note_paths: - target_path = link_path if link_path.endswith('.md') else link_path + '.md' - elif link_path_with_md in note_paths: - target_path = link_path_with_md - # Case-insensitive path match - elif link_path_lower in note_paths_lower: - target_path = note_paths_lower[link_path_lower] - elif link_path_with_md_lower in note_paths_lower: - target_path = note_paths_lower[link_path_with_md_lower] - - # No global filename fallback for markdown links - they must resolve as paths - - if target_path and target_path != note['path']: - edges.append({ - "source": note['path'], - "target": target_path, - "type": "markdown" - }) - - # Remove duplicate edges - seen = set() - unique_edges = [] - for edge in edges: - key = (edge['source'], edge['target']) - if key not in seen: - seen.add(key) - unique_edges.append(edge) - - return {"nodes": nodes, "edges": unique_edges} + if not note_index.get_index().is_built(): + scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False) + nodes_paths, edges_tuples = note_index.get_graph_data() + return { + "nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths], + "edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples], + } except Exception as e: raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to generate graph data")) @@ -1692,6 +1558,19 @@ async def toggle_plugin(request: Request, plugin_name: str, enabled: dict): raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to toggle plugin")) +# ============================================================================ +# Index observability — internal counters & sizes for debugging +# +# Useful for confirming the index is doing what we expect (build_count >0, +# search_terms not empty, fingerprint_short_circuits incrementing on idle +# warm scans). Not rate-limited — cheap, no I/O, dict lookups only. +# ============================================================================ + +@api_router.get("/index/stats", tags=["System"]) +async def get_index_stats(): + return note_index.stats() + + # ============================================================================ # Stats Endpoint (for dashboards) # ============================================================================ @@ -1699,55 +1578,33 @@ async def toggle_plugin(request: Request, plugin_name: str, enabled: dict): @api_router.get("/stats", tags=["Stats"]) @limiter.limit("30/minute") async def get_stats(request: Request): - """ - Get application statistics at a glance. + """At-a-glance counts for dashboard widgets (Homepage etc.). - Designed for dashboard widgets (e.g., Homepage) - lightweight and cached. - Returns counts of notes, folders, tags, templates, media, and other metadata. - """ + All vault aggregates are read from the in-memory index — no file walk on + the request path. Templates / plugins / version are looked up directly.""" try: notes_dir = config['storage']['notes_dir'] - - # Get notes and folders (cached) - notes, folders = scan_notes_fast_walk(notes_dir, include_media=True) - - # Separate notes from media - note_items = [n for n in notes if n.get('type') == 'note'] - media_items = [n for n in notes if n.get('type') != 'note'] - - # Count unique tags - all_tags = set() - for note in note_items: - all_tags.update(note.get('tags', [])) - - # Get templates count - templates = get_templates(notes_dir) - - # Calculate total size - total_size = sum(n.get('size', 0) for n in notes) - - # Get last modified (notes are already sorted by modified desc) - last_modified = note_items[0].get('modified') if note_items else None - - # Count enabled plugins + ensure_index_built(notes_dir) + s = note_index.summary() + + templates_count = len(get_templates(notes_dir)) enabled_plugins = sum(1 for p in plugin_manager.plugins.values() if p.enabled) - - # Read version + version = "unknown" version_file = Path(__file__).parent.parent / "VERSION" if version_file.exists(): version = version_file.read_text().strip() - + return { - "notes_count": len(note_items), - "folders_count": len(folders), - "tags_count": len(all_tags), - "templates_count": len(templates), - "media_count": len(media_items), - "total_size_bytes": total_size, - "last_modified": last_modified, + "notes_count": s["notes_count"], + "folders_count": s["folders_count"], + "tags_count": s["tags_count"], + "templates_count": templates_count, + "media_count": s["media_count"], + "total_size_bytes": s["total_size_bytes"], + "last_modified": s["last_modified"], "plugins_enabled": enabled_plugins, - "version": version + "version": version, } except Exception as e: raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to get stats")) @@ -1985,6 +1842,26 @@ app.include_router(api_router) app.include_router(pages_router) +# ============================================================================ +# Startup warmup +# ============================================================================ +# Pre-build the note index off the request path. Mid-warmup requests are safe +# (bulk_set is serialized and short-circuits on the fingerprint). +# Success is logged from inside bulk_set so we get a single line for both +# the initial build and any subsequent rebuilds triggered by external changes. +@app.on_event("startup") +def _warmup_note_index() -> None: + import threading + + def _build() -> None: + try: + ensure_index_built(config['storage']['notes_dir']) + except Exception as exc: + logger.warning("Vault index rebuild failed (will retry on first request): %s", exc) + + threading.Thread(target=_build, name="note-index-warmup", daemon=True).start() + + if __name__ == "__main__": import uvicorn uvicorn.run( diff --git a/backend/note_index.py b/backend/note_index.py new file mode 100644 index 0000000..dbce629 --- /dev/null +++ b/backend/note_index.py @@ -0,0 +1,1016 @@ +""" +NoteIndex — in-memory index of vault state: notes, folders, tags, links, search. + +Backs /api/notes, /api/tags, /api/backlinks, /api/graph, /api/search. Keeps +everything thread-safe under one RLock. Process-memory only — rebuilds on the +first /api/notes request after each process start (~1-3s on a 10K-note vault). + +Updated incrementally on every save/delete/move/rename through the on_* +facades at the bottom of this file. +""" + +from __future__ import annotations + +import logging +import os +import re +import threading +import time +import urllib.parse +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple + +logger = logging.getLogger("uvicorn.error") + + +# Below this many markdown files, parallel tag extraction adds more overhead +# than it saves. +_PARALLEL_CUTOFF = 50 +_PARALLEL_WORKERS = min(8, (os.cpu_count() or 4)) + +# Search tokenization. Min length 2 keeps single-letter noise out without +# losing common short queries like "go", "ai". +_SEARCH_TOKEN_RE = re.compile(r"[A-Za-z0-9_\-]{2,}") +_SEARCH_MIN_QUERY_LEN = 2 + +WIKILINK_RE = re.compile(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]') +MDLINK_RE = re.compile(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)') + + +def extract_links_from_content(content: str) -> Dict[str, List[str]]: + """Pull raw wikilink targets and markdown-link paths out of note content.""" + wikilinks = [m.strip() for m in WIKILINK_RE.findall(content)] + mdlinks = [link_path for _, link_path in MDLINK_RE.findall(content)] + return {"wikilinks": wikilinks, "mdlinks": mdlinks} + + +def extract_search_terms(content: str) -> Set[str]: + """Tokenize content for the inverted search index.""" + return {m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(content)} + + +@dataclass +class NoteRecord: + """One note's metadata. No content, no resolved links.""" + path: str # vault-relative POSIX + name: str # stem (no extension) + folder: str # vault-relative POSIX, "" for root + modified: str # ISO timestamp + size: int # bytes + type: str # "note" | "image" | "audio" | ... + mtime: float # raw stat mtime + tags: Tuple[str, ...] = field(default_factory=tuple) + + +class NoteIndex: + """Thread-safe in-memory index of vault state. Reads return copies so + callers don't have to hold the lock.""" + + def __init__(self) -> None: + self._lock = threading.RLock() + + self._notes: Dict[str, NoteRecord] = {} # path -> record + self._folders: Set[str] = set() + + self._tags_forward: Dict[str, Tuple[str, ...]] = {} # path -> tags + self._tags_backward: Dict[str, Set[str]] = {} # tag -> paths + + self._raw_links: Dict[str, Dict[str, List[str]]] = {} # path -> {"wikilinks":[], "mdlinks":[]} + self._links_forward: Dict[str, Dict[str, str]] = {} # src -> {tgt: type} + self._links_backward: Dict[str, Set[str]] = {} # tgt -> {srcs} + self._wikilink_tokens: Dict[str, Set[str]] = {} # lower token -> {srcs} (loose match) + + self._search_terms: Dict[str, Set[str]] = {} # term -> {paths} + + # _search_built tracked separately — the search index is built lazily + # on the first /api/search call, after the cheaper notes/tags/links + # part is already built. + self._built = False + self._search_built = False + self._raw_fingerprint: Optional[int] = None # short-circuits no-op rebuilds + + self._stats = { + "build_count": 0, + "last_build_ms": 0.0, + "last_built_at": None, + "incremental_updates": 0, + "fingerprint_short_circuits": 0, + "search_build_count": 0, + "last_search_build_ms": 0.0, + } + + def is_built(self) -> bool: + with self._lock: + return self._built + + def invalidate(self) -> None: + """Mark as needing rebuild on next scan.""" + with self._lock: + self._built = False + self._raw_fingerprint = None + + def reset(self) -> None: + with self._lock: + self._notes.clear() + self._folders.clear() + self._tags_forward.clear() + self._tags_backward.clear() + self._raw_links.clear() + self._links_forward.clear() + self._links_backward.clear() + self._wikilink_tokens.clear() + self._search_terms.clear() + self._built = False + self._search_built = False + self._raw_fingerprint = None + + def is_search_built(self) -> bool: + with self._lock: + return self._search_built + + def ensure_search_index_built(self, notes_dir: str) -> bool: + """Build the search index on demand. Returns True when ready, False + only if the main index isn't built yet. File I/O happens OUTSIDE the + lock so other reads aren't blocked.""" + if self._search_built: + return True + + with self._lock: + if self._search_built: + return True + if not self._built: + return False + paths_to_read = [p for p, r in self._notes.items() if r.type == "note"] + + t0 = time.perf_counter() + base = Path(notes_dir) + terms_per_path: Dict[str, Set[str]] = {} + for rel in paths_to_read: + try: + with open(base / rel, "r", encoding="utf-8") as f: + terms_per_path[rel] = extract_search_terms(f.read()) + except Exception: + terms_per_path[rel] = set() + + with self._lock: + if self._search_built: + return True + self._search_terms.clear() + for path, terms in terms_per_path.items(): + for term in terms: + self._search_terms.setdefault(term, set()).add(path) + self._search_built = True + self._stats["search_build_count"] += 1 + self._stats["last_search_build_ms"] = (time.perf_counter() - t0) * 1000 + return True + + def bulk_set( + self, + notes_meta: List[NoteRecord], + folders: Iterable[str], + sources_raw: Dict[str, Dict[str, List[str]]], + ) -> None: + """Replace the entire index from a fresh scan. Short-circuits when + the input fingerprints to the same state we already hold (warm + scan, nothing changed).""" + new_fp = _fingerprint(notes_meta, sources_raw) + with self._lock: + if self._built and self._raw_fingerprint == new_fp: + self._stats["fingerprint_short_circuits"] += 1 + return + + t0 = time.perf_counter() + + self._notes = {n.path: n for n in notes_meta} + self._folders = set(folders) + self._raw_links = {k: v for k, v in sources_raw.items()} + + self._rebuild_tags_unlocked() + self._rebuild_links_unlocked() + if self._search_built: + self._prune_search_unlocked() + + self._raw_fingerprint = new_fp + self._built = True + + elapsed = time.perf_counter() - t0 + self._stats["build_count"] += 1 + self._stats["last_build_ms"] = elapsed * 1000 + self._stats["last_built_at"] = datetime.now(tz=timezone.utc).isoformat() + + logger.info("Vault index rebuilt in %.2fs (%d notes)", elapsed, len(notes_meta)) + + def update_note( + self, + record: NoteRecord, + raw_links: Dict[str, List[str]], + content: Optional[str] = None, + ) -> None: + """Patch the index in place after a note is created or saved.""" + with self._lock: + old_record = self._notes.get(record.path) + old_tags = old_record.tags if old_record else () + + self._notes[record.path] = record + if record.folder: + self._folders.add(record.folder) + + new_tags = record.tags + if old_tags != new_tags: + for t in set(old_tags) - set(new_tags): + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(record.path) + if not bucket: + del self._tags_backward[t] + for t in set(new_tags) - set(old_tags): + self._tags_backward.setdefault(t, set()).add(record.path) + self._tags_forward[record.path] = new_tags + + self._raw_links[record.path] = raw_links + self._resolve_single_source_unlocked(record.path) + + # Search index only gets patched if it's already been built — + # otherwise the first /api/search will build it from scratch. + if content is not None and self._search_built: + self._update_search_for_note_unlocked(record.path, content) + + self._raw_fingerprint = None + self._stats["incremental_updates"] += 1 + + def remove_note(self, path: str) -> None: + """A note was deleted. Drop everything that mentions it.""" + with self._lock: + old_record = self._notes.pop(path, None) + if old_record is None: + return + + # Tags + for t in old_record.tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._tags_backward[t] + self._tags_forward.pop(path, None) + + # Links: drop as source. + self._raw_links.pop(path, None) + old_targets = self._links_forward.pop(path, {}) + for t in old_targets: + bucket = self._links_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._links_backward[t] + + # Links: drop as loose wikilink source. + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + sources.discard(path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + # Links: drop as target from every other source's forward dict. + sources_pointing_here = self._links_backward.pop(path, set()) + for src in sources_pointing_here: + fwd = self._links_forward.get(src) + if fwd is not None and path in fwd: + del fwd[path] + if not fwd: + del self._links_forward[src] + + # Search: drop from every term bucket. Linear in distinct terms + # this note contributed to, which is bounded by note size. + empty_terms = [] + for term, paths in self._search_terms.items(): + if path in paths: + paths.discard(path) + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + + self._raw_fingerprint = None + self._stats["incremental_updates"] += 1 + + def rename_note(self, old_path: str, new_path: str) -> None: + """Move all references from old_path to new_path. The path change can + affect how other notes' wikilinks resolve, so we wipe the resolved + link indexes for this note and rely on the next bulk_set to rebuild.""" + if old_path == new_path: + return + with self._lock: + old_record = self._notes.pop(old_path, None) + if old_record is None: + return + new_record = NoteRecord( + path=new_path, + name=Path(new_path).stem, + folder=str(Path(new_path).parent).replace("\\", "/").lstrip(".").lstrip("/") or "", + modified=old_record.modified, + size=old_record.size, + type=old_record.type, + mtime=old_record.mtime, + tags=old_record.tags, + ) + # Re-derive folder cleanly (the above lstrip dance is brittle). + folder = str(Path(new_path).parent).replace("\\", "/") + new_record.folder = "" if folder == "." else folder + self._notes[new_path] = new_record + if new_record.folder: + self._folders.add(new_record.folder) + + if old_path in self._tags_forward: + tags = self._tags_forward.pop(old_path) + self._tags_forward[new_path] = tags + for t in tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(old_path) + bucket.add(new_path) + + for paths in self._search_terms.values(): + if old_path in paths: + paths.discard(old_path) + paths.add(new_path) + + if old_path in self._raw_links: + self._raw_links[new_path] = self._raw_links.pop(old_path) + + # Other sources that linked to old_path by stem name may now + # resolve to new_path (or not), so we wipe both forward & backward + # for old_path and let the next bulk_set re-resolve from scratch. + self._links_forward.pop(old_path, None) + for bucket in self._links_backward.values(): + bucket.discard(old_path) + old_backlinks = self._links_backward.pop(old_path, set()) + for src in old_backlinks: + fwd = self._links_forward.get(src) + if fwd is not None and old_path in fwd: + del fwd[old_path] + if not fwd: + del self._links_forward[src] + + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + if old_path in sources: + sources.discard(old_path) + sources.add(new_path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + self._raw_fingerprint = None + self._built = False # force re-resolve on next bulk_set + self._stats["incremental_updates"] += 1 + + def rename_folder_prefix(self, old_prefix: str, new_prefix: str) -> None: + """Migrate every entry under `old_prefix/` to `new_prefix/`. Much + cheaper than a full rebuild — microseconds of key swaps.""" + old_prefix = old_prefix.rstrip("/") + new_prefix = new_prefix.rstrip("/") + if old_prefix == new_prefix: + return + with self._lock: + affected_paths = [p for p in self._notes if p == old_prefix or p.startswith(old_prefix + "/")] + for old_path in affected_paths: + suffix = old_path[len(old_prefix):] + self._rename_note_unlocked(old_path, new_prefix + suffix) + + folders_to_rename = [ + f for f in self._folders if f == old_prefix or f.startswith(old_prefix + "/") + ] + for f in folders_to_rename: + self._folders.discard(f) + suffix = f[len(old_prefix):] + self._folders.add(new_prefix + suffix) + + self._raw_fingerprint = None + self._built = False + + def remove_folder_prefix(self, prefix: str) -> None: + """Drop every entry under `prefix/`.""" + prefix = prefix.rstrip("/") + with self._lock: + affected = [p for p in self._notes if p == prefix or p.startswith(prefix + "/")] + for path in affected: + self._remove_note_unlocked(path) + folders_to_drop = [ + f for f in self._folders if f == prefix or f.startswith(prefix + "/") + ] + for f in folders_to_drop: + self._folders.discard(f) + self._raw_fingerprint = None + + # ------------------------------------------------------------------ + # Read API — every method returns a snapshot copy + # ------------------------------------------------------------------ + + def get_backlink_candidate_sources(self, target_path: str) -> Set[str]: + """Superset of true backlink sources. Combines strict resolved + backward links with the loose wikilink-token reverse index. Caller + runs the per-line matcher against each candidate to filter.""" + target_lower = target_path.lower() + target_no_ext_lower = target_lower[:-3] if target_lower.endswith(".md") else target_lower + target_name = Path(target_path).stem.lower() + + with self._lock: + candidates: Set[str] = set(self._links_backward.get(target_path, set())) + for key in (target_lower, target_no_ext_lower, target_name): + candidates.update(self._wikilink_tokens.get(key, set())) + candidates.discard(target_path) + return candidates + + def get_graph_data(self) -> Tuple[List[str], List[Tuple[str, str, str]]]: + """Snapshot of (sorted note paths, (source, target, type) edges).""" + with self._lock: + nodes = sorted(p for p, r in self._notes.items() if r.type == "note") + edges: List[Tuple[str, str, str]] = [] + for src, targets in self._links_forward.items(): + for target, edge_type in targets.items(): + edges.append((src, target, edge_type)) + return nodes, edges + + def get_all_tags(self) -> Dict[str, int]: + """Snapshot {tag: count}, sorted by tag name.""" + with self._lock: + return {tag: len(paths) for tag, paths in sorted(self._tags_backward.items())} + + def get_paths_for_tag(self, tag: str) -> Set[str]: + """Snapshot set of paths tagged with `tag` (case-insensitive).""" + with self._lock: + return set(self._tags_backward.get(tag.lower(), set())) + + def get_note_record(self, path: str) -> Optional[NoteRecord]: + with self._lock: + return self._notes.get(path) + + def all_note_records(self) -> List[Tuple[str, NoteRecord]]: + """Snapshot list of (path, record) for every indexed markdown note.""" + with self._lock: + return [(p, r) for p, r in self._notes.items() if r.type == "note"] + + def try_get_extraction( + self, + rel_path: str, + mtime: float, + ) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: + """Return (tags, raw_links) from the index only if the recorded mtime + matches `mtime` exactly. Lets scan_notes_fast_walk skip the per-file + read on a warm scan.""" + with self._lock: + rec = self._notes.get(rel_path) + if rec is None or rec.mtime != mtime: + return None + raw = self._raw_links.get(rel_path) + if raw is None: + return None + return ( + list(rec.tags), + { + "wikilinks": list(raw.get("wikilinks", [])), + "mdlinks": list(raw.get("mdlinks", [])), + }, + ) + + def get_search_candidates(self, query: str) -> Optional[Set[str]]: + """Superset of paths whose content COULD contain `query`. Caller + still runs the substring match per candidate for confirmation + + snippet extraction. Returns None when the query is too short or + tokenizes to nothing — caller should iterate every note instead.""" + if not self._search_built: + return None + if len(query) < _SEARCH_MIN_QUERY_LEN: + return None + tokens = [m.group(0).lower() for m in _SEARCH_TOKEN_RE.finditer(query)] + if not tokens: + return None + with self._lock: + candidate = self._search_terms.get(tokens[0]) + if candidate is None: + return set() + result: Set[str] = set(candidate) + for tok in tokens[1:]: + bucket = self._search_terms.get(tok) + if bucket is None: + return set() + result &= bucket + if not result: + break + return result + + def summary(self) -> Dict[str, Any]: + """Aggregate counts + total size + last-modified note. Powers /api/stats + without a vault scan — every field is computed from records already in + memory.""" + with self._lock: + notes_count = 0 + media_count = 0 + total_size = 0 + last_modified: Optional[str] = None + last_mtime = -1.0 + for rec in self._notes.values(): + total_size += rec.size + if rec.type == "note": + notes_count += 1 + if rec.mtime > last_mtime: + last_mtime = rec.mtime + last_modified = rec.modified + else: + media_count += 1 + return { + "notes_count": notes_count, + "media_count": media_count, + "folders_count": len(self._folders), + "tags_count": len(self._tags_backward), + "total_size_bytes": total_size, + "last_modified": last_modified, + } + + def stats(self) -> Dict[str, Any]: + """Snapshot of counters + size metrics.""" + with self._lock: + return { + "built": self._built, + "search_built": self._search_built, + "notes": len(self._notes), + "folders": len(self._folders), + "tags": len(self._tags_backward), + "links_forward_entries": len(self._links_forward), + "links_backward_entries": len(self._links_backward), + "wikilink_tokens": len(self._wikilink_tokens), + "search_terms": len(self._search_terms), + "counters": dict(self._stats), + } + + # ================================================================== + # Internal — must hold _lock when called + # ================================================================== + + def _rebuild_tags_unlocked(self) -> None: + self._tags_forward.clear() + self._tags_backward.clear() + for path, record in self._notes.items(): + if record.type != "note": + continue + tags = record.tags + if not tags: + continue + self._tags_forward[path] = tags + for tag in tags: + self._tags_backward.setdefault(tag, set()).add(path) + + def _rebuild_links_unlocked(self) -> None: + """Full link re-resolution. Reuses one _Resolver across all sources + (O(N+K*L) instead of O(N*K)).""" + self._links_forward.clear() + self._links_backward.clear() + self._wikilink_tokens.clear() + note_paths = {p for p, r in self._notes.items() if r.type == "note"} + resolver = _Resolver(note_paths) + for source_path in self._raw_links: + self._resolve_single_source_unlocked(source_path, resolver=resolver, skip_cleanup=True) + + def _prune_search_unlocked(self) -> None: + """Drop search-term entries for paths no longer in the index.""" + live_paths = set(self._notes.keys()) + empty_terms = [] + for term, paths in self._search_terms.items(): + stale = paths - live_paths + if stale: + paths -= stale + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + + def _update_search_for_note_unlocked(self, path: str, content: str) -> None: + """Replace one note's terms in the inverted index.""" + empty_terms = [] + for term, paths in self._search_terms.items(): + if path in paths: + paths.discard(path) + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + for term in extract_search_terms(content): + self._search_terms.setdefault(term, set()).add(path) + + def _resolve_single_source_unlocked( + self, + source_path: str, + resolver: Optional["_Resolver"] = None, + skip_cleanup: bool = False, + ) -> None: + if not skip_cleanup: + old_targets = self._links_forward.pop(source_path, {}) + for t in old_targets: + bucket = self._links_backward.get(t) + if bucket is not None: + bucket.discard(source_path) + if not bucket: + del self._links_backward[t] + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + sources.discard(source_path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + raw = self._raw_links.get(source_path) + if not raw: + return + + if resolver is None: + note_paths = {p for p, r in self._notes.items() if r.type == "note"} + resolver = _Resolver(note_paths) + + source_folder = str(Path(source_path).parent).replace("\\", "/") + if source_folder == ".": + source_folder = "" + + targets: Dict[str, str] = {} + + # Wikilinks first (they win the "first wins" dedup that /api/graph uses). + for target in raw.get("wikilinks", []): + resolved = resolver.resolve_wikilink(target, source_folder) + if resolved and resolved != source_path and resolved not in targets: + targets[resolved] = "wikilink" + # Loose token index — populated even when strict resolution fails, + # because backlink matching uses stem comparison independently. + t_lower = target.strip().lower() + if t_lower: + self._wikilink_tokens.setdefault(t_lower, set()).add(source_path) + t_no_ext = t_lower[:-3] if t_lower.endswith(".md") else t_lower + if t_no_ext != t_lower: + self._wikilink_tokens.setdefault(t_no_ext, set()).add(source_path) + + for link_path in raw.get("mdlinks", []): + resolved = resolver.resolve_mdlink(link_path, source_folder) + if resolved and resolved != source_path and resolved not in targets: + targets[resolved] = "markdown" + + if targets: + self._links_forward[source_path] = targets + for t in targets: + self._links_backward.setdefault(t, set()).add(source_path) + + def _rename_note_unlocked(self, old_path: str, new_path: str) -> None: + """rename_note() body without acquiring the lock — used by + rename_folder_prefix to avoid re-locking per file.""" + if old_path == new_path: + return + old_record = self._notes.pop(old_path, None) + if old_record is None: + return + folder = str(Path(new_path).parent).replace("\\", "/") + new_record = NoteRecord( + path=new_path, + name=Path(new_path).stem, + folder="" if folder == "." else folder, + modified=old_record.modified, + size=old_record.size, + type=old_record.type, + mtime=old_record.mtime, + tags=old_record.tags, + ) + self._notes[new_path] = new_record + if new_record.folder: + self._folders.add(new_record.folder) + + if old_path in self._tags_forward: + tags = self._tags_forward.pop(old_path) + self._tags_forward[new_path] = tags + for t in tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(old_path) + bucket.add(new_path) + + for paths in self._search_terms.values(): + if old_path in paths: + paths.discard(old_path) + paths.add(new_path) + + if old_path in self._raw_links: + self._raw_links[new_path] = self._raw_links.pop(old_path) + + self._links_forward.pop(old_path, None) + for bucket in self._links_backward.values(): + bucket.discard(old_path) + old_backlinks = self._links_backward.pop(old_path, set()) + for src in old_backlinks: + fwd = self._links_forward.get(src) + if fwd is not None and old_path in fwd: + del fwd[old_path] + if not fwd: + del self._links_forward[src] + + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + if old_path in sources: + sources.discard(old_path) + sources.add(new_path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + + def _remove_note_unlocked(self, path: str) -> None: + """Same as remove_note() but assumes the caller already holds the lock.""" + old_record = self._notes.pop(path, None) + if old_record is None: + return + for t in old_record.tags: + bucket = self._tags_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._tags_backward[t] + self._tags_forward.pop(path, None) + self._raw_links.pop(path, None) + old_targets = self._links_forward.pop(path, {}) + for t in old_targets: + bucket = self._links_backward.get(t) + if bucket is not None: + bucket.discard(path) + if not bucket: + del self._links_backward[t] + empty_keys = [] + for key, sources in self._wikilink_tokens.items(): + sources.discard(path) + if not sources: + empty_keys.append(key) + for k in empty_keys: + del self._wikilink_tokens[k] + sources_pointing_here = self._links_backward.pop(path, set()) + for src in sources_pointing_here: + fwd = self._links_forward.get(src) + if fwd is not None and path in fwd: + del fwd[path] + if not fwd: + del self._links_forward[src] + empty_terms = [] + for term, paths in self._search_terms.items(): + if path in paths: + paths.discard(path) + if not paths: + empty_terms.append(term) + for term in empty_terms: + del self._search_terms[term] + + +# ============================================================================ +# Resolver — link-target matching (mirrors legacy /api/graph rules exactly) +# ============================================================================ + +class _Resolver: + """Link-target lookup tables. Build once per resolution batch, then call + resolve_* repeatedly across many sources.""" + + def __init__(self, all_notes: Set[str]) -> None: + self.note_paths: Set[str] = set(all_notes) + self.note_paths_lower: Dict[str, str] = {} + self.note_names: Dict[str, str] = {} + for p in all_notes: + self.note_paths_lower[p.lower()] = p + if p.endswith(".md"): + self.note_paths_lower[p[:-3].lower()] = p + stem = Path(p).stem + self.note_names[stem.lower()] = p + self.note_names[Path(p).name.lower()] = p + + def resolve_wikilink(self, target: str, source_folder: str) -> Optional[str]: + target = target.strip() + if not target: + return None + target_lower = target.lower() + + # 1. Relative to source folder (only for bare names with no slash). + if source_folder and "/" not in target: + relative_path = f"{source_folder}/{target}" + relative_path_lower = relative_path.lower() + if relative_path in self.note_paths: + return relative_path if relative_path.endswith(".md") else relative_path + ".md" + if relative_path + ".md" in self.note_paths: + return relative_path + ".md" + if relative_path_lower in self.note_paths_lower: + return self.note_paths_lower[relative_path_lower] + if (relative_path_lower + ".md") in self.note_paths_lower: + return self.note_paths_lower[relative_path_lower + ".md"] + + if target in self.note_paths: + return target if target.endswith(".md") else target + ".md" + if (target + ".md") in self.note_paths: + return target + ".md" + if target_lower in self.note_paths_lower: + return self.note_paths_lower[target_lower] + if (target_lower + ".md") in self.note_paths_lower: + return self.note_paths_lower[target_lower + ".md"] + if target_lower in self.note_names: + return self.note_names[target_lower] + return None + + def resolve_mdlink(self, link_path: str, source_folder: str) -> Optional[str]: + if not link_path: + return None + link_path = link_path.split("#")[0] + if not link_path: + return None + link_path = urllib.parse.unquote(link_path) + if link_path.startswith("./"): + link_path = link_path[2:] + link_path_with_md = link_path if link_path.endswith(".md") else link_path + ".md" + + if source_folder and not link_path.startswith("/"): + relative_path = f"{source_folder}/{link_path}" + relative_path_with_md = f"{source_folder}/{link_path_with_md}" + relative_path_lower = relative_path.lower() + relative_path_with_md_lower = relative_path_with_md.lower() + if relative_path in self.note_paths: + return relative_path if relative_path.endswith(".md") else relative_path + ".md" + if relative_path_with_md in self.note_paths: + return relative_path_with_md + if relative_path_lower in self.note_paths_lower: + return self.note_paths_lower[relative_path_lower] + if relative_path_with_md_lower in self.note_paths_lower: + return self.note_paths_lower[relative_path_with_md_lower] + + link_path_lower = link_path.lower() + link_path_with_md_lower = link_path_with_md.lower() + if link_path in self.note_paths: + return link_path if link_path.endswith(".md") else link_path + ".md" + if link_path_with_md in self.note_paths: + return link_path_with_md + if link_path_lower in self.note_paths_lower: + return self.note_paths_lower[link_path_lower] + if link_path_with_md_lower in self.note_paths_lower: + return self.note_paths_lower[link_path_with_md_lower] + return None + + +# ============================================================================ +# Internal helpers +# ============================================================================ + +def _fingerprint( + notes_meta: List[NoteRecord], + sources_raw: Dict[str, Dict[str, List[str]]], +) -> int: + """Cheap hash of a scan result. Short-circuits bulk_set when unchanged.""" + notes_fp = hash(frozenset((n.path, n.mtime) for n in notes_meta)) + raw_items = ( + (src, tuple(raw.get("wikilinks", [])), tuple(raw.get("mdlinks", []))) + for src, raw in sources_raw.items() + ) + return hash((notes_fp, hash(frozenset(raw_items)))) + + +# ============================================================================ +# Module singleton + facade. Callers in utils.py / main.py go through these. +# ============================================================================ + +_index = NoteIndex() + + +def get_index() -> NoteIndex: + return _index + + +# --- Lifecycle hooks (one-line calls from utils.py mutators) ---------------- + +def on_note_saved(notes_dir: str, full_path: Path, content: str) -> None: + """A note was created or updated on disk.""" + try: + rel_path = full_path.relative_to(Path(notes_dir)).as_posix() + st = full_path.stat() + folder = str(Path(rel_path).parent).replace("\\", "/") + record = NoteRecord( + path=rel_path, + name=Path(rel_path).stem, + folder="" if folder == "." else folder, + modified=datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), + size=st.st_size, + type="note", + mtime=st.st_mtime, + tags=tuple(_parse_tags_for_record(content)), + ) + _index.update_note(record, extract_links_from_content(content), content=content) + except Exception as e: + logger.error("note_index: on_note_saved failed for %s: %s", full_path, e) + + +def on_note_deleted(notes_dir: str, full_path: Path) -> None: + try: + rel_path = full_path.relative_to(Path(notes_dir)).as_posix() + _index.remove_note(rel_path) + except Exception as e: + logger.error("note_index: on_note_deleted failed for %s: %s", full_path, e) + + +def on_note_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: + try: + base = Path(notes_dir) + _index.rename_note( + old_full_path.relative_to(base).as_posix(), + new_full_path.relative_to(base).as_posix(), + ) + except Exception as e: + logger.error("note_index: on_note_renamed failed: %s", e) + + +def on_folder_renamed(notes_dir: str, old_full_path: Path, new_full_path: Path) -> None: + """Re-key every entry under the folder. No disk reads.""" + try: + base = Path(notes_dir) + _index.rename_folder_prefix( + old_full_path.relative_to(base).as_posix(), + new_full_path.relative_to(base).as_posix(), + ) + except Exception as e: + logger.error("note_index: on_folder_renamed failed: %s", e) + _index.invalidate() + + +def on_folder_deleted(notes_dir: str, full_path: Path) -> None: + try: + rel_prefix = full_path.relative_to(Path(notes_dir)).as_posix() + _index.remove_folder_prefix(rel_prefix) + except Exception as e: + logger.error("note_index: on_folder_deleted failed: %s", e) + _index.invalidate() + + +def populate_from_scan( + notes_meta: List[NoteRecord], + folders: Iterable[str], + sources_raw: Dict[str, Dict[str, List[str]]], +) -> None: + """Bulk-replace the index after a full scan_notes_fast_walk.""" + try: + _index.bulk_set(notes_meta, folders, sources_raw) + except Exception as e: + logger.error("note_index: populate_from_scan failed: %s", e) + + +def ensure_search_index(notes_dir: str) -> bool: + """Lazy-build the search index on first /api/search. Returns False only + if the main index isn't built yet.""" + if not _index.is_built(): + return False + return _index.ensure_search_index_built(notes_dir) + + +# --- Read facade ------------------------------------------------------------- + +def get_backlink_candidates(target_path: str) -> Set[str]: + return _index.get_backlink_candidate_sources(target_path) + + +def get_graph_data() -> Tuple[List[str], List[Tuple[str, str, str]]]: + return _index.get_graph_data() + + +def get_search_candidates(query: str) -> Optional[Set[str]]: + """Returns None when query is too short / untokenizable — caller iterates + every indexed note instead.""" + return _index.get_search_candidates(query) + + +def get_all_tags() -> Dict[str, int]: + return _index.get_all_tags() + + +def get_paths_for_tag(tag: str) -> Set[str]: + return _index.get_paths_for_tag(tag) + + +def try_get_extraction( + rel_path: str, + mtime: float, +) -> Optional[Tuple[List[str], Dict[str, List[str]]]]: + """(tags, raw_links) from the index when mtime matches, else None so the + caller reads the file. Used by scan_notes_fast_walk.""" + return _index.try_get_extraction(rel_path, mtime) + + +def summary() -> Dict[str, Any]: + return _index.summary() + + +def stats() -> Dict[str, Any]: + return _index.stats() + + +# Late import — utils imports this module, so we delay the reverse direction. +def _parse_tags_for_record(content: str) -> List[str]: + from .utils import parse_tags + return parse_tags(content) diff --git a/backend/plugins.py b/backend/plugins.py index 73a8a03..3f9cd91 100644 --- a/backend/plugins.py +++ b/backend/plugins.py @@ -5,10 +5,13 @@ Plugins can hook into events like note save, delete, etc. import os import json +import logging import importlib.util from pathlib import Path from typing import List, Dict, Callable +logger = logging.getLogger("uvicorn.error") + class Plugin: """Base plugin class""" @@ -113,7 +116,7 @@ class PluginManager: plugin = module.Plugin() self.plugins[plugin_file.stem] = plugin except Exception as e: - print(f"Failed to load plugin {plugin_file.stem}: {e}") + logger.error("Failed to load plugin %s: %s", plugin_file.stem, e) def _create_example_plugin(self): """Create an example plugin to show developers how to build plugins""" @@ -167,7 +170,7 @@ class Plugin: with open(self.config_file, 'r', encoding='utf-8') as f: return json.load(f) except Exception as e: - print(f"Failed to load plugin config: {e}") + logger.error("Failed to load plugin config: %s", e) return {} def _save_config(self): @@ -180,7 +183,7 @@ class Plugin: with open(self.config_file, 'w', encoding='utf-8') as f: json.dump(config, f, indent=2) except Exception as e: - print(f"Failed to save plugin config: {e}") + logger.error("Failed to save plugin config: %s", e) def _apply_saved_state(self): """Apply saved plugin states after loading plugins""" @@ -188,7 +191,7 @@ class Plugin: for plugin_id, enabled in saved_config.items(): if plugin_id in self.plugins: self.plugins[plugin_id].enabled = enabled - print(f"Plugin '{plugin_id}': {'enabled' if enabled else 'disabled'} (from config)") + logger.info("Plugin '%s': %s (from config)", plugin_id, 'enabled' if enabled else 'disabled') def enable_plugin(self, plugin_id: str): """Enable a plugin and persist the state""" @@ -238,7 +241,7 @@ class Plugin: method(**kwargs) except Exception as e: - print(f"Plugin {plugin.name} error in {hook_name}: {e}") + logger.error("Plugin %s error in %s: %s", plugin.name, hook_name, e) return result if 'content' in kwargs else None @@ -266,7 +269,7 @@ class Plugin: if 'initial_content' in kwargs and result is not None: kwargs['initial_content'] = result except Exception as e: - print(f"Plugin {plugin.name} error in {hook_name}: {e}") + logger.error("Plugin %s error in %s: %s", plugin.name, hook_name, e) # Return the final modified value return kwargs.get('initial_content', '') diff --git a/backend/share.py b/backend/share.py index 01b79ff..1d0fde8 100644 --- a/backend/share.py +++ b/backend/share.py @@ -4,6 +4,7 @@ Handles creating, storing, and revoking share tokens for public note access. """ import json +import logging import secrets import string from pathlib import Path @@ -13,6 +14,8 @@ import threading from .utils import validate_path_security +logger = logging.getLogger("uvicorn.error") + # Thread lock for safe concurrent access _lock = threading.Lock() @@ -99,7 +102,7 @@ def save_tokens(data_dir: str, tokens: Dict[str, Dict[str, Any]]) -> bool: json.dump(tokens, f, indent=2, ensure_ascii=False) return True except IOError as e: - print(f"Failed to save share tokens: {e}") + logger.error("Failed to save share tokens: %s", e) return False diff --git a/backend/themes.py b/backend/themes.py index 32cd31b..8945393 100644 --- a/backend/themes.py +++ b/backend/themes.py @@ -2,10 +2,13 @@ Theme management for NoteDiscovery """ +import logging from pathlib import Path from typing import List, Dict import re +logger = logging.getLogger("uvicorn.error") + def parse_theme_metadata(theme_path: Path) -> Dict[str, str]: """Parse theme metadata from CSS file comments""" @@ -28,7 +31,7 @@ def parse_theme_metadata(theme_path: Path) -> Dict[str, str]: metadata["type"] = match.group(1) break except Exception as e: - print(f"Error parsing theme metadata from {theme_path}: {e}") + logger.error("Error parsing theme metadata from %s: %s", theme_path, e) return metadata diff --git a/backend/utils.py b/backend/utils.py index 795aaf0..b7e607d 100644 --- a/backend/utils.py +++ b/backend/utils.py @@ -1,18 +1,29 @@ """ -Utility functions for file operations, search, and markdown processing +Utility functions for file operations, search, and markdown processing. + +Heavy work (backlinks, graph, full-text search, tag aggregation) is delegated +to the in-memory index in note_index.py. """ +import logging import os import re import shutil import threading import time +import traceback import urllib.parse +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path from typing import List, Dict, Optional, Tuple, Any, TypeVar, Callable from datetime import datetime, timezone +from . import note_index +from .note_index import NoteRecord, extract_links_from_content + +logger = logging.getLogger("uvicorn.error") + # ============================================================================ # Pagination Support @@ -111,14 +122,23 @@ def paginate( ) -# In-memory cache for parsed tags -# Format: {file_path: (mtime, tags)} -_tag_cache: Dict[str, Tuple[float, List[str]]] = {} - -# Notes tree scan cache (TTL). +# ============================================================================ +# In-memory caches # -# This avoids repeated full-directory walks when multiple endpoints (or the UI) -# request indexes in quick succession. +# Two layers: +# _tag_cache / _links_cache : per-file, mtime-keyed. Avoid re-parsing the +# same file on warm scans. Used by get_tags_and_links_cached. +# _SCAN_WALK_CACHE : per-scan TTL cache. Avoids repeated full- +# directory walks when several endpoints fire in quick succession. +# +# All three are invalidated on every mutation (save/delete/move). The +# note_index in note_index.py is the system of record for derived data +# (links, backlinks, tags-by-name, search) — these caches just exist to +# avoid double-reading files we already parsed. +# ============================================================================ + +_tag_cache: Dict[str, Tuple[float, List[str]]] = {} +_links_cache: Dict[str, Tuple[float, Dict[str, List[str]]]] = {} _SCAN_WALK_CACHE_LOCK = threading.Lock() _SCAN_WALK_CACHE_TTL_SECONDS = 1.0 @@ -144,6 +164,63 @@ def _scan_cache_set(key: Tuple[str, bool], value: Tuple[List[Dict], List[str]]) _SCAN_WALK_CACHE[key] = (time.monotonic(), value) +def _scan_cache_invalidate() -> None: + """Drop the TTL scan cache. Called from every mutation handler.""" + with _SCAN_WALK_CACHE_LOCK: + _SCAN_WALK_CACHE.clear() + + +def ensure_index_built(notes_dir: str) -> None: + """Trigger a fresh scan when the NoteIndex hasn't been populated yet. + Scans with include_media=True so every index consumer (stats, notes, + backlinks, search) shares one fingerprint and never thrashes.""" + if not note_index.get_index().is_built(): + scan_notes_fast_walk(notes_dir, use_cache=False, include_media=True) + + +def get_tags_and_links_cached( + file_path: Path, + rel_path: Optional[str] = None, +) -> Tuple[List[str], Dict[str, List[str]]]: + """Fused tags + raw-links extraction, mtime-cached. Tries in-process + per-file caches first, then the NoteIndex (when rel_path is provided), + then reads the file.""" + try: + mtime = file_path.stat().st_mtime + file_key = str(file_path) + + tag_cached = _tag_cache.get(file_key) + link_cached = _links_cache.get(file_key) + tags_ok = tag_cached is not None and tag_cached[0] == mtime + links_ok = link_cached is not None and link_cached[0] == mtime + + if tags_ok and links_ok: + return tag_cached[1], link_cached[1] + + if rel_path is not None: + indexed = note_index.try_get_extraction(rel_path, mtime) + if indexed is not None: + idx_tags, idx_links = indexed + _tag_cache[file_key] = (mtime, idx_tags) + _links_cache[file_key] = (mtime, idx_links) + return idx_tags, idx_links + + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + tags = tag_cached[1] if tags_ok else parse_tags(content) + links = link_cached[1] if links_ok else extract_links_from_content(content) + + if not tags_ok: + _tag_cache[file_key] = (mtime, tags) + if not links_ok: + _links_cache[file_key] = (mtime, links) + + return tags, links + except Exception: + return [], {"wikilinks": [], "mdlinks": []} + + def validate_path_security(notes_dir: str, path: Path) -> bool: """ Validate that a path is within the notes directory (security check). @@ -175,15 +252,12 @@ def ensure_directories(config: dict): def create_folder(notes_dir: str, folder_path: str) -> bool: - """Create a new folder in the notes directory""" + """Create a new folder in the notes directory.""" full_path = Path(notes_dir) / folder_path - - # Security check if not validate_path_security(notes_dir, full_path): return False - full_path.mkdir(parents=True, exist_ok=True) - + _scan_cache_invalidate() return True @@ -218,11 +292,13 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: _scan_cache_set(cache_key, normalized_value) return normalized_value + # Walk the vault once. Tag/link extraction is parallelized below across + # the markdown files we collected here. notes: List[Dict] = [] folders_set = set() + md_to_extract: List[Tuple[int, Path, str]] = [] # (index, full_path, rel_path) for root, dirnames, filenames in os.walk(notes_path): - # Skip descending into dot-dirs dirnames[:] = [d for d in dirnames if not d.startswith('.')] root_path = Path(root) @@ -244,22 +320,61 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: media_type = get_media_type(filename) if include_media else None is_markdown = full_path.suffix.lower() == '.md' should_include = is_markdown or (include_media and media_type is not None) - if not should_include: continue folder = relative_path.parent.as_posix() - # Get tags for this note (cached) - tags = get_tags_cached(full_path) if is_markdown else [] + rel_str = relative_path.as_posix() notes.append({ "name": full_path.stem, - "path": relative_path.as_posix(), + "path": rel_str, "folder": "" if folder == "." else folder, "modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), "size": st.st_size, "type": media_type if media_type else "note", - "tags": tags, + "tags": [], + "_mtime": st.st_mtime, # internal — popped before returning }) + if is_markdown: + md_to_extract.append((len(notes) - 1, full_path, rel_str)) + + # Tag + raw-link extraction in parallel. Search-index terms are built + # later, lazily, on the first /api/search call. + sources_raw: Dict[str, Dict[str, List[str]]] = {} + if md_to_extract: + path_pairs = [(full, rel) for (_, full, rel) in md_to_extract] + if len(md_to_extract) >= 50: + workers = min(8, (os.cpu_count() or 4)) + with ThreadPoolExecutor(max_workers=workers) as ex: + extraction = list(ex.map( + lambda pair: get_tags_and_links_cached(pair[0], pair[1]), + path_pairs, + )) + else: + extraction = [get_tags_and_links_cached(full, rel) for (full, rel) in path_pairs] + + for (idx, _full, rel_str), (tags, links) in zip(md_to_extract, extraction): + notes[idx]["tags"] = tags + sources_raw[rel_str] = links + + # Push the result into the NoteIndex. Short-circuits on warm scans. + notes_meta = [ + NoteRecord( + path=n["path"], + name=n["name"], + folder=n["folder"], + modified=n["modified"], + size=n["size"], + type=n["type"], + mtime=n["_mtime"], + tags=tuple(n["tags"]), + ) + for n in notes + ] + note_index.populate_from_scan(notes_meta, folders_set, sources_raw) + + for n in notes: + n.pop("_mtime", None) value = (sorted(notes, key=lambda x: x.get('modified', ''), reverse=True), sorted(folders_set)) if use_cache: @@ -267,132 +382,106 @@ def scan_notes_fast_walk(notes_dir: str, use_cache: bool = True, include_media: return value def move_note(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]: - """Move a note to a different location - - Returns: - Tuple of (success: bool, error_message: str) - """ + """Move a note. Returns (success, error_message).""" old_full_path = Path(notes_dir) / old_path new_full_path = Path(notes_dir) / new_path - - # Security checks + if not validate_path_security(notes_dir, old_full_path): return False, "Invalid source path" if not validate_path_security(notes_dir, new_full_path): return False, "Invalid destination path" - if not old_full_path.exists(): return False, f"Source note does not exist: {old_path}" - - # Check if target already exists (prevent overwriting) if new_full_path.exists(): return False, f"A note already exists at: {new_path}" - - # Invalidate cache for old path - old_key = str(old_full_path) - if old_key in _tag_cache: - del _tag_cache[old_key] - + + _drop_path_caches(old_full_path) + try: - # Create parent directory if needed new_full_path.parent.mkdir(parents=True, exist_ok=True) - - # Move the file old_full_path.rename(new_full_path) except Exception as e: return False, f"Failed to move file: {str(e)}" - - # Note: We don't automatically delete empty folders to preserve user's folder structure - + + note_index.on_note_renamed(notes_dir, old_full_path, new_full_path) + _scan_cache_invalidate() return True, "" def move_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]: - """Move a folder to a different location - - Returns: - Tuple of (success: bool, error_message: str) - """ - import shutil - + """Move a folder. Returns (success, error_message).""" old_full_path = Path(notes_dir) / old_path new_full_path = Path(notes_dir) / new_path - - # Security checks + if not validate_path_security(notes_dir, old_full_path): return False, "Invalid source path" if not validate_path_security(notes_dir, new_full_path): return False, "Invalid destination path" - if not old_full_path.exists() or not old_full_path.is_dir(): return False, f"Source folder does not exist: {old_path}" - - # Check if target already exists if new_full_path.exists(): return False, f"A folder already exists at: {new_path}" - - # Invalidate cache for all notes in this folder - global _tag_cache - old_path_str = str(old_full_path) - keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(old_path_str)] - for key in keys_to_delete: - del _tag_cache[key] - + + _drop_prefix_caches(old_full_path) + try: - # Create parent directory if needed new_full_path.parent.mkdir(parents=True, exist_ok=True) - - # Move the folder shutil.move(str(old_full_path), str(new_full_path)) except Exception as e: return False, f"Failed to move folder: {str(e)}" - - # Note: We don't automatically delete empty folders to preserve user's folder structure - + + note_index.on_folder_renamed(notes_dir, old_full_path, new_full_path) + _scan_cache_invalidate() return True, "" def rename_folder(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]: - """Rename a folder (same as move but for clarity)""" + """Rename a folder (same as move, named for clarity).""" return move_folder(notes_dir, old_path, new_path) def delete_folder(notes_dir: str, folder_path: str) -> bool: - """Delete a folder and all its contents""" + """Delete a folder and all its contents.""" try: full_path = Path(notes_dir) / folder_path - - # Security check: ensure the path is within notes_dir + if not validate_path_security(notes_dir, full_path): - print(f"Security: Path is outside notes directory: {full_path}") + logger.warning("Security: Path is outside notes directory: %s", full_path) return False - if not full_path.exists(): - print(f"Folder does not exist: {full_path}") + logger.warning("Folder does not exist: %s", full_path) return False - if not full_path.is_dir(): - print(f"Path is not a directory: {full_path}") + logger.warning("Path is not a directory: %s", full_path) return False - - # Invalidate cache for all notes in this folder - global _tag_cache - folder_path_str = str(full_path) - keys_to_delete = [key for key in _tag_cache.keys() if key.startswith(folder_path_str)] - for key in keys_to_delete: - del _tag_cache[key] - - # Delete the folder and all its contents + + _drop_prefix_caches(full_path) shutil.rmtree(full_path) - print(f"Successfully deleted folder: {full_path}") + note_index.on_folder_deleted(notes_dir, full_path) + _scan_cache_invalidate() return True except Exception as e: - print(f"Error deleting folder '{folder_path}': {e}") - import traceback - traceback.print_exc() + logger.error("Error deleting folder '%s': %s", folder_path, e) + logger.error(traceback.format_exc()) return False +def _drop_path_caches(full_path: Path) -> None: + """Evict the per-file mtime caches for a single note.""" + key = str(full_path) + _tag_cache.pop(key, None) + _links_cache.pop(key, None) + + +def _drop_prefix_caches(folder_full_path: Path) -> None: + """Evict per-file mtime caches for every entry under a folder.""" + prefix = str(folder_full_path) + for k in [k for k in _tag_cache if k.startswith(prefix)]: + _tag_cache.pop(k, None) + for k in [k for k in _links_cache if k.startswith(prefix)]: + _links_cache.pop(k, None) + + def get_note_content(notes_dir: str, note_path: str) -> Optional[str]: @@ -411,114 +500,116 @@ def get_note_content(notes_dir: str, note_path: str) -> Optional[str]: def save_note(notes_dir: str, note_path: str, content: str) -> bool: - """Save or update a note""" + """Save or update a note.""" full_path = Path(notes_dir) / note_path - - # Ensure .md extension if not note_path.endswith('.md'): full_path = full_path.with_suffix('.md') - - # Security check + if not validate_path_security(notes_dir, full_path): return False - - # Create parent directories if needed + full_path.parent.mkdir(parents=True, exist_ok=True) - with open(full_path, 'w', encoding='utf-8') as f: f.write(content) - + + # Refresh the per-file mtime caches with what we just wrote, so the next + # scan doesn't re-parse this file. extract_links + parse_tags + save are + # tiny relative to the file write. + try: + mtime = full_path.stat().st_mtime + file_key = str(full_path) + _tag_cache[file_key] = (mtime, parse_tags(content)) + _links_cache[file_key] = (mtime, extract_links_from_content(content)) + except Exception: + pass # caches are best-effort + + note_index.on_note_saved(notes_dir, full_path, content) + _scan_cache_invalidate() return True def delete_note(notes_dir: str, note_path: str) -> bool: - """Delete a note""" + """Delete a note.""" full_path = Path(notes_dir) / note_path - if not full_path.exists(): return False - - # Security check if not validate_path_security(notes_dir, full_path): return False - - # Invalidate cache for this note - file_key = str(full_path) - if file_key in _tag_cache: - del _tag_cache[file_key] - + + _drop_path_caches(full_path) full_path.unlink() - - # Note: We don't automatically delete empty folders to preserve user's folder structure - + note_index.on_note_deleted(notes_dir, full_path) + _scan_cache_invalidate() return True def search_notes(notes_dir: str, query: str) -> List[Dict]: - """ - Full-text search through note contents only. - Does NOT search in file names, folder names, or paths - only note content. - Uses character-based context extraction with highlighted matches. - """ + """Full-text search through note contents. Narrow the candidate set via + the inverted index, then run the snippet extractor on each candidate.""" from html import escape - results = [] - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) + results: List[Dict] = [] - for note in notes: - md_file = Path(notes_dir) / note["path"] + ensure_index_built(notes_dir) + note_index.ensure_search_index(notes_dir) + candidates = note_index.get_search_candidates(query) + + idx = note_index.get_index() + if candidates is None: + # Query too short to tokenize — iterate every indexed note instead. + candidate_records = idx.all_note_records() + else: + candidate_records = [(p, idx.get_note_record(p)) for p in candidates] + candidate_records = [(p, r) for (p, r) in candidate_records if r is not None and r.type == "note"] + + candidate_records.sort(key=lambda pr: pr[1].mtime, reverse=True) + iterable = [{"path": p, "name": r.name} for (p, r) in candidate_records] + + for note in iterable: + path = note["path"] + md_file = Path(notes_dir) / path try: with open(md_file, 'r', encoding='utf-8') as f: content = f.read() - - # Find all matches using regex (case-insensitive) + matches = list(re.finditer(re.escape(query), content, re.IGNORECASE)) - - if matches: - matched_lines = [] - - for match in matches[:3]: # Limit to 3 matches per file - start_index = match.start() - end_index = match.end() - matched_text = match.group() # Preserve original case - - # Create slice window: ±15 characters around match - context_start = max(0, start_index - 15) - context_end = min(len(content), end_index + 15) - - # Extract and clean parts (newlines → spaces) - before = escape(content[context_start:start_index].replace('\n', ' ')) - after = escape(content[end_index:context_end].replace('\n', ' ')) - matched_clean = escape(matched_text.replace('\n', ' ')) - - # Build snippet with highlight (styled via CSS) - snippet = f'{before}{matched_clean}{after}' - - # Add ellipsis if truncated at start - if context_start > 0: - snippet = '...' + snippet - - # Add ellipsis if truncated at end - if context_end < len(content): - snippet = snippet + '...' - - # Calculate line number by counting newlines up to match start - line_number = content.count('\n', 0, start_index) + 1 - - matched_lines.append({ - "line_number": line_number, - "context": snippet - }) - - relative_path = Path(note["path"]) - results.append({ - "name": md_file.stem, - "path": str(relative_path.as_posix()), - "folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "", - "matches": matched_lines + if not matches: + continue + + matched_lines = [] + for match in matches[:3]: + start_index = match.start() + end_index = match.end() + matched_text = match.group() + + context_start = max(0, start_index - 15) + context_end = min(len(content), end_index + 15) + + before = escape(content[context_start:start_index].replace('\n', ' ')) + after = escape(content[end_index:context_end].replace('\n', ' ')) + matched_clean = escape(matched_text.replace('\n', ' ')) + + snippet = f'{before}{matched_clean}{after}' + if context_start > 0: + snippet = '...' + snippet + if context_end < len(content): + snippet = snippet + '...' + + line_number = content.count('\n', 0, start_index) + 1 + matched_lines.append({ + "line_number": line_number, + "context": snippet, }) + + relative_path = Path(path) + results.append({ + "name": md_file.stem, + "path": str(relative_path.as_posix()), + "folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "", + "matches": matched_lines, + }) except Exception: continue - + return results @@ -644,9 +735,10 @@ def save_uploaded_image( try: with open(full_path, "wb") as f: f.write(file_data) + _scan_cache_invalidate() return str(full_path.relative_to(base).as_posix()) except OSError as e: - print(f"Error saving image: {e}") + logger.error("Error saving image: %s", e) return None sanitized_name = sanitize_filename(filename) @@ -658,14 +750,15 @@ def save_uploaded_image( attachments_dir.mkdir(parents=True, exist_ok=True) full_path = attachments_dir / final_filename if not validate_path_security(notes_dir, full_path): - print(f"Security: Attempted to save image outside notes directory: {full_path}") + logger.warning("Security: Attempted to save image outside notes directory: %s", full_path) return None try: with open(full_path, "wb") as f: f.write(file_data) + _scan_cache_invalidate() return str(full_path.relative_to(base).as_posix()) except OSError as e: - print(f"Error saving image: {e}") + logger.error("Error saving image: %s", e) return None @@ -781,110 +874,35 @@ def parse_tags(content: str) -> List[str]: return sorted(list(set(tags))) except Exception as e: - # If parsing fails, return empty list - print(f"Error parsing tags: {e}") + logger.error("Error parsing tags: %s", e) return [] -def get_tags_cached(file_path: Path) -> List[str]: - """ - Get tags for a file with caching based on modification time. - - Args: - file_path: Path to the markdown file - - Returns: - List of tags from the file (cached if mtime unchanged) - """ - global _tag_cache - - try: - # Get current modification time - mtime = file_path.stat().st_mtime - file_key = str(file_path) - - # Check cache - if file_key in _tag_cache: - cached_mtime, cached_tags = _tag_cache[file_key] - if cached_mtime == mtime: - # Cache hit! Return cached tags - return cached_tags - - # Cache miss or stale - parse tags - with open(file_path, 'r', encoding='utf-8') as f: - content = f.read() - tags = parse_tags(content) - - # Update cache - _tag_cache[file_key] = (mtime, tags) - return tags - - except Exception: - # If anything fails, return empty list - return [] - - -def clear_tag_cache(): - """Clear the tag cache (useful for testing or manual cache invalidation)""" - global _tag_cache - _tag_cache.clear() - - def get_all_tags(notes_dir: str) -> Dict[str, int]: - """ - Get all tags used across all notes with their count (cached). - - Args: - notes_dir: Directory containing notes - - Returns: - Dictionary mapping tag names to note counts - """ - tag_counts = {} - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) - - for note in notes: - md_file = Path(notes_dir) / note["path"] - # Get tags using cache - tags = get_tags_cached(md_file) - - for tag in tags: - tag_counts[tag] = tag_counts.get(tag, 0) + 1 - - return dict(sorted(tag_counts.items())) + """All tags in the vault with note counts.""" + ensure_index_built(notes_dir) + return note_index.get_all_tags() def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]: - """ - Get all notes that have a specific tag (cached). - - Args: - notes_dir: Directory containing notes - tag: Tag to filter by (case-insensitive) - - Returns: - List of note dictionaries matching the tag - """ - matching_notes = [] - tag_lower = tag.lower() - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) - - for note in notes: - md_file = Path(notes_dir) / note["path"] - # Get tags using cache - tags = get_tags_cached(md_file) - - if tag_lower in tags: - matching_notes.append({ - "name": note["name"], - "path": note["path"], - "folder": note["folder"], - "modified": note["modified"], - "size": note["size"], - "tags": tags - }) - - return matching_notes + """All notes carrying `tag` (case-insensitive).""" + ensure_index_built(notes_dir) + idx = note_index.get_index() + records = [idx.get_note_record(p) for p in note_index.get_paths_for_tag(tag.lower())] + matching = [ + { + "name": r.name, + "path": r.path, + "folder": r.folder, + "modified": r.modified, + "size": r.size, + "tags": list(r.tags), + } + for r in records + if r is not None and r.type == "note" + ] + matching.sort(key=lambda n: n["modified"], reverse=True) + return matching # ============================================================================ @@ -909,15 +927,14 @@ def get_templates(notes_dir: str) -> List[Dict]: # Security check: ensure _templates folder is within notes directory if not validate_path_security(notes_dir, templates_path): - print(f"Security: Templates directory is outside notes directory: {templates_path}") + logger.warning("Security: Templates directory is outside notes directory: %s", templates_path) return templates try: for template_file in templates_path.glob("*.md"): try: - # Security check: ensure each template is within notes directory if not validate_path_security(notes_dir, template_file): - print(f"Security: Skipping template outside notes directory: {template_file}") + logger.warning("Security: Skipping template outside notes directory: %s", template_file) continue stat = template_file.stat() @@ -927,10 +944,10 @@ def get_templates(notes_dir: str) -> List[Dict]: "modified": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat() }) except Exception as e: - print(f"Error reading template {template_file}: {e}") + logger.error("Error reading template %s: %s", template_file, e) continue except Exception as e: - print(f"Error accessing templates directory: {e}") + logger.error("Error accessing templates directory: %s", e) return sorted(templates, key=lambda x: x['name']) @@ -953,14 +970,14 @@ def get_template_content(notes_dir: str, template_name: str) -> Optional[str]: # Security check: ensure template is within notes directory if not validate_path_security(notes_dir, template_path): - print(f"Security: Template path is outside notes directory: {template_path}") + logger.warning("Security: Template path is outside notes directory: %s", template_path) return None try: with open(template_path, 'r', encoding='utf-8') as f: return f.read() except Exception as e: - print(f"Error reading template {template_name}: {e}") + logger.error("Error reading template %s: %s", template_name, e) return None @@ -1041,140 +1058,125 @@ def apply_template_placeholders(content: str, note_path: str) -> str: return result +def _extract_backlink_references( + notes_dir: str, + source_path: str, + target_path: str, + target_path_lower: str, + target_path_no_ext_lower: str, + wikilink_refs: set, +) -> List[Dict]: + """Read one source file and pull line-level references that point to target. + + Same regex + resolution rules as the legacy get_backlinks loop body; just + factored out so both the index-driven and the legacy code paths use the + exact same context-extraction logic. + """ + source_folder = str(Path(source_path).parent).replace('\\', '/') + if source_folder == '.': + source_folder = '' + + full_path = Path(notes_dir) / source_path + try: + with open(full_path, 'r', encoding='utf-8') as f: + content = f.read() + except Exception: + return [] + + lines = content.split('\n') + found_links: List[Dict] = [] + + for line_num, line in enumerate(lines, 1): + for match in re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line): + link_target = match.group(1).strip().lower() + link_target_no_ext = link_target.replace('.md', '') + if link_target in wikilink_refs or link_target_no_ext in wikilink_refs: + start = max(0, match.start() - 30) + end = min(len(line), match.end() + 30) + context = line[start:end] + if start > 0: + context = '...' + context + if end < len(line): + context = context + '...' + found_links.append({ + "line_number": line_num, + "context": context, + "type": "wikilink", + }) + + for match in re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line): + link_path = match.group(2).split('#')[0] + if not link_path: + continue + link_path = urllib.parse.unquote(link_path) + if link_path.startswith('./'): + link_path = link_path[2:] + link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md' + + resolved_path = None + if source_folder and not link_path.startswith('/'): + relative_path = f"{source_folder}/{link_path_with_md}" + if relative_path.lower() == target_path_lower: + resolved_path = target_path + elif f"{source_folder}/{link_path}".lower() == target_path_no_ext_lower: + resolved_path = target_path + if not resolved_path: + if link_path_with_md.lower() == target_path_lower: + resolved_path = target_path + elif link_path.lower() == target_path_no_ext_lower: + resolved_path = target_path + + if resolved_path: + start = max(0, match.start() - 30) + end = min(len(line), match.end() + 30) + context = line[start:end] + if start > 0: + context = '...' + context + if end < len(line): + context = context + '...' + found_links.append({ + "line_number": line_num, + "context": context, + "type": "markdown", + }) + + return found_links + + def get_backlinks(notes_dir: str, target_note_path: str) -> List[Dict]: - """ - Find all notes that link TO the specified note (reverse links / backlinks). - - Args: - notes_dir: Base directory containing notes - target_note_path: Path of the note to find backlinks for - - Returns: - List of backlink objects with path, context, and line_number - """ - backlinks = [] - notes, _folders = scan_notes_fast_walk(notes_dir, include_media=False) - - # Normalize target path for matching + """All notes that link TO `target_note_path`. The index narrows the + candidate set; we only read the candidate files for line context.""" target_path = target_note_path target_path_lower = target_path.lower() - target_path_no_ext = target_path.replace('.md', '') - target_path_no_ext_lower = target_path_no_ext.lower() - target_name = Path(target_path).stem.lower() - - # For wikilinks: global name matching (find note anywhere by name) + target_path_no_ext_lower = target_path_lower.replace('.md', '') wikilink_refs = { target_path_lower, target_path_no_ext_lower, - target_name, + Path(target_path).stem.lower(), } - - for note in notes: - if note.get('type') != 'note': - continue - - source_path = note['path'] - - # Skip self-references - if source_path == target_path: - continue - - # Get source folder for resolving relative markdown links - source_folder = str(Path(source_path).parent).replace('\\', '/') - if source_folder == '.': - source_folder = '' - - # Read note content - full_path = Path(notes_dir) / source_path - try: - with open(full_path, 'r', encoding='utf-8') as f: - content = f.read() - except Exception: - continue - - lines = content.split('\n') - found_links = [] - - for line_num, line in enumerate(lines, 1): - # Find wikilinks: [[target]] or [[target|display]] - # Wikilinks use GLOBAL matching (find note anywhere by name) - wikilink_matches = re.finditer(r'\[\[([^\]|]+)(?:\|[^\]]+)?\]\]', line) - for match in wikilink_matches: - link_target = match.group(1).strip().lower() - link_target_no_ext = link_target.replace('.md', '') - - # Check if this wikilink points to our target (global match) - if link_target in wikilink_refs or link_target_no_ext in wikilink_refs: - start = max(0, match.start() - 30) - end = min(len(line), match.end() + 30) - context = line[start:end] - if start > 0: - context = '...' + context - if end < len(line): - context = context + '...' - - found_links.append({ - "line_number": line_num, - "context": context, - "type": "wikilink" - }) - - # Find markdown links: [text](path) - # Markdown links must RESOLVE as paths (relative to source or absolute) - markdown_matches = re.finditer(r'\[([^\]]+)\]\((?!https?://|mailto:|#|data:)([^\)]+)\)', line) - for match in markdown_matches: - link_path = match.group(2).split('#')[0] # Remove anchor - if not link_path: - continue - - link_path = urllib.parse.unquote(link_path) - if link_path.startswith('./'): - link_path = link_path[2:] - - # Add .md if not present - link_path_with_md = link_path if link_path.endswith('.md') else link_path + '.md' - - # Resolve the link path to get the actual target - resolved_path = None - - # 1. Try resolving relative to source folder - if source_folder and not link_path.startswith('/'): - relative_path = f"{source_folder}/{link_path_with_md}" - if relative_path.lower() == target_path_lower: - resolved_path = target_path - elif f"{source_folder}/{link_path}".lower() == target_path_no_ext_lower: - resolved_path = target_path - - # 2. Try as absolute path from root - if not resolved_path: - if link_path_with_md.lower() == target_path_lower: - resolved_path = target_path - elif link_path.lower() == target_path_no_ext_lower: - resolved_path = target_path - - # Only add if the resolved path matches the target - if resolved_path: - start = max(0, match.start() - 30) - end = min(len(line), match.end() + 30) - context = line[start:end] - if start > 0: - context = '...' + context - if end < len(line): - context = context + '...' - - found_links.append({ - "line_number": line_num, - "context": context, - "type": "markdown" - }) - - # If we found links in this note, add it to backlinks - if found_links: + + ensure_index_built(notes_dir) + idx = note_index.get_index() + candidates = note_index.get_backlink_candidates(target_path) + records = [(p, idx.get_note_record(p)) for p in candidates] + records = [(p, r) for (p, r) in records if r is not None and r.type == "note" and p != target_path] + records.sort(key=lambda pr: pr[1].mtime, reverse=True) + + backlinks: List[Dict] = [] + for source_path, record in records: + refs = _extract_backlink_references( + notes_dir, + source_path, + target_path, + target_path_lower, + target_path_no_ext_lower, + wikilink_refs, + ) + if refs: backlinks.append({ "path": source_path, - "name": note['name'].replace('.md', ''), - "references": found_links[:3] # Limit to 3 references per note + "name": record.name.replace('.md', ''), + "references": refs[:3], }) - return backlinks diff --git a/documentation/API.md b/documentation/API.md index ad21981..270af9c 100644 --- a/documentation/API.md +++ b/documentation/API.md @@ -580,6 +580,57 @@ Returns application statistics at a glance. Designed for dashboard widgets (e.g. label: Version ``` +### Get Index Stats +```http +GET /api/index/stats +``` +Returns internal counters and sizes for the in-memory note index. Useful for confirming the index is built and observing its growth — handy when debugging slow endpoints or verifying that incremental updates are firing on every save/delete. + +The index is rebuilt on the first scan after each process start and updated incrementally on every save/delete/move. It lives in process memory only — no disk persistence. + +**Response:** +```json +{ + "built": true, + "search_built": false, + "notes": 142, + "folders": 12, + "tags": 37, + "links_forward_entries": 89, + "links_backward_entries": 76, + "wikilink_tokens": 134, + "search_terms": 0, + "counters": { + "build_count": 1, + "last_build_ms": 12.4, + "last_built_at": "2026-03-17T14:32:00+00:00", + "incremental_updates": 8, + "fingerprint_short_circuits": 3, + "search_build_count": 0, + "last_search_build_ms": 0.0 + } +} +``` + +| Field | Description | +|-------|-------------| +| `built` | `true` after the first vault scan completes | +| `search_built` | `true` after the first `/api/search` request (search index is built lazily) | +| `notes` | Number of note records currently held in the index | +| `folders` | Number of folder paths | +| `tags` | Number of unique tags | +| `links_forward_entries` | Number of notes that link out to at least one other note | +| `links_backward_entries` | Number of notes that have at least one incoming backlink | +| `wikilink_tokens` | Number of unique wikilink tokens (used for loose backlink matching by stem name) | +| `search_terms` | Number of unique terms in the full-text inverted search index (0 until first search) | +| `counters.build_count` | Total number of full index rebuilds since process start | +| `counters.last_build_ms` | Wall-clock time of the most recent full rebuild | +| `counters.last_built_at` | ISO timestamp of the most recent full rebuild | +| `counters.incremental_updates` | Number of single-note updates applied (save/delete/rename) | +| `counters.fingerprint_short_circuits` | Times a re-scan was skipped because the vault hash matched the previous scan | +| `counters.search_build_count` | Total number of search-index rebuilds | +| `counters.last_search_build_ms` | Wall-clock time of the most recent search-index rebuild | + ### Health Check ```http GET /health diff --git a/documentation/ENVIRONMENT_VARIABLES.md b/documentation/ENVIRONMENT_VARIABLES.md index 138c1d3..4eeed87 100644 --- a/documentation/ENVIRONMENT_VARIABLES.md +++ b/documentation/ENVIRONMENT_VARIABLES.md @@ -12,6 +12,30 @@ NoteDiscovery supports environment variables to override configuration settings, > **Note**: Advanced server settings (CORS origins, debug mode) are configured via `config.yaml` only, not via environment variables. See [config.yaml](#advanced-server-configuration) for details. +### Storage + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `NOTES_DIR` | string | `./data` | Path to the notes vault | +| `PLUGINS_DIR` | string | `./plugins` | Path to the plugins directory | + +The resolved paths are logged at startup so you can confirm what's in use: + +``` +INFO: Notes directory: /home/me/MyVault (from NOTES_DIR env var) +INFO: Plugins directory: ./plugins (from config.yaml) +``` + +#### Example: Pointing at an existing vault + +```bash +# Local +NOTES_DIR=/home/me/MyVault python run.py + +# Docker +docker run -e NOTES_DIR=/vault -v /home/me/MyVault:/vault ... +``` + ### Authentication | Variable | Type | Default | Description | diff --git a/documentation/PLUGIN_NOTE_STATISTICS.md b/documentation/PLUGIN_NOTE_STATISTICS.md index 1dd8c67..0ed68ad 100644 --- a/documentation/PLUGIN_NOTE_STATISTICS.md +++ b/documentation/PLUGIN_NOTE_STATISTICS.md @@ -103,19 +103,25 @@ When a note is saved, the plugin: 3. Click to expand/collapse the stats panel 4. Statistics update in real-time as you type -### In Docker Logs +### In Server Logs ```bash -docker-compose logs -f | grep "📊" +# Docker (any host OS) +docker-compose logs -f | grep "note_stats" + +# Running locally (Linux / macOS) +python run.py 2>&1 | grep "note_stats" + +# Running locally (Windows / PowerShell) +python run.py 2>&1 | findstr "note_stats" ``` -Example output: +Example output (single line, prefixed with uvicorn's `INFO:`): ``` -📊 projects/website.md: - 1,234 words | 6m read | 89 lines - 15 links (5 internal) - 8/12 tasks completed +INFO: note_stats projects/website.md | 1,234 words | 6 sentences | ~6m read | 89 lines | 15 links (5 internal) | 8/12 tasks ``` +Optional sections (`lists`, `tables`, `links`, `tasks`) only appear when their count is non-zero. + --- ## Configuration @@ -124,11 +130,11 @@ No configuration needed. The plugin works out of the box. ### Customization (Optional) -To change reading speed calculation, edit `note_stats.py`: +To change the reading-speed assumption used for `reading_time_minutes`, +edit the constant near the top of `note_stats.py`: ```python -# Line 36 -words_per_minute = 200 # Change to your reading speed +WORDS_PER_MINUTE = 200 # Change to your average reading speed ``` --- diff --git a/frontend/app.js b/frontend/app.js index 5ff7d7d..95c84b8 100644 --- a/frontend/app.js +++ b/frontend/app.js @@ -262,6 +262,13 @@ function noteApp() { alreadyDonated: false, autosaveDelayMs: CONFIG.AUTOSAVE_DELAY, // hydrated from /api/config in loadConfig() notes: [], + + // True while /api/notes is in flight. Drives the "Loading your vault…" + // placeholder + the delayed overlay (notesLoadingShowOverlay). + notesLoading: true, + notesLoadingShowOverlay: false, + _notesLoadingOverlayTimer: null, + currentNote: '', currentNoteName: '', noteContent: '', @@ -1522,18 +1529,32 @@ function noteApp() { // ==================== END INTERNATIONALIZATION ==================== - // Load all notes - async loadNotes() { + // Load all notes. Pass {silent: true} from error-recovery paths so the + // 800ms loading overlay never appears on background re-syncs. + async loadNotes({ silent = false } = {}) { + this.notesLoading = true; + clearTimeout(this._notesLoadingOverlayTimer); + if (!silent) { + this._notesLoadingOverlayTimer = setTimeout(() => { + if (this.notesLoading && this.notes.length === 0 && this.allFolders.length === 0) { + this.notesLoadingShowOverlay = true; + } + }, 800); + } try { const response = await fetch('/api/notes'); const data = await response.json(); this.notes = data.notes; this.allFolders = data.folders || []; - this.buildNoteLookupMaps(); // Build O(1) lookup maps + this.buildNoteLookupMaps(); this.buildFolderTree(); - await this.loadTags(); // Load tags after notes are loaded + await this.loadTags(); } catch (error) { ErrorHandler.handle('load notes', error); + } finally { + clearTimeout(this._notesLoadingOverlayTimer); + this.notesLoading = false; + this.notesLoadingShowOverlay = false; } }, @@ -1738,37 +1759,43 @@ function noteApp() { return; } - // Create note from template - const response = await fetch('/api/templates/create-note', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - templateName: this.selectedTemplate, - notePath: notePath - }) - }); + // Optimistic stub: add empty entry so sidebar updates instantly. + // Server rendering may inject content/tags; loadNote() fetches the + // real body and tags are refreshed via loadTagsDebounced below. + this._optimisticAddNote(notePath, { content: '' }); + const folderPart = notePath.includes('/') ? notePath.substring(0, notePath.lastIndexOf('/')) : ''; + if (folderPart) this.expandedFolders.add(folderPart); + this._rebuildTreeAfterMutation(); - if (!response.ok) { - const error = await response.json(); - this.toast(error.detail || this.t('templates.create_failed'), { type: 'error' }); - return; + try { + const response = await fetch('/api/templates/create-note', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + templateName: this.selectedTemplate, + notePath: notePath + }) + }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || this.t('templates.create_failed')); + } + const data = await response.json(); + + this.lastUsedTemplate = this.selectedTemplate; + try { localStorage.setItem('lastUsedTemplate', this.selectedTemplate); } catch (_) {} + + this.showTemplateModal = false; + this.selectedTemplate = ''; + this.newTemplateNoteName = ''; + + await this.loadNote(data.path); + this.focusEditorForNewNote(); + this.loadTagsDebounced(); + } catch (err) { + this.toast(err.message || this.t('templates.create_failed'), { type: 'error' }); + await this.loadNotes({ silent: true }); } - - const data = await response.json(); - - this.lastUsedTemplate = this.selectedTemplate; - try { localStorage.setItem('lastUsedTemplate', this.selectedTemplate); } catch (_) {} - - // Close modal and reset state - this.showTemplateModal = false; - this.selectedTemplate = ''; - this.newTemplateNoteName = ''; - - // Reload notes and open the new note - await this.loadNotes(); - await this.loadNote(data.path); - this.focusEditorForNewNote(); - } catch (error) { ErrorHandler.handle('create note from template', error); } @@ -2259,6 +2286,108 @@ function noteApp() { this.folderTree = tree; }, + // ===================================================================== + // OPTIMISTIC MUTATION HELPERS + // Mirror server-side index updates locally so file ops feel instant on + // big vaults. Mutations apply the change to this.notes/this.allFolders + // immediately, fire the request, and on error fall back to + // loadNotes({silent: true}) to resync from disk. + // ===================================================================== + + _isoNow() { + return new Date().toISOString(); + }, + + _folderFromPath(path) { + const i = path.lastIndexOf('/'); + return i === -1 ? '' : path.substring(0, i); + }, + + _filenameFromPath(path) { + return path.split('/').pop(); + }, + + _inferTypeFromPath(path) { + const m = /\.([^./]+)$/.exec(path); + const ext = m ? m[1].toLowerCase() : ''; + if (ext === 'md' || ext === '') return 'note'; + if (['png', 'jpg', 'jpeg', 'gif', 'svg', 'webp'].includes(ext)) return 'image'; + if (['mp3', 'wav', 'ogg', 'm4a', 'flac'].includes(ext)) return 'audio'; + if (['mp4', 'webm', 'mov', 'avi'].includes(ext)) return 'video'; + if (ext === 'pdf') return 'document'; + return 'note'; + }, + + // Refresh sidebar tree + wikilink lookup maps after any optimistic update. + _rebuildTreeAfterMutation() { + this.buildNoteLookupMaps(); + this.buildFolderTree(); + }, + + // Add a note/media file to the local list. No-op if already present. + _optimisticAddNote(path, { content = '', type = null, size = null } = {}) { + if (this.notes.some(n => n.path === path)) return; + const inferredType = type || this._inferTypeFromPath(path); + const filename = this._filenameFromPath(path); + const name = inferredType === 'note' ? filename.replace(/\.md$/i, '') : filename; + this.notes.push({ + path, + name, + folder: this._folderFromPath(path), + type: inferredType, + size: size != null ? size : (content ? new Blob([content]).size : 0), + modified: this._isoNow(), + tags: (inferredType === 'note' && content) ? this.parseTagsFromContent(content) : [], + }); + }, + + _optimisticRemoveNote(path) { + this.notes = this.notes.filter(n => n.path !== path); + }, + + // Used by single-note rename and move (path changes, identity preserved). + _optimisticRenameNote(oldPath, newPath) { + const note = this.notes.find(n => n.path === oldPath); + if (!note) return; + note.path = newPath; + note.folder = this._folderFromPath(newPath); + const filename = this._filenameFromPath(newPath); + note.name = note.type === 'note' ? filename.replace(/\.md$/i, '') : filename; + }, + + _optimisticAddFolder(folderPath) { + if (!folderPath) return; + if (!this.allFolders.includes(folderPath)) { + this.allFolders.push(folderPath); + } + }, + + // Cascade: remove the folder, its descendant folders, and every note inside. + _optimisticRemoveFolderTree(folderPath) { + const prefix = folderPath + '/'; + this.allFolders = this.allFolders.filter(f => f !== folderPath && !f.startsWith(prefix)); + this.notes = this.notes.filter(n => !n.path.startsWith(prefix)); + }, + + // Cascade: rename the folder, its descendant folders, and rewrite paths + // of every note inside. + _optimisticRenameFolderTree(oldPath, newPath) { + if (oldPath === newPath) return; + const oldPrefix = oldPath + '/'; + const newPrefix = newPath + '/'; + this.allFolders = this.allFolders.map(f => { + if (f === oldPath) return newPath; + if (f.startsWith(oldPrefix)) return newPrefix + f.substring(oldPrefix.length); + return f; + }); + this.notes.forEach(n => { + if (n.path.startsWith(oldPrefix)) { + n.path = newPrefix + n.path.substring(oldPrefix.length); + n.folder = this._folderFromPath(n.path); + } + }); + }, + // ===================================================================== // DATA-ATTRIBUTE BASED HANDLERS // These read path/name/type from data-* attributes, avoiding JS escaping issues @@ -2774,23 +2903,17 @@ function noteApp() { if (cursorPos < 0) cursorPos = textarea.selectionStart || 0; } - let uploaded = false; for (const file of mediaFiles) { try { const mediaPath = await this.uploadMedia(file, notePath); - if (mediaPath) { - uploaded = true; - if (this.currentNote) { - await this.insertMediaMarkdown(mediaPath, file.name, cursorPos); - } + if (mediaPath && this.currentNote) { + await this.insertMediaMarkdown(mediaPath, file.name, cursorPos); } } catch (error) { ErrorHandler.handle(`upload file ${file.name}`, error); } } - if (uploaded && !this.currentNote) { - await this.loadNotes(); - } + // uploadMedia already injects the file into this.notes optimistically. }, // Upload a media file (image, audio, video, PDF) @@ -2807,49 +2930,38 @@ function noteApp() { formData.append('note_path', notePath || ''); } - try { - const response = await fetch('/api/upload-media', { - method: 'POST', - body: formData - }); - - if (!response.ok) { - const error = await response.json(); - throw new Error(error.detail || 'Upload failed'); - } - - const data = await response.json(); - return data.path; - } catch (error) { - throw error; + const response = await fetch('/api/upload-media', { + method: 'POST', + body: formData + }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Upload failed'); } + const data = await response.json(); + // Drop the new file into the local note list so the wikilink + // resolver finds it without a full /api/notes refresh. + if (data.path) { + this._optimisticAddNote(data.path, { size: file.size }); + this._rebuildTreeAfterMutation(); + } + return data.path; }, // Insert media markdown at cursor position using wiki-style syntax - // This ensures media links don't break when notes are moved + // (media links don't break when notes are moved). The uploaded file is + // already in this.notes thanks to uploadMedia()'s optimistic add. async insertMediaMarkdown(mediaPath, altText, cursorPos) { - // Extract just the filename from the path (e.g., "folder/_attachments/image.png" -> "image.png") const filename = mediaPath.split('/').pop(); - - // Use wiki-style embed link: ![[filename.png]] or ![[filename.png|alt text]] - // The alt text is optional - only add if different from filename const filenameWithoutExt = filename.replace(/\.[^/.]+$/, ''); const altWithoutExt = altText.replace(/\.[^/.]+$/, ''); - - // If alt text is meaningful (not just "pasted-image"), include it const markdown = (altWithoutExt && altWithoutExt !== filenameWithoutExt && !altWithoutExt.startsWith('pasted-image')) ? `![[${filename}|${altWithoutExt}]]` : `![[${filename}]]`; - // Reload notes FIRST to update image lookup maps before preview renders - await this.loadNotes(); - const textBefore = this.noteContent.substring(0, cursorPos); const textAfter = this.noteContent.substring(cursorPos); - this.noteContent = textBefore + markdown + '\n' + textAfter; - - // Trigger autosave this.autoSave(); }, @@ -2989,23 +3101,18 @@ function noteApp() { }); if (!ok) return; + this._optimisticRemoveNote(mediaPath); + this._rebuildTreeAfterMutation(); + if (this.currentMedia === mediaPath) this.currentMedia = ''; + try { const response = await fetch(`/api/notes/${encodeURIComponent(mediaPath)}`, { method: 'DELETE' }); - - if (response.ok) { - await this.loadNotes(); // Refresh tree - - // Clear viewer if deleting currently viewed media - if (this.currentMedia === mediaPath) { - this.currentMedia = ''; - } - } else { - throw new Error('Failed to delete media file'); - } + if (!response.ok) throw new Error('Failed to delete media file'); } catch (error) { ErrorHandler.handle('delete media', error); + await this.loadNotes({ silent: true }); } }, @@ -3047,10 +3154,14 @@ function noteApp() { nextToNotes: true, contentFolder: targetFolder, }); - await this.loadNotes(); + // Server returns the final upload path (may differ from + // 'drawing.png' if it added a timestamp suffix). + this._optimisticAddNote(path, { type: 'image', size: blob.size }); + this._rebuildTreeAfterMutation(); this.viewMedia(path, 'drawing'); } catch (error) { ErrorHandler.handle('create drawing', error); + await this.loadNotes({ silent: true }); } }, @@ -3941,7 +4052,13 @@ function noteApp() { } throw new Error(detail || res.statusText); } - await this.loadNotes(); + // Drawing file already in this.notes — only metadata changed + // (size/mtime). Bump locally instead of a full /api/notes scan. + const rec = this.notes.find(n => n.path === this.currentMedia); + if (rec) { + rec.size = blob.size; + rec.modified = this._isoNow(); + } this.lastSaved = true; setTimeout(() => { this.lastSaved = false; @@ -4108,9 +4225,30 @@ function noteApp() { if (newPath === draggedPath) return; - // Capture favorites info before async call const oldPrefix = draggedPath + '/'; const newPrefix = newPath + '/'; + const wasExpanded = this.expandedFolders.has(draggedPath); + + this._optimisticRenameFolderTree(draggedPath, newPath); + this._rebuildTreeAfterMutation(); + + const favoritesInFolder = this.favorites.filter(f => f.startsWith(oldPrefix)); + if (favoritesInFolder.length > 0) { + const newFavorites = this.favorites.map(f => + f.startsWith(oldPrefix) ? newPrefix + f.substring(oldPrefix.length) : f + ); + this.favorites = newFavorites; + this.favoritesSet = new Set(newFavorites); + this.saveFavorites(); + } + if (wasExpanded) { + this.expandedFolders.delete(draggedPath); + this.expandedFolders.add(newPath); + this.saveExpandedFolders(); + } + if (this.currentNote && this.currentNote.startsWith(oldPrefix)) { + this.currentNote = newPrefix + this.currentNote.substring(oldPrefix.length); + } try { const response = await fetch('/api/folders/move', { @@ -4118,37 +4256,15 @@ function noteApp() { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ oldPath: draggedPath, newPath }) }); - - if (response.ok) { - // Update favorites for notes inside moved folder - const favoritesInFolder = this.favorites.filter(f => f.startsWith(oldPrefix)); - if (favoritesInFolder.length > 0) { - const newFavorites = this.favorites.map(f => - f.startsWith(oldPrefix) ? newPrefix + f.substring(oldPrefix.length) : f - ); - this.favorites = newFavorites; - this.favoritesSet = new Set(newFavorites); - this.saveFavorites(); - } - - // Keep folder expanded if it was - const wasExpanded = this.expandedFolders.has(draggedPath); - - await this.loadNotes(); - await this.loadSharedNotePaths(); - - if (wasExpanded) { - this.expandedFolders.delete(draggedPath); - this.expandedFolders.add(newPath); - this.saveExpandedFolders(); - } - } else { + if (!response.ok) { const errorData = await response.json().catch(() => ({})); - this.toast(errorData.detail || this.t('move.failed_folder'), { type: 'error' }); + throw new Error(errorData.detail || this.t('move.failed_folder')); } + await this.loadSharedNotePaths(); } catch (error) { console.error('Failed to move folder:', error); - this.toast(this.t('move.failed_folder'), { type: 'error' }); + this.toast(error.message || this.t('move.failed_folder'), { type: 'error' }); + await this.loadNotes({ silent: true }); } return; } @@ -4162,47 +4278,38 @@ function noteApp() { if (newPath === draggedPath) return; - // Check if note is favorited (only for notes) const wasFavorited = isNote && this.favoritesSet.has(draggedPath); + const wasCurrentNote = this.currentNote === draggedPath; + const wasCurrentMedia = this.currentMedia === draggedPath; + + this._optimisticRenameNote(draggedPath, newPath); + this._rebuildTreeAfterMutation(); + + if (wasFavorited) { + this.favorites = this.favorites.map(f => f === draggedPath ? newPath : f); + this.favoritesSet = new Set(this.favorites); + this.saveFavorites(); + } + if (wasCurrentNote) this.currentNote = newPath; + if (wasCurrentMedia) this.currentMedia = newPath; try { - // Use different endpoint for media vs notes const endpoint = isMedia ? '/api/media/move' : '/api/notes/move'; const response = await fetch(endpoint, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ oldPath: draggedPath, newPath }) }); - - if (response.ok) { - // Update favorites if the moved note was favorited - if (wasFavorited) { - const newFavorites = this.favorites.map(f => f === draggedPath ? newPath : f); - this.favorites = newFavorites; - this.favoritesSet = new Set(newFavorites); - this.saveFavorites(); - } - - // Keep current item open if it was the moved one - const wasCurrentNote = this.currentNote === draggedPath; - const wasCurrentMedia = this.currentMedia === draggedPath; - - await this.loadNotes(); - if (isNote) { - await this.loadSharedNotePaths(); - } - - if (wasCurrentNote) this.currentNote = newPath; - if (wasCurrentMedia) this.currentMedia = newPath; - } else { + if (!response.ok) { const errorData = await response.json().catch(() => ({})); const errorKey = isMedia ? 'move.failed_media' : 'move.failed_note'; - this.toast(errorData.detail || this.t(errorKey), { type: 'error' }); + throw new Error(errorData.detail || this.t(errorKey)); } + if (isNote) await this.loadSharedNotePaths(); } catch (error) { console.error(`Failed to move ${isMedia ? 'media' : 'note'}:`, error); - const errorKey = isMedia ? 'move.failed_media' : 'move.failed_note'; - this.toast(this.t(errorKey), { type: 'error' }); + this.toast(error.message || this.t(isMedia ? 'move.failed_media' : 'move.failed_note'), { type: 'error' }); + await this.loadNotes({ silent: true }); } }, @@ -4812,50 +4919,48 @@ function noteApp() { }, async _finalizeCreateNote(notePath) { + // Optimistic add — sidebar reflects the new note instantly. Server + // confirms; on failure we resync silently. + this._optimisticAddNote(notePath, { content: '' }); + const folderPart = notePath.includes('/') ? notePath.substring(0, notePath.lastIndexOf('/')) : ''; + if (folderPart) this.expandedFolders.add(folderPart); + this._rebuildTreeAfterMutation(); + try { const response = await fetch(`/api/notes/${notePath}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ content: '' }) }); - - if (response.ok) { - const folderPart = notePath.includes('/') ? notePath.substring(0, notePath.lastIndexOf('/')) : ''; - if (folderPart) this.expandedFolders.add(folderPart); - await this.loadNotes(); - await this.loadNote(notePath); - this.focusEditorForNewNote(); - return true; - } - ErrorHandler.handle('create note', new Error('Server returned error')); - return false; + if (!response.ok) throw new Error('Server returned error'); + await this.loadNote(notePath); + this.focusEditorForNewNote(); + return true; } catch (error) { ErrorHandler.handle('create note', error); + await this.loadNotes({ silent: true }); return false; } }, async _finalizeCreateFolder(folderPath, targetFolder) { + this._optimisticAddFolder(folderPath); + if (targetFolder) this.expandedFolders.add(targetFolder); + this.expandedFolders.add(folderPath); + this._rebuildTreeAfterMutation(); + try { const response = await fetch('/api/folders', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ path: folderPath }) }); - - if (response.ok) { - if (targetFolder) { - this.expandedFolders.add(targetFolder); - } - this.expandedFolders.add(folderPath); - await this.loadNotes(); - this.goToHomepageFolder(folderPath); - return true; - } - ErrorHandler.handle('create folder', new Error('Server returned error')); - return false; + if (!response.ok) throw new Error('Server returned error'); + this.goToHomepageFolder(folderPath); + return true; } catch (error) { ErrorHandler.handle('create folder', error); + await this.loadNotes({ silent: true }); return false; } }, @@ -4952,90 +5057,80 @@ function noteApp() { }, async _finalizeRenameFolder(folderPath, newPath) { + const folderPrefix = folderPath + '/'; + const newFolderPrefix = newPath + '/'; + + // Optimistic cascade: rewrite folder + every nested folder + every + // note path. Sidebar reflects the rename instantly. + this._optimisticRenameFolderTree(folderPath, newPath); + this._rebuildTreeAfterMutation(); + + if (this.expandedFolders.has(folderPath)) { + this.expandedFolders.delete(folderPath); + this.expandedFolders.add(newPath); + } + const newFavorites = this.favorites.map(f => + f.startsWith(folderPrefix) ? newFolderPrefix + f.substring(folderPrefix.length) : f + ); + if (JSON.stringify(newFavorites) !== JSON.stringify(this.favorites)) { + this.favorites = newFavorites; + this.favoritesSet = new Set(newFavorites); + this.saveFavorites(); + } + if (this.currentNote && this.currentNote.startsWith(folderPrefix)) { + this.currentNote = newFolderPrefix + this.currentNote.substring(folderPrefix.length); + } + try { const response = await fetch('/api/folders/rename', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - oldPath: folderPath, - newPath: newPath - }) + body: JSON.stringify({ oldPath: folderPath, newPath: newPath }) }); - - if (response.ok) { - if (this.expandedFolders.has(folderPath)) { - this.expandedFolders.delete(folderPath); - this.expandedFolders.add(newPath); - } - - const folderPrefix = folderPath + '/'; - const newFolderPrefix = newPath + '/'; - const newFavorites = this.favorites.map(f => { - if (f.startsWith(folderPrefix)) { - return f.replace(folderPrefix, newFolderPrefix); - } - return f; - }); - if (JSON.stringify(newFavorites) !== JSON.stringify(this.favorites)) { - this.favorites = newFavorites; - this.favoritesSet = new Set(newFavorites); - this.saveFavorites(); - } - - if (this.currentNote && this.currentNote.startsWith(folderPrefix)) { - this.currentNote = this.currentNote.replace(folderPrefix, newFolderPrefix); - } - - await this.loadNotes(); - return true; - } - ErrorHandler.handle('rename folder', new Error('Server returned error')); - return false; + if (!response.ok) throw new Error('Server returned error'); + return true; } catch (error) { ErrorHandler.handle('rename folder', error); + await this.loadNotes({ silent: true }); return false; } }, - // Delete folder + // Delete folder (cascade: every nested folder + note) async deleteFolder(folderPath, folderName) { const ok = await this.confirmModalAsk({ message: this.t('folders.confirm_delete', { name: folderName }), }); if (!ok) return; + const folderPrefix = folderPath + '/'; + + this._optimisticRemoveFolderTree(folderPath); + this._rebuildTreeAfterMutation(); + + this.expandedFolders.delete(folderPath); + const newFavorites = this.favorites.filter(f => !f.startsWith(folderPrefix)); + if (newFavorites.length !== this.favorites.length) { + this.favorites = newFavorites; + this.favoritesSet = new Set(newFavorites); + this.saveFavorites(); + } + if (this.currentNote && this.currentNote.startsWith(folderPrefix)) { + this.currentNote = ''; + this.noteContent = ''; + document.title = this.appName; + } + try { const response = await fetch(`/api/folders/${encodeURIComponent(folderPath)}`, { method: 'DELETE', headers: { 'Content-Type': 'application/json' } }); - - if (response.ok) { - // Remove from expanded folders - this.expandedFolders.delete(folderPath); - - // Remove any favorites that were in the deleted folder - const folderPrefix = folderPath + '/'; - const newFavorites = this.favorites.filter(f => !f.startsWith(folderPrefix)); - if (newFavorites.length !== this.favorites.length) { - this.favorites = newFavorites; - this.favoritesSet = new Set(newFavorites); - this.saveFavorites(); - } - - // Clear current note if it was in the deleted folder - if (this.currentNote && this.currentNote.startsWith(folderPrefix)) { - this.currentNote = ''; - this.noteContent = ''; - document.title = this.appName; - } - - await this.loadNotes(); - } else { - ErrorHandler.handle('delete folder', new Error('Server returned error')); - } + if (!response.ok) throw new Error('Server returned error'); + this.loadTagsDebounced(); } catch (error) { ErrorHandler.handle('delete folder', error); + await this.loadNotes({ silent: true }); } }, @@ -5479,33 +5574,29 @@ function noteApp() { return; } - // Create new note with same content + // Optimistic rename: rewrite local path now + favorites + current + // note pointer + URL. POST new content, then DELETE old. + this._optimisticRenameNote(oldPath, newPath); + this._rebuildTreeAfterMutation(); + + if (this.favoritesSet.has(oldPath)) { + this.favorites = this.favorites.map(f => f === oldPath ? newPath : f); + this.favoritesSet = new Set(this.favorites); + this.saveFavorites(); + } + this.currentNote = newPath; + try { const response = await fetch(`/api/notes/${newPath}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ content: this.noteContent }) }); - - if (response.ok) { - // Delete old note - await fetch(`/api/notes/${oldPath}`, { method: 'DELETE' }); - - // Update favorites if the renamed note was favorited - if (this.favoritesSet.has(oldPath)) { - const newFavorites = this.favorites.map(f => f === oldPath ? newPath : f); - this.favorites = newFavorites; - this.favoritesSet = new Set(newFavorites); - this.saveFavorites(); - } - - this.currentNote = newPath; - await this.loadNotes(); - } else { - ErrorHandler.handle('rename note', new Error('Server returned error')); - } + if (!response.ok) throw new Error('Server returned error'); + await fetch(`/api/notes/${oldPath}`, { method: 'DELETE' }); } catch (error) { ErrorHandler.handle('rename note', error); + await this.loadNotes({ silent: true }); } }, @@ -5524,39 +5615,36 @@ function noteApp() { }); if (!ok) return; + // Optimistic: remove locally + clear current note + drop favorite + // before the network round-trip. Sidebar refreshes in <1ms on any + // vault size. On error we resync silently from disk. + this._optimisticRemoveNote(notePath); + this._rebuildTreeAfterMutation(); + + if (this.favoritesSet.has(notePath)) { + this.favorites = this.favorites.filter(f => f !== notePath); + this.favoritesSet = new Set(this.favorites); + this.saveFavorites(); + } + + if (this.currentNote === notePath) { + this.currentNote = ''; + this.noteContent = ''; + this.currentNoteName = ''; + this._lastRenderedContent = ''; + this._lastRenderedNote = ''; + this._cachedRenderedHTML = ''; + document.title = this.appName; + window.history.replaceState({}, '', '/'); + } + try { - const response = await fetch(`/api/notes/${notePath}`, { - method: 'DELETE' - }); - - if (response.ok) { - // Remove from favorites if it was favorited - if (this.favoritesSet.has(notePath)) { - const newFavorites = this.favorites.filter(f => f !== notePath); - this.favorites = newFavorites; - this.favoritesSet = new Set(newFavorites); - this.saveFavorites(); - } - - // If the deleted note is currently open, clear it - if (this.currentNote === notePath) { - this.currentNote = ''; - this.noteContent = ''; - this.currentNoteName = ''; - this._lastRenderedContent = ''; // Clear render cache - this._lastRenderedNote = ''; - this._cachedRenderedHTML = ''; - document.title = this.appName; - // Redirect to root - window.history.replaceState({}, '', '/'); - } - - await this.loadNotes(); - } else { - ErrorHandler.handle('delete note', new Error('Server returned error')); - } + const response = await fetch(`/api/notes/${notePath}`, { method: 'DELETE' }); + if (!response.ok) throw new Error('Server returned error'); + this.loadTagsDebounced(); } catch (error) { ErrorHandler.handle('delete note', error); + await this.loadNotes({ silent: true }); } }, diff --git a/frontend/index.html b/frontend/index.html index a167237..b98079e 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1397,7 +1397,30 @@ - + + +
+
+ + + + + +
+
+