global in memory index for large vaults
This commit is contained in:
Gamosoft
parent
030565299a
commit
4b316b2cc7
|
|
@ -45,6 +45,8 @@ from .utils import (
|
|||
paginate,
|
||||
get_backlinks,
|
||||
)
|
||||
from . import note_index
|
||||
from .note_index import USE_NOTE_INDEX
|
||||
from .plugins import PluginManager
|
||||
from .themes import get_available_themes, get_theme_css
|
||||
from .share import (
|
||||
|
|
@ -265,6 +267,7 @@ plugin_manager = PluginManager(config['storage']['plugins_dir'])
|
|||
# Run app startup hooks
|
||||
plugin_manager.run_hook('on_app_startup')
|
||||
|
||||
|
||||
# Mount static files
|
||||
static_path = Path(__file__).parent.parent / "frontend"
|
||||
app.mount("/static", StaticFiles(directory=static_path), name="static")
|
||||
|
|
@ -1483,10 +1486,25 @@ async def search(
|
|||
|
||||
@api_router.get("/graph", tags=["Graph"])
|
||||
async def get_graph():
|
||||
"""Get graph data for note visualization with wikilink and markdown link detection"""
|
||||
"""Get graph data for note visualization with wikilink and markdown link detection."""
|
||||
try:
|
||||
import re
|
||||
import urllib.parse
|
||||
|
||||
# Fast path: index already holds resolved edges. Ensure a scan has
|
||||
# populated it (first request after startup, or after an invalidate)
|
||||
# then ask the facade for a snapshot.
|
||||
if USE_NOTE_INDEX:
|
||||
if not note_index.get_index().is_built():
|
||||
scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
|
||||
indexed = note_index.try_graph_data()
|
||||
if indexed is not None:
|
||||
nodes_paths, edges_tuples = indexed
|
||||
return {
|
||||
"nodes": [{"id": p, "label": Path(p).stem} for p in nodes_paths],
|
||||
"edges": [{"source": s, "target": t, "type": et} for (s, t, et) in edges_tuples],
|
||||
}
|
||||
|
||||
notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], include_media=False)
|
||||
nodes = []
|
||||
edges = []
|
||||
|
|
@ -1692,6 +1710,19 @@ async def toggle_plugin(request: Request, plugin_name: str, enabled: dict):
|
|||
raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to toggle plugin"))
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Index observability — internal counters & sizes for debugging
|
||||
#
|
||||
# Useful for confirming the index is doing what we expect (build_count >0,
|
||||
# search_terms not empty, fingerprint_short_circuits incrementing on idle
|
||||
# warm scans). Not rate-limited — cheap, no I/O, dict lookups only.
|
||||
# ============================================================================
|
||||
|
||||
@api_router.get("/index/stats", tags=["System"])
|
||||
async def get_index_stats():
|
||||
return note_index.stats()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Stats Endpoint (for dashboards)
|
||||
# ============================================================================
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
880
backend/utils.py
880
backend/utils.py
File diff suppressed because it is too large
Load Diff
|
|
@ -580,6 +580,59 @@ Returns application statistics at a glance. Designed for dashboard widgets (e.g.
|
|||
label: Version
|
||||
```
|
||||
|
||||
### Get Index Stats
|
||||
```http
|
||||
GET /api/index/stats
|
||||
```
|
||||
Returns internal counters and sizes for the in-memory note index. Useful for confirming the index is built and observing its growth — handy when debugging slow endpoints or verifying that incremental updates are firing on every save/delete.
|
||||
|
||||
The index is rebuilt on the first scan after each process start and updated incrementally on every save/delete/move. It lives in process memory only — no disk persistence.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"enabled": true,
|
||||
"built": true,
|
||||
"search_built": false,
|
||||
"notes": 142,
|
||||
"folders": 12,
|
||||
"tags": 37,
|
||||
"links_forward_entries": 89,
|
||||
"links_backward_entries": 76,
|
||||
"wikilink_tokens": 134,
|
||||
"search_terms": 0,
|
||||
"counters": {
|
||||
"build_count": 1,
|
||||
"last_build_ms": 12.4,
|
||||
"last_built_at": "2026-03-17T14:32:00+00:00",
|
||||
"incremental_updates": 8,
|
||||
"fingerprint_short_circuits": 3,
|
||||
"search_build_count": 0,
|
||||
"last_search_build_ms": 0.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `enabled` | Whether the index is active (always `true` in current builds) |
|
||||
| `built` | `true` after the first vault scan completes |
|
||||
| `search_built` | `true` after the first `/api/search` request (search index is built lazily) |
|
||||
| `notes` | Number of note records currently held in the index |
|
||||
| `folders` | Number of folder paths |
|
||||
| `tags` | Number of unique tags |
|
||||
| `links_forward_entries` | Number of notes that link out to at least one other note |
|
||||
| `links_backward_entries` | Number of notes that have at least one incoming backlink |
|
||||
| `wikilink_tokens` | Number of unique wikilink tokens (used for loose backlink matching by stem name) |
|
||||
| `search_terms` | Number of unique terms in the full-text inverted search index (0 until first search) |
|
||||
| `counters.build_count` | Total number of full index rebuilds since process start |
|
||||
| `counters.last_build_ms` | Wall-clock time of the most recent full rebuild |
|
||||
| `counters.last_built_at` | ISO timestamp of the most recent full rebuild |
|
||||
| `counters.incremental_updates` | Number of single-note updates applied (save/delete/rename) |
|
||||
| `counters.fingerprint_short_circuits` | Times a re-scan was skipped because the vault hash matched the previous scan |
|
||||
| `counters.search_build_count` | Total number of search-index rebuilds |
|
||||
| `counters.last_search_build_ms` | Wall-clock time of the most recent search-index rebuild |
|
||||
|
||||
### Health Check
|
||||
```http
|
||||
GET /health
|
||||
|
|
|
|||
Loading…
Reference in New Issue