builderadmin
/
NoteDiscovery
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

refactor(api): single scan_notes_fast_walk, remove get_all_*

This commit is contained in:
rdavis 2026-02-20 00:48:31 -06:00
parent 7073904143
commit 4d4c68b46c
3 changed files with 85 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/export.py
View File

@ -13,8 +13,8 @@ from pathlib import Path
from typing import Optional, Tuple
import mimetypes
# Import shared media type definitions from utils to avoid duplication
from backend.utils import MEDIA_EXTENSIONS, get_media_type
# Import shared media type definitions and scanner from utils to avoid duplication
from backend.utils import MEDIA_EXTENSIONS, get_media_type, scan_notes_fast_walk
def get_media_as_base64(media_path: Path) -> Optional[Tuple[str, str]]:
@ -111,8 +111,10 @@ def find_media_in_attachments(media_name: str, note_folder: Path, notes_dir: Pat
# Fallback: search all _attachments folders recursively (slower but thorough)
# This handles cross-folder media references like in Obsidian
try:
for attachment_folder in notes_dir.rglob('_attachments'):
if attachment_folder.is_dir():
_files, folders = scan_notes_fast_walk(str(notes_dir), "*")
for folder in folders:
if folder == '_attachments' or folder.endswith('/_attachments'):
attachment_folder = notes_dir / folder
candidate = attachment_folder / media_name
if candidate.exists() and candidate.is_file():
try:

8
backend/main.py
View File

@ -21,7 +21,7 @@ from slowapi.util import get_remote_address
from slowapi.errors import RateLimitExceeded
from .utils import (
get_all_notes,
scan_notes_fast_walk,
get_note_content,
save_note,
delete_note,
@ -29,7 +29,6 @@ from .utils import (
create_note_metadata,
ensure_directories,
create_folder,
get_all_folders,
move_note,
move_folder,
rename_folder,
@ -900,8 +899,7 @@ async def create_note_from_template(request: Request, data: dict):
async def list_notes():
"""List all notes with metadata"""
try:
notes = get_all_notes(config['storage']['notes_dir'])
folders = get_all_folders(config['storage']['notes_dir'])
notes, folders = scan_notes_fast_walk(config['storage']['notes_dir'], "*.md")
return {"notes": notes, "folders": folders}
except Exception as e:
raise HTTPException(status_code=500, detail=safe_error_message(e, "Failed to list notes"))
@ -1023,7 +1021,7 @@ async def get_graph():
try:
import re
import urllib.parse
notes = get_all_notes(config['storage']['notes_dir'])
notes, _folders = scan_notes_fast_walk(config['storage']['notes_dir'], "*.md")
nodes = []
edges = []

153
backend/utils.py
View File

@ -15,6 +15,7 @@ from datetime import datetime, timezone
_tag_cache: Dict[str, Tuple[float, List[str]]] = {}
def validate_path_security(notes_dir: str, path: Path) -> bool:
"""
Validate that a path is within the notes directory (security check).
@ -58,20 +59,54 @@ def create_folder(notes_dir: str, folder_path: str) -> bool:
return True
def get_all_folders(notes_dir: str) -> List[str]:
"""Get all folders in the notes directory, including empty ones"""
folders = []
notes_path = Path(notes_dir)
for item in notes_path.rglob("*"):
if item.is_dir():
relative_path = item.relative_to(notes_path)
folder_path = str(relative_path.as_posix())
if folder_path and not folder_path.startswith('.'):
folders.append(folder_path)
return sorted(folders)
def scan_notes_fast_walk(notes_dir: str, file_filter: Optional[str] = None, use_cache: bool = True) -> Tuple[List[Dict], List[str]]:
"""Fast scanner using os.walk (pure Python + stdlib).
Args:
notes_dir: Base notes directory
file_filter: Optional glob-style filter (e.g., "*", "*.md") similar to Path.rglob
"""
notes_path = Path(notes_dir)
notes: List[Dict] = []
folders_set = set()
for root, dirnames, filenames in os.walk(notes_path):
# Skip descending into dot-dirs
dirnames[:] = [d for d in dirnames if not d.startswith('.')]
root_path = Path(root)
rel_folder = root_path.relative_to(notes_path).as_posix()
if rel_folder != "." and not rel_folder.startswith('.'):
folders_set.add(rel_folder)
for filename in filenames:
if filename.startswith('.'):
continue
full_path = root_path / filename
try:
st = full_path.stat()
except OSError:
continue
relative_path = full_path.relative_to(notes_path)
if file_filter and not relative_path.match(file_filter):
continue
folder = relative_path.parent.as_posix()
notes.append({
"name": full_path.stem,
"path": relative_path.as_posix(),
"folder": "" if folder == "." else folder,
"modified": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
"size": st.st_size,
"type": "note",
"tags": [],
})
return sorted(notes, key=lambda x: x.get('modified', ''), reverse=True), sorted(folders_set)
def move_note(notes_dir: str, old_path: str, new_path: str) -> tuple[bool, str]:
"""Move a note to a different location
@ -200,34 +235,6 @@ def delete_folder(notes_dir: str, folder_path: str) -> bool:
return False
def get_all_notes(notes_dir: str) -> List[Dict]:
"""Recursively get all markdown notes and images"""
items = []
notes_path = Path(notes_dir)
# Get all markdown notes
for md_file in notes_path.rglob("*.md"):
relative_path = md_file.relative_to(notes_path)
stat = md_file.stat()
# Get tags for this note (cached)
tags = get_tags_cached(md_file)
items.append({
"name": md_file.stem,
"path": str(relative_path.as_posix()),
"folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "",
"modified": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
"size": stat.st_size,
"type": "note",
"tags": tags
})
# Get all images
images = get_all_images(notes_dir)
items.extend(images)
return sorted(items, key=lambda x: x['modified'], reverse=True)
def get_note_content(notes_dir: str, note_path: str) -> Optional[str]:
@ -297,9 +304,10 @@ def search_notes(notes_dir: str, query: str) -> List[Dict]:
"""
from html import escape
results = []
notes_path = Path(notes_dir)
for md_file in notes_path.rglob("*.md"):
notes, _folders = scan_notes_fast_walk(notes_dir, "*.md")
for note in notes:
md_file = Path(notes_dir) / note["path"]
try:
with open(md_file, 'r', encoding='utf-8') as f:
content = f.read()
@ -343,7 +351,7 @@ def search_notes(notes_dir: str, query: str) -> List[Dict]:
"context": snippet
})
relative_path = md_file.relative_to(notes_path)
relative_path = Path(note["path"])
results.append({
"name": md_file.stem,
"path": str(relative_path.as_posix()),
@ -517,28 +525,20 @@ def get_all_images(notes_dir: str) -> List[Dict]:
Note: Function name kept as 'get_all_images' for backward compatibility.
"""
media_files = []
notes_path = Path(notes_dir)
media_candidates, _folders = scan_notes_fast_walk(notes_dir, "*")
# Find all media files recursively in the entire notes directory
for media_file in notes_path.rglob("*"):
# Skip directories and hidden files/folders
if media_file.is_dir():
continue
if any(part.startswith('.') for part in media_file.parts):
continue
for media_file in media_candidates:
# Check if it's a media file
media_type = get_media_type(media_file.name)
filename = Path(media_file["path"]).name
media_type = get_media_type(filename)
if media_type:
relative_path = media_file.relative_to(notes_path)
stat = media_file.stat()
media_files.append({
"name": media_file.name,
"path": str(relative_path.as_posix()),
"folder": str(relative_path.parent.as_posix()) if relative_path.parent != Path('.') else "",
"modified": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
"size": stat.st_size,
"name": filename,
"path": media_file["path"],
"folder": media_file["folder"],
"modified": media_file["modified"],
"size": media_file["size"],
"type": media_type # 'image', 'audio', 'video', or 'document'
})
@ -688,9 +688,10 @@ def get_all_tags(notes_dir: str) -> Dict[str, int]:
Dictionary mapping tag names to note counts
"""
tag_counts = {}
notes_path = Path(notes_dir)
for md_file in notes_path.rglob("*.md"):
notes, _folders = scan_notes_fast_walk(notes_dir, "*.md")
for note in notes:
md_file = Path(notes_dir) / note["path"]
# Get tags using cache
tags = get_tags_cached(md_file)
@ -713,22 +714,20 @@ def get_notes_by_tag(notes_dir: str, tag: str) -> List[Dict]:
"""
matching_notes = []
tag_lower = tag.lower()
notes_path = Path(notes_dir)
for md_file in notes_path.rglob("*.md"):
notes, _folders = scan_notes_fast_walk(notes_dir, "*.md")
for note in notes:
md_file = Path(notes_dir) / note["path"]
# Get tags using cache
tags = get_tags_cached(md_file)
if tag_lower in tags:
relative_path = md_file.relative_to(notes_path)
stat = md_file.stat()
matching_notes.append({
"name": md_file.stem,
"path": str(relative_path.as_posix()),
"folder": str(relative_path.parent.as_posix()) if str(relative_path.parent) != "." else "",
"modified": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
"size": stat.st_size,
"name": note["name"],
"path": note["path"],
"folder": note["folder"],
"modified": note["modified"],
"size": note["size"],
"tags": tags
})