From a47c7991b4e4e1c45f88283065a60a02fe00980f Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 7 Jun 2026 05:21:07 -0700 Subject: [PATCH] Library-wide full-text search (SQLite FTS5), in both UIs (3.x) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new FTS5 index (`video_search`) over every video's title, channel, and description β€” searchable across the whole library, not just the loaded grid. `search_meta` tracks each video's mtime so a routine rescan only re-reads a description sidecar when the video actually changed; vanished videos are evicted. The index is refreshed after every scan in both front-ends via the shared `library::build_search_entries`. - database.rs: schema + `sync_search_index` (mtime-gated, one txn) + `search_videos` (ranked, with a highlighted snippet) + a safe prefix MATCH builder. Unit-tested (index/search/prefix/AND/evict/garbage). - web.rs: `GET /api/search?q=&limit=`; index synced after the initial scan, rescan, and maintenance-remove. - Web UI: a πŸ” header button + `f` shortcut open a debounced search modal with ranked results and highlighted description snippets; clicking a result jumps to it. - Desktop (egui): a πŸ”Ž Search button opens a floating results window querying the same index; clicking a result plays the video. Closes the desktop/web parity gap for search. Integration test seeds a real video + description, rescans, and asserts title / prefix / description-only matches hit and unrelated misses. Co-Authored-By: Claude Opus 4.8 --- src/app.rs | 95 ++++++++++++++++++++ src/database.rs | 205 ++++++++++++++++++++++++++++++++++++++++++ src/library.rs | 23 +++++ src/web.rs | 36 ++++++++ src/web_ui/index.html | 56 +++++++++++- tests/api.rs | 33 +++++++ 6 files changed, 447 insertions(+), 1 deletion(-) diff --git a/src/app.rs b/src/app.rs index a544efc..ebf8f1e 100644 --- a/src/app.rs +++ b/src/app.rs @@ -158,6 +158,11 @@ pub struct App { // Bulk selection bulk_mode: bool, bulk_selected: HashSet, + // Full-text search (floating results window) + search_query: String, + search_results: Vec, + show_search: bool, + search_focus: bool, // Scheduler last_scheduled_check: Option, // Cards cache β€” recomputed only when inputs change @@ -349,6 +354,9 @@ impl App { if let Ok(folder_map) = db.get_all_channel_assignments() { library::apply_channel_folders(&mut library, &folder_map); } + if let Err(e) = db.sync_search_index(&library::build_search_entries(&library)) { + eprintln!("search index sync failed: {e}"); + } let folders = db.list_folders().unwrap_or_default(); let watched = db.get_watched().unwrap_or_default(); let flags = db.get_video_flags().unwrap_or_default(); @@ -475,6 +483,10 @@ impl App { mpv_rx: None, bulk_mode: false, bulk_selected: HashSet::new(), + search_query: String::new(), + search_results: Vec::new(), + show_search: false, + search_focus: false, last_scheduled_check: None, cards_cache: Vec::new(), cards_cache_key: None, @@ -588,6 +600,9 @@ impl App { if let Ok(folder_map) = self.db.get_all_channel_assignments() { library::apply_channel_folders(&mut new_lib, &folder_map); } + if let Err(e) = self.db.sync_search_index(&library::build_search_entries(&new_lib)) { + eprintln!("search index sync failed: {e}"); + } self.folders = self.db.list_folders().unwrap_or_default(); self.library = new_lib; self.music_library = library::scan_music(&self.music_root); @@ -847,6 +862,78 @@ impl App { } } + /// Find a video by id anywhere in the library and play it. Used by the + /// full-text search results, where we only have the id. + fn play_by_id(&mut self, id: &str) { + let path = self.library.iter() + .flat_map(|c| c.videos.iter().chain(c.playlists.iter().flat_map(|p| p.videos.iter()))) + .find(|v| v.id == id) + .and_then(|v| v.video_path.clone()); + match path { + Some(p) => { self.selected_video = Some(id.to_string()); self.play_with_tracking(&p, id.to_string()); } + None => self.status = format!("'{id}' has no playable file on disk"), + } + } + + /// Floating full-text search window. Mirrors the web UI's πŸ” search β€” + /// queries the same FTS index (`db.search_videos`) and plays a result on + /// click. Distinct from the top-bar `self.search` filter, which only + /// narrows the already-loaded grid by title/id. + fn search_window(&mut self, ctx: &egui::Context) { + if !self.show_search { return; } + let mut open = true; + let mut to_play: Option = None; + egui::Window::new("πŸ”Ž Search library") + .open(&mut open) + .default_width(560.0) + .show(ctx, |ui| { + let resp = ui.add( + egui::TextEdit::singleline(&mut self.search_query) + .hint_text("titles, channels, descriptions…") + .desired_width(f32::INFINITY), + ); + if std::mem::take(&mut self.search_focus) { + resp.request_focus(); + } + if resp.changed() { + let q = self.search_query.trim(); + self.search_results = + self.db.search_videos(q, 100).unwrap_or_default(); + } + ui.separator(); + if self.search_query.trim().is_empty() { + ui.weak("Type to search every title, channel, and description in the library."); + } else if self.search_results.is_empty() { + ui.weak("No matches."); + } else { + ui.weak(format!("{} result(s)", self.search_results.len())); + egui::ScrollArea::vertical().max_height(420.0).show(ui, |ui| { + for hit in &self.search_results { + ui.add_space(4.0); + let title = ui.add( + egui::Label::new(egui::RichText::new(&hit.title).strong()) + .sense(egui::Sense::click()), + ); + if title.clicked() { to_play = Some(hit.video_id.clone()); } + if title.hovered() { + ui.ctx().set_cursor_icon(egui::CursorIcon::PointingHand); + } + ui.weak(format!("{} Β· {}", hit.channel, hit.platform)); + if !hit.snippet.is_empty() { + // Strip the STX/ETX match markers egui can't style. + let clean: String = hit.snippet + .chars().filter(|c| *c != '\u{2}' && *c != '\u{3}').collect(); + ui.label(egui::RichText::new(clean).small().weak()); + } + ui.separator(); + } + }); + } + }); + if !open { self.show_search = false; } + if let Some(id) = to_play { self.play_by_id(&id); } + } + fn play_with_tracking(&mut self, path: &Path, video_id: String) { let cmd = self.config.player.command.clone(); // Only enable IPC for genuine mpv invocations β€” substring matching @@ -1092,6 +1179,13 @@ impl App { .step_by(0.1), ); ui.separator(); + if ui.button("πŸ”Ž Search") + .on_hover_text("Full-text search titles + descriptions across the whole library") + .clicked() + { + self.show_search = true; + self.search_focus = true; + } if ui.button("⟳ Rescan").clicked() { self.rescan(); } @@ -3846,6 +3940,7 @@ impl eframe::App for App { self.channel_options_window(ctx); self.folder_manager_window(ctx); self.move_to_folder_window(ctx); + self.search_window(ctx); match self.current_screen { Screen::Library => { diff --git a/src/database.rs b/src/database.rs index e203029..61373d6 100644 --- a/src/database.rs +++ b/src/database.rs @@ -34,6 +34,46 @@ pub struct FolderRecord { pub parent_id: Option, } +/// One video's searchable fields, fed to [`Database::sync_search_index`]. +/// `description_path` is read lazily β€” only when the video is new or its +/// `mtime_unix` changed since the last index β€” so a routine rescan doesn't +/// re-read every description sidecar. +#[derive(Clone, Debug)] +pub struct SearchEntry { + pub video_id: String, + pub mtime_unix: i64, + pub platform: String, + pub channel: String, + pub title: String, + pub description_path: Option, +} + +/// A full-text search result row from [`Database::search_videos`]. +#[derive(Debug, Clone, serde::Serialize)] +pub struct SearchHit { + pub video_id: String, + pub platform: String, + pub channel: String, + pub title: String, + /// Description excerpt with the matched terms wrapped in `[`…`]`. + pub snippet: String, +} + +/// Build a safe FTS5 MATCH expression from free-form user input: each +/// whitespace token becomes a quoted prefix term, AND-ed together. Quoting +/// neutralizes FTS5 operators in the input; the trailing `*` gives +/// type-ahead prefix matching. Returns "" when nothing is searchable. +fn fts_match_expr(query: &str) -> String { + query + .split_whitespace() + .map(|t| t.replace('"', " ")) + .map(|t| t.trim().to_string()) + .filter(|t| !t.is_empty()) + .map(|t| format!("\"{t}\"*")) + .collect::>() + .join(" ") +} + /// In-memory representation of the `video_flags` table. Each set holds the /// video IDs that have the named flag enabled β€” kept small (a few hundred /// to a few thousand entries in practice). @@ -185,6 +225,24 @@ impl Database { body TEXT NOT NULL, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (target_kind, target_id) + ); + -- Full-text search over the library. `video_search` is a standalone + -- FTS5 index (available in rusqlite's bundled SQLite); `search_meta` + -- tracks each indexed video's mtime so [`Database::sync_search_index`] + -- only re-reads a description sidecar when the video actually + -- changed. video_id/platform are UNINDEXED β€” stored for retrieval, + -- not matched. + CREATE VIRTUAL TABLE IF NOT EXISTS video_search USING fts5( + video_id UNINDEXED, + platform UNINDEXED, + channel, + title, + description, + tokenize = 'porter unicode61' + ); + CREATE TABLE IF NOT EXISTS search_meta ( + video_id TEXT PRIMARY KEY, + mtime_unix INTEGER NOT NULL );", )?; @@ -887,6 +945,94 @@ impl Database { let _ = conn.execute("DETACH DATABASE bk", []); result } + + /// Refresh the full-text search index against the current library. + /// + /// `entries` is the full set of videos currently on disk. A video whose + /// `mtime_unix` already matches the index is skipped; new/changed videos + /// get their description sidecar re-read and reindexed; videos that + /// vanished from disk are dropped. Returns how many rows were + /// (re)indexed (0 means the index was already current). Runs in one + /// transaction so a crash mid-sync can't leave the index half-written. + pub fn sync_search_index(&self, entries: &[SearchEntry]) -> Result { + let mut conn = self.conn(); + let tx = conn.transaction()?; + + // What's already indexed: video_id -> mtime. + let mut existing: HashMap = HashMap::new(); + { + let mut stmt = tx.prepare("SELECT video_id, mtime_unix FROM search_meta")?; + let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))?; + for row in rows { let (id, m) = row?; existing.insert(id, m); } + } + + let mut seen: HashSet<&str> = HashSet::with_capacity(entries.len()); + let mut changed = 0usize; + for e in entries { + seen.insert(e.video_id.as_str()); + if existing.get(&e.video_id) == Some(&e.mtime_unix) { + continue; // unchanged β€” leave the indexed row in place + } + // Only new/changed videos pay the description-read cost. + let description = e.description_path.as_ref() + .and_then(|p| std::fs::read_to_string(p).ok()) + .unwrap_or_default(); + tx.execute("DELETE FROM video_search WHERE video_id = ?1", [&e.video_id])?; + tx.execute( + "INSERT INTO video_search (video_id, platform, channel, title, description) + VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![e.video_id, e.platform, e.channel, e.title, description], + )?; + tx.execute( + "INSERT OR REPLACE INTO search_meta (video_id, mtime_unix) VALUES (?1, ?2)", + rusqlite::params![e.video_id, e.mtime_unix], + )?; + changed += 1; + } + + // Evict videos that no longer exist on disk. + let stale: Vec = existing.keys() + .filter(|id| !seen.contains(id.as_str())) + .cloned() + .collect(); + for id in &stale { + tx.execute("DELETE FROM video_search WHERE video_id = ?1", [id])?; + tx.execute("DELETE FROM search_meta WHERE video_id = ?1", [id])?; + } + + tx.commit()?; + Ok(changed) + } + + /// Full-text search the library, newest-relevance first. Returns up to + /// `limit` hits, each with a highlighted description snippet. An empty or + /// punctuation-only query yields no results rather than an error. + pub fn search_videos(&self, query: &str, limit: usize) -> Result> { + let match_expr = fts_match_expr(query); + if match_expr.is_empty() { return Ok(Vec::new()); } + let conn = self.conn(); + let mut stmt = conn.prepare( + // STX/ETX (\u{2}/\u{3}) delimit matched terms β€” control chars + // that won't collide with literal '[' / ']' in a description. + // The UI turns them into highlight markup. + "SELECT video_id, platform, channel, title, + snippet(video_search, 4, char(2), char(3), '…', 12) + FROM video_search + WHERE video_search MATCH ?1 + ORDER BY rank + LIMIT ?2", + )?; + let rows = stmt.query_map(rusqlite::params![match_expr, limit as i64], |r| { + Ok(SearchHit { + video_id: r.get(0)?, + platform: r.get(1)?, + channel: r.get(2)?, + title: r.get(3)?, + snippet: r.get(4)?, + }) + })?; + rows.collect() + } } /// Per-table row counts that landed in the live DB during a restore. @@ -903,6 +1049,65 @@ pub struct RestoreSummary { pub notes_added: u64, } +#[cfg(test)] +mod search_tests { + use super::*; + + fn entry(id: &str, mtime: i64, channel: &str, title: &str) -> SearchEntry { + SearchEntry { + video_id: id.into(), mtime_unix: mtime, + platform: "channels".into(), channel: channel.into(), + title: title.into(), description_path: None, + } + } + + #[test] + fn indexes_searches_and_evicts() { + let db = Database::open_in_memory().unwrap(); + let entries = vec![ + entry("a", 1, "Rustaceans", "Async Rust deep dive"), + entry("b", 1, "Cooking", "Sourdough bread from scratch"), + ]; + assert_eq!(db.sync_search_index(&entries).unwrap(), 2); + + // Title match. + let hits = db.search_videos("rust", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].video_id, "a"); + + // Prefix (type-ahead) match. + assert_eq!(db.search_videos("sourd", 10).unwrap().len(), 1); + + // The channel field is searched too. + let hits = db.search_videos("cooking", 10).unwrap(); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].video_id, "b"); + + // Multi-token is AND-ed. + assert_eq!(db.search_videos("async dive", 10).unwrap().len(), 1); + assert_eq!(db.search_videos("async sourdough", 10).unwrap().len(), 0); + + // Re-syncing unchanged entries is a no-op. + assert_eq!(db.sync_search_index(&entries).unwrap(), 0); + + // A changed mtime forces a reindex of just that row. + let changed = vec![ + entry("a", 2, "Rustaceans", "Async Rust deep dive β€” updated"), + entry("b", 1, "Cooking", "Sourdough bread from scratch"), + ]; + assert_eq!(db.sync_search_index(&changed).unwrap(), 1); + assert_eq!(db.search_videos("updated", 10).unwrap().len(), 1); + + // Dropping "b" from disk evicts it from the index. + assert_eq!(db.sync_search_index(&changed[..1]).unwrap(), 0); + assert_eq!(db.search_videos("sourdough", 10).unwrap().len(), 0); + + // Garbage / empty queries return nothing, not an error. + assert!(db.search_videos("", 10).unwrap().is_empty()); + assert!(db.search_videos(" \" ", 10).unwrap().is_empty()); + } +} + #[cfg(test)] mod restore_tests { use super::*; diff --git a/src/library.rs b/src/library.rs index 74877f6..c35886c 100644 --- a/src/library.rs +++ b/src/library.rs @@ -684,3 +684,26 @@ fn track_from_path(path: &Path, folder_artist: &str) -> Option { file_size: std::fs::metadata(path).ok().map(|m| m.len()), }) } + +/// Flatten a scanned library into [`crate::database::SearchEntry`] rows for +/// the full-text index β€” every video in every channel and playlist, tagged +/// with its platform + channel name and a pointer to its description sidecar. +/// Shared by both front-ends so the index is built identically. +pub fn build_search_entries(lib: &[Channel]) -> Vec { + let mut out = Vec::new(); + for ch in lib { + let platform = ch.platform.dir_name(); + let playlist_videos = ch.playlists.iter().flat_map(|p| p.videos.iter()); + for v in ch.videos.iter().chain(playlist_videos) { + out.push(crate::database::SearchEntry { + video_id: v.id.clone(), + mtime_unix: v.mtime_unix.map(|m| m as i64).unwrap_or(0), + platform: platform.to_string(), + channel: ch.name.clone(), + title: v.title.clone(), + description_path: v.description_path.clone(), + }); + } + } + out +} diff --git a/src/web.rs b/src/web.rs index 6725329..08ef2a7 100644 --- a/src/web.rs +++ b/src/web.rs @@ -894,6 +894,16 @@ fn bump_library_version(state: &WebState) { state.library_version.fetch_add(1, Ordering::Relaxed); } +/// Bring the full-text search index in line with the current library. Cheap +/// after the first build (only new/changed videos re-read a description), so +/// it's fine to call inline after every (re)scan. Errors are logged, not +/// fatal β€” search degrading is better than a scan failing. +fn refresh_search_index(db: &Database, lib: &[library::Channel]) { + if let Err(e) = db.sync_search_index(&library::build_search_entries(lib)) { + eprintln!("search index sync failed: {e}"); + } +} + /// Re-read the password setting from the DB and update the cache. Called /// after any change that could affect whether a password exists. fn refresh_password_cache(state: &WebState) { @@ -1861,6 +1871,28 @@ async fn get_comments( (StatusCode::OK, Json(serde_json::json!({"comments": comments}))).into_response() } +#[derive(serde::Deserialize)] +struct SearchQuery { + #[serde(default)] + q: String, + limit: Option, +} + +/// `GET /api/search?q=…&limit=…` β€” full-text search over the library's +/// titles, channel names, and descriptions. Returns ranked hits with a +/// highlighted snippet. The index is kept current by [`refresh_search_index`] +/// after each scan. +async fn get_search( + State(state): State>, + Query(params): Query, +) -> impl IntoResponse { + let limit = params.limit.unwrap_or(50).clamp(1, 200); + match state.db.search_videos(¶ms.q, limit) { + Ok(results) => Json(serde_json::json!({ "results": results })).into_response(), + Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, format!("search failed: {e}")).into_response(), + } +} + async fn get_metadata( State(state): State>, Path(video_id): Path, @@ -1908,6 +1940,7 @@ async fn post_rescan(State(state): State>) -> impl IntoResponse { if let Ok(w) = state.db.get_watched() { *state.watched.lock_recover() = w; } + refresh_search_index(&state.db, &new_lib); *state.library.lock_recover() = new_lib; bump_library_version(&state); (StatusCode::OK, "rescanned") @@ -1958,6 +1991,7 @@ async fn post_maintenance_remove( if let Ok(folder_map) = state.db.get_all_channel_assignments() { library::apply_channel_folders(&mut new_lib, &folder_map); } + refresh_search_index(&state.db, &new_lib); *state.library.lock_recover() = new_lib; bump_library_version(&state); Json(serde_json::json!({ "removed": removed, "errors": errors })) @@ -2533,6 +2567,7 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) { } let watched = db.get_watched().unwrap_or_default(); let positions = db.get_positions().unwrap_or_default(); + refresh_search_index(&db, &library); let mut downloader = Downloader::new( channels_root.clone(), @@ -2678,6 +2713,7 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) { .route("/api/description/:id", get(get_description)) .route("/api/chapters/:id", get(get_chapters)) .route("/api/comments/:id", get(get_comments)) + .route("/api/search", get(get_search)) .route("/api/metadata/:id", get(get_metadata)) .route("/api/settings", get(get_settings).post(post_settings)) .route("/api/transcode/:id", get(get_transcode)) diff --git a/src/web_ui/index.html b/src/web_ui/index.html index a52bca3..e55b764 100644 --- a/src/web_ui/index.html +++ b/src/web_ui/index.html @@ -196,6 +196,7 @@ + @@ -1420,6 +1421,54 @@ function renderComments(){ } } +/* ── Full-text library search ───────────────────────────────────── */ +let ftsSeq=0; +function openSearch(){ + if(document.getElementById('fts-modal'))return; + const bg=document.createElement('div');bg.className='modal-bg';bg.id='fts-modal'; + bg.onclick=e=>{if(e.target===bg)bg.remove()}; + bg.innerHTML=``; + document.body.appendChild(bg); + const input=document.getElementById('fts-input'); + input.focus(); + let timer=null; + input.oninput=()=>{clearTimeout(timer);timer=setTimeout(()=>runFtsSearch(input.value.trim()),180)}; +} +async function runFtsSearch(q){ + const my=++ftsSeq; + const status=document.getElementById('fts-status'); + const box=document.getElementById('fts-results'); + if(!box)return; + if(!q){box.innerHTML='';if(status)status.textContent='';return} + if(status)status.textContent='Searching…'; + try{ + const r=await(await api('/api/search?limit=100&q='+encodeURIComponent(q))).json(); + if(my!==ftsSeq)return; // a newer keystroke superseded this response + const res=r.results||[]; + if(!res.length){box.innerHTML='
No matches.
';if(status)status.textContent='0 results';return} + box.innerHTML=res.map(h=>`
+
${esc(h.title)}
+
${esc(h.channel)} Β· ${esc(h.platform)}
+ ${h.snippet?`
${ftsSnippet(h.snippet)}
`:''} +
`).join(''); + if(status)status.textContent=res.length+' result'+(res.length===1?'':'s'); + }catch(e){box.innerHTML=`
Search failed: ${esc(e.message)}
`} +} +// FTS5 wraps matched terms in STX/ETX control chars; escape first, then mark. +function ftsSnippet(s){return esc(s).replace(/\x02/g,'').replace(/\x03/g,'')} +function ftsOpen(id){ + document.getElementById('fts-modal')?.remove(); + if(!findVideo(id)){setStatus('Video not in the loaded library β€” try Rescan');return} + selectVideo(id); + document.getElementById('details')?.scrollIntoView({behavior:'smooth',block:'nearest'}); +} + /* ── Select / Details ───────────────────────────────────────────── */ function selectVideo(id){selectedId=selectedId===id?null:id;renderGrid();renderDetails()} function closeDetails(){selectedId=null;renderGrid();renderDetails()} @@ -2253,6 +2302,10 @@ document.addEventListener('keydown',(e)=>{ document.getElementById('search')?.focus(); e.preventDefault(); break; + case 'f': + openSearch(); + e.preventDefault(); + break; case 'r': rescan(); e.preventDefault(); @@ -2275,7 +2328,8 @@ function showShortcutsHelp(){ bg.innerHTML=`