Library-wide full-text search (SQLite FTS5), in both UIs (3.x)

A new FTS5 index (`video_search`) over every video's title, channel, and
description — searchable across the whole library, not just the loaded
grid. `search_meta` tracks each video's mtime so a routine rescan only
re-reads a description sidecar when the video actually changed; vanished
videos are evicted. The index is refreshed after every scan in both
front-ends via the shared `library::build_search_entries`.

- database.rs: schema + `sync_search_index` (mtime-gated, one txn) +
  `search_videos` (ranked, with a highlighted snippet) + a safe prefix
  MATCH builder. Unit-tested (index/search/prefix/AND/evict/garbage).
- web.rs: `GET /api/search?q=&limit=`; index synced after the initial
  scan, rescan, and maintenance-remove.
- Web UI: a 🔍 header button + `f` shortcut open a debounced search modal
  with ranked results and highlighted description snippets; clicking a
  result jumps to it.
- Desktop (egui): a 🔎 Search button opens a floating results window
  querying the same index; clicking a result plays the video. Closes the
  desktop/web parity gap for search.

Integration test seeds a real video + description, rescans, and asserts
title / prefix / description-only matches hit and unrelated misses.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Luna 2026-06-07 05:21:07 -07:00
parent 3031b5b0e5
commit a47c7991b4
6 changed files with 447 additions and 1 deletions

View file

@ -158,6 +158,11 @@ pub struct App {
// Bulk selection
bulk_mode: bool,
bulk_selected: HashSet<String>,
// Full-text search (floating results window)
search_query: String,
search_results: Vec<crate::database::SearchHit>,
show_search: bool,
search_focus: bool,
// Scheduler
last_scheduled_check: Option<Instant>,
// Cards cache — recomputed only when inputs change
@ -349,6 +354,9 @@ impl App {
if let Ok(folder_map) = db.get_all_channel_assignments() {
library::apply_channel_folders(&mut library, &folder_map);
}
if let Err(e) = db.sync_search_index(&library::build_search_entries(&library)) {
eprintln!("search index sync failed: {e}");
}
let folders = db.list_folders().unwrap_or_default();
let watched = db.get_watched().unwrap_or_default();
let flags = db.get_video_flags().unwrap_or_default();
@ -475,6 +483,10 @@ impl App {
mpv_rx: None,
bulk_mode: false,
bulk_selected: HashSet::new(),
search_query: String::new(),
search_results: Vec::new(),
show_search: false,
search_focus: false,
last_scheduled_check: None,
cards_cache: Vec::new(),
cards_cache_key: None,
@ -588,6 +600,9 @@ impl App {
if let Ok(folder_map) = self.db.get_all_channel_assignments() {
library::apply_channel_folders(&mut new_lib, &folder_map);
}
if let Err(e) = self.db.sync_search_index(&library::build_search_entries(&new_lib)) {
eprintln!("search index sync failed: {e}");
}
self.folders = self.db.list_folders().unwrap_or_default();
self.library = new_lib;
self.music_library = library::scan_music(&self.music_root);
@ -847,6 +862,78 @@ impl App {
}
}
/// Find a video by id anywhere in the library and play it. Used by the
/// full-text search results, where we only have the id.
fn play_by_id(&mut self, id: &str) {
let path = self.library.iter()
.flat_map(|c| c.videos.iter().chain(c.playlists.iter().flat_map(|p| p.videos.iter())))
.find(|v| v.id == id)
.and_then(|v| v.video_path.clone());
match path {
Some(p) => { self.selected_video = Some(id.to_string()); self.play_with_tracking(&p, id.to_string()); }
None => self.status = format!("'{id}' has no playable file on disk"),
}
}
/// Floating full-text search window. Mirrors the web UI's 🔍 search —
/// queries the same FTS index (`db.search_videos`) and plays a result on
/// click. Distinct from the top-bar `self.search` filter, which only
/// narrows the already-loaded grid by title/id.
fn search_window(&mut self, ctx: &egui::Context) {
if !self.show_search { return; }
let mut open = true;
let mut to_play: Option<String> = None;
egui::Window::new("🔎 Search library")
.open(&mut open)
.default_width(560.0)
.show(ctx, |ui| {
let resp = ui.add(
egui::TextEdit::singleline(&mut self.search_query)
.hint_text("titles, channels, descriptions…")
.desired_width(f32::INFINITY),
);
if std::mem::take(&mut self.search_focus) {
resp.request_focus();
}
if resp.changed() {
let q = self.search_query.trim();
self.search_results =
self.db.search_videos(q, 100).unwrap_or_default();
}
ui.separator();
if self.search_query.trim().is_empty() {
ui.weak("Type to search every title, channel, and description in the library.");
} else if self.search_results.is_empty() {
ui.weak("No matches.");
} else {
ui.weak(format!("{} result(s)", self.search_results.len()));
egui::ScrollArea::vertical().max_height(420.0).show(ui, |ui| {
for hit in &self.search_results {
ui.add_space(4.0);
let title = ui.add(
egui::Label::new(egui::RichText::new(&hit.title).strong())
.sense(egui::Sense::click()),
);
if title.clicked() { to_play = Some(hit.video_id.clone()); }
if title.hovered() {
ui.ctx().set_cursor_icon(egui::CursorIcon::PointingHand);
}
ui.weak(format!("{} · {}", hit.channel, hit.platform));
if !hit.snippet.is_empty() {
// Strip the STX/ETX match markers egui can't style.
let clean: String = hit.snippet
.chars().filter(|c| *c != '\u{2}' && *c != '\u{3}').collect();
ui.label(egui::RichText::new(clean).small().weak());
}
ui.separator();
}
});
}
});
if !open { self.show_search = false; }
if let Some(id) = to_play { self.play_by_id(&id); }
}
fn play_with_tracking(&mut self, path: &Path, video_id: String) {
let cmd = self.config.player.command.clone();
// Only enable IPC for genuine mpv invocations — substring matching
@ -1092,6 +1179,13 @@ impl App {
.step_by(0.1),
);
ui.separator();
if ui.button("🔎 Search")
.on_hover_text("Full-text search titles + descriptions across the whole library")
.clicked()
{
self.show_search = true;
self.search_focus = true;
}
if ui.button("⟳ Rescan").clicked() {
self.rescan();
}
@ -3846,6 +3940,7 @@ impl eframe::App for App {
self.channel_options_window(ctx);
self.folder_manager_window(ctx);
self.move_to_folder_window(ctx);
self.search_window(ctx);
match self.current_screen {
Screen::Library => {

View file

@ -34,6 +34,46 @@ pub struct FolderRecord {
pub parent_id: Option<i64>,
}
/// One video's searchable fields, fed to [`Database::sync_search_index`].
/// `description_path` is read lazily — only when the video is new or its
/// `mtime_unix` changed since the last index — so a routine rescan doesn't
/// re-read every description sidecar.
#[derive(Clone, Debug)]
pub struct SearchEntry {
pub video_id: String,
pub mtime_unix: i64,
pub platform: String,
pub channel: String,
pub title: String,
pub description_path: Option<std::path::PathBuf>,
}
/// A full-text search result row from [`Database::search_videos`].
#[derive(Debug, Clone, serde::Serialize)]
pub struct SearchHit {
pub video_id: String,
pub platform: String,
pub channel: String,
pub title: String,
/// Description excerpt with the matched terms wrapped in `[`…`]`.
pub snippet: String,
}
/// Build a safe FTS5 MATCH expression from free-form user input: each
/// whitespace token becomes a quoted prefix term, AND-ed together. Quoting
/// neutralizes FTS5 operators in the input; the trailing `*` gives
/// type-ahead prefix matching. Returns "" when nothing is searchable.
fn fts_match_expr(query: &str) -> String {
query
.split_whitespace()
.map(|t| t.replace('"', " "))
.map(|t| t.trim().to_string())
.filter(|t| !t.is_empty())
.map(|t| format!("\"{t}\"*"))
.collect::<Vec<_>>()
.join(" ")
}
/// In-memory representation of the `video_flags` table. Each set holds the
/// video IDs that have the named flag enabled — kept small (a few hundred
/// to a few thousand entries in practice).
@ -185,6 +225,24 @@ impl Database {
body TEXT NOT NULL,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (target_kind, target_id)
);
-- Full-text search over the library. `video_search` is a standalone
-- FTS5 index (available in rusqlite's bundled SQLite); `search_meta`
-- tracks each indexed video's mtime so [`Database::sync_search_index`]
-- only re-reads a description sidecar when the video actually
-- changed. video_id/platform are UNINDEXED stored for retrieval,
-- not matched.
CREATE VIRTUAL TABLE IF NOT EXISTS video_search USING fts5(
video_id UNINDEXED,
platform UNINDEXED,
channel,
title,
description,
tokenize = 'porter unicode61'
);
CREATE TABLE IF NOT EXISTS search_meta (
video_id TEXT PRIMARY KEY,
mtime_unix INTEGER NOT NULL
);",
)?;
@ -887,6 +945,94 @@ impl Database {
let _ = conn.execute("DETACH DATABASE bk", []);
result
}
/// Refresh the full-text search index against the current library.
///
/// `entries` is the full set of videos currently on disk. A video whose
/// `mtime_unix` already matches the index is skipped; new/changed videos
/// get their description sidecar re-read and reindexed; videos that
/// vanished from disk are dropped. Returns how many rows were
/// (re)indexed (0 means the index was already current). Runs in one
/// transaction so a crash mid-sync can't leave the index half-written.
pub fn sync_search_index(&self, entries: &[SearchEntry]) -> Result<usize> {
let mut conn = self.conn();
let tx = conn.transaction()?;
// What's already indexed: video_id -> mtime.
let mut existing: HashMap<String, i64> = HashMap::new();
{
let mut stmt = tx.prepare("SELECT video_id, mtime_unix FROM search_meta")?;
let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))?;
for row in rows { let (id, m) = row?; existing.insert(id, m); }
}
let mut seen: HashSet<&str> = HashSet::with_capacity(entries.len());
let mut changed = 0usize;
for e in entries {
seen.insert(e.video_id.as_str());
if existing.get(&e.video_id) == Some(&e.mtime_unix) {
continue; // unchanged — leave the indexed row in place
}
// Only new/changed videos pay the description-read cost.
let description = e.description_path.as_ref()
.and_then(|p| std::fs::read_to_string(p).ok())
.unwrap_or_default();
tx.execute("DELETE FROM video_search WHERE video_id = ?1", [&e.video_id])?;
tx.execute(
"INSERT INTO video_search (video_id, platform, channel, title, description)
VALUES (?1, ?2, ?3, ?4, ?5)",
rusqlite::params![e.video_id, e.platform, e.channel, e.title, description],
)?;
tx.execute(
"INSERT OR REPLACE INTO search_meta (video_id, mtime_unix) VALUES (?1, ?2)",
rusqlite::params![e.video_id, e.mtime_unix],
)?;
changed += 1;
}
// Evict videos that no longer exist on disk.
let stale: Vec<String> = existing.keys()
.filter(|id| !seen.contains(id.as_str()))
.cloned()
.collect();
for id in &stale {
tx.execute("DELETE FROM video_search WHERE video_id = ?1", [id])?;
tx.execute("DELETE FROM search_meta WHERE video_id = ?1", [id])?;
}
tx.commit()?;
Ok(changed)
}
/// Full-text search the library, newest-relevance first. Returns up to
/// `limit` hits, each with a highlighted description snippet. An empty or
/// punctuation-only query yields no results rather than an error.
pub fn search_videos(&self, query: &str, limit: usize) -> Result<Vec<SearchHit>> {
let match_expr = fts_match_expr(query);
if match_expr.is_empty() { return Ok(Vec::new()); }
let conn = self.conn();
let mut stmt = conn.prepare(
// STX/ETX (\u{2}/\u{3}) delimit matched terms — control chars
// that won't collide with literal '[' / ']' in a description.
// The UI turns them into highlight markup.
"SELECT video_id, platform, channel, title,
snippet(video_search, 4, char(2), char(3), '…', 12)
FROM video_search
WHERE video_search MATCH ?1
ORDER BY rank
LIMIT ?2",
)?;
let rows = stmt.query_map(rusqlite::params![match_expr, limit as i64], |r| {
Ok(SearchHit {
video_id: r.get(0)?,
platform: r.get(1)?,
channel: r.get(2)?,
title: r.get(3)?,
snippet: r.get(4)?,
})
})?;
rows.collect()
}
}
/// Per-table row counts that landed in the live DB during a restore.
@ -903,6 +1049,65 @@ pub struct RestoreSummary {
pub notes_added: u64,
}
#[cfg(test)]
mod search_tests {
use super::*;
fn entry(id: &str, mtime: i64, channel: &str, title: &str) -> SearchEntry {
SearchEntry {
video_id: id.into(), mtime_unix: mtime,
platform: "channels".into(), channel: channel.into(),
title: title.into(), description_path: None,
}
}
#[test]
fn indexes_searches_and_evicts() {
let db = Database::open_in_memory().unwrap();
let entries = vec![
entry("a", 1, "Rustaceans", "Async Rust deep dive"),
entry("b", 1, "Cooking", "Sourdough bread from scratch"),
];
assert_eq!(db.sync_search_index(&entries).unwrap(), 2);
// Title match.
let hits = db.search_videos("rust", 10).unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].video_id, "a");
// Prefix (type-ahead) match.
assert_eq!(db.search_videos("sourd", 10).unwrap().len(), 1);
// The channel field is searched too.
let hits = db.search_videos("cooking", 10).unwrap();
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].video_id, "b");
// Multi-token is AND-ed.
assert_eq!(db.search_videos("async dive", 10).unwrap().len(), 1);
assert_eq!(db.search_videos("async sourdough", 10).unwrap().len(), 0);
// Re-syncing unchanged entries is a no-op.
assert_eq!(db.sync_search_index(&entries).unwrap(), 0);
// A changed mtime forces a reindex of just that row.
let changed = vec![
entry("a", 2, "Rustaceans", "Async Rust deep dive — updated"),
entry("b", 1, "Cooking", "Sourdough bread from scratch"),
];
assert_eq!(db.sync_search_index(&changed).unwrap(), 1);
assert_eq!(db.search_videos("updated", 10).unwrap().len(), 1);
// Dropping "b" from disk evicts it from the index.
assert_eq!(db.sync_search_index(&changed[..1]).unwrap(), 0);
assert_eq!(db.search_videos("sourdough", 10).unwrap().len(), 0);
// Garbage / empty queries return nothing, not an error.
assert!(db.search_videos("", 10).unwrap().is_empty());
assert!(db.search_videos(" \" ", 10).unwrap().is_empty());
}
}
#[cfg(test)]
mod restore_tests {
use super::*;

View file

@ -684,3 +684,26 @@ fn track_from_path(path: &Path, folder_artist: &str) -> Option<Track> {
file_size: std::fs::metadata(path).ok().map(|m| m.len()),
})
}
/// Flatten a scanned library into [`crate::database::SearchEntry`] rows for
/// the full-text index — every video in every channel and playlist, tagged
/// with its platform + channel name and a pointer to its description sidecar.
/// Shared by both front-ends so the index is built identically.
pub fn build_search_entries(lib: &[Channel]) -> Vec<crate::database::SearchEntry> {
let mut out = Vec::new();
for ch in lib {
let platform = ch.platform.dir_name();
let playlist_videos = ch.playlists.iter().flat_map(|p| p.videos.iter());
for v in ch.videos.iter().chain(playlist_videos) {
out.push(crate::database::SearchEntry {
video_id: v.id.clone(),
mtime_unix: v.mtime_unix.map(|m| m as i64).unwrap_or(0),
platform: platform.to_string(),
channel: ch.name.clone(),
title: v.title.clone(),
description_path: v.description_path.clone(),
});
}
}
out
}

View file

@ -894,6 +894,16 @@ fn bump_library_version(state: &WebState) {
state.library_version.fetch_add(1, Ordering::Relaxed);
}
/// Bring the full-text search index in line with the current library. Cheap
/// after the first build (only new/changed videos re-read a description), so
/// it's fine to call inline after every (re)scan. Errors are logged, not
/// fatal — search degrading is better than a scan failing.
fn refresh_search_index(db: &Database, lib: &[library::Channel]) {
if let Err(e) = db.sync_search_index(&library::build_search_entries(lib)) {
eprintln!("search index sync failed: {e}");
}
}
/// Re-read the password setting from the DB and update the cache. Called
/// after any change that could affect whether a password exists.
fn refresh_password_cache(state: &WebState) {
@ -1861,6 +1871,28 @@ async fn get_comments(
(StatusCode::OK, Json(serde_json::json!({"comments": comments}))).into_response()
}
#[derive(serde::Deserialize)]
struct SearchQuery {
#[serde(default)]
q: String,
limit: Option<usize>,
}
/// `GET /api/search?q=…&limit=…` — full-text search over the library's
/// titles, channel names, and descriptions. Returns ranked hits with a
/// highlighted snippet. The index is kept current by [`refresh_search_index`]
/// after each scan.
async fn get_search(
State(state): State<Arc<WebState>>,
Query(params): Query<SearchQuery>,
) -> impl IntoResponse {
let limit = params.limit.unwrap_or(50).clamp(1, 200);
match state.db.search_videos(&params.q, limit) {
Ok(results) => Json(serde_json::json!({ "results": results })).into_response(),
Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, format!("search failed: {e}")).into_response(),
}
}
async fn get_metadata(
State(state): State<Arc<WebState>>,
Path(video_id): Path<String>,
@ -1908,6 +1940,7 @@ async fn post_rescan(State(state): State<Arc<WebState>>) -> impl IntoResponse {
if let Ok(w) = state.db.get_watched() {
*state.watched.lock_recover() = w;
}
refresh_search_index(&state.db, &new_lib);
*state.library.lock_recover() = new_lib;
bump_library_version(&state);
(StatusCode::OK, "rescanned")
@ -1958,6 +1991,7 @@ async fn post_maintenance_remove(
if let Ok(folder_map) = state.db.get_all_channel_assignments() {
library::apply_channel_folders(&mut new_lib, &folder_map);
}
refresh_search_index(&state.db, &new_lib);
*state.library.lock_recover() = new_lib;
bump_library_version(&state);
Json(serde_json::json!({ "removed": removed, "errors": errors }))
@ -2533,6 +2567,7 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) {
}
let watched = db.get_watched().unwrap_or_default();
let positions = db.get_positions().unwrap_or_default();
refresh_search_index(&db, &library);
let mut downloader = Downloader::new(
channels_root.clone(),
@ -2678,6 +2713,7 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) {
.route("/api/description/:id", get(get_description))
.route("/api/chapters/:id", get(get_chapters))
.route("/api/comments/:id", get(get_comments))
.route("/api/search", get(get_search))
.route("/api/metadata/:id", get(get_metadata))
.route("/api/settings", get(get_settings).post(post_settings))
.route("/api/transcode/:id", get(get_transcode))

View file

@ -196,6 +196,7 @@
<option value="size-desc">Largest</option>
</select>
<span id="hdr-stats"></span>
<button onclick="openSearch()" title="Search titles + descriptions (f)">🔍</button>
<button onclick="shufflePlay()" title="Play a random unwatched video">🎲</button>
<button onclick="rescan()" title="Rescan library"></button>
<button id="dl-btn" onclick="openDownloads()" title="Downloads"><span id="dl-badge">0</span></button>
@ -1420,6 +1421,54 @@ function renderComments(){
}
}
/* ── Full-text library search ───────────────────────────────────── */
let ftsSeq=0;
function openSearch(){
if(document.getElementById('fts-modal'))return;
const bg=document.createElement('div');bg.className='modal-bg';bg.id='fts-modal';
bg.onclick=e=>{if(e.target===bg)bg.remove()};
bg.innerHTML=`<div class="modal" style="max-width:720px;width:100%">
<div class="modal-hdr"><h2>🔍 Search library</h2><button onclick="this.closest('.modal-bg').remove()"></button></div>
<input id="fts-input" type="search" placeholder="Search titles, channels, descriptions…" autocomplete="off"
style="width:100%;padding:8px 10px;background:var(--bg);border:1px solid var(--border);border-radius:4px;color:var(--text);font-size:14px">
<div id="fts-status" style="font-size:11px;color:var(--muted);padding:6px 2px"></div>
<div id="fts-results" style="overflow-y:auto;max-height:64vh"></div>
</div>`;
document.body.appendChild(bg);
const input=document.getElementById('fts-input');
input.focus();
let timer=null;
input.oninput=()=>{clearTimeout(timer);timer=setTimeout(()=>runFtsSearch(input.value.trim()),180)};
}
async function runFtsSearch(q){
const my=++ftsSeq;
const status=document.getElementById('fts-status');
const box=document.getElementById('fts-results');
if(!box)return;
if(!q){box.innerHTML='';if(status)status.textContent='';return}
if(status)status.textContent='Searching…';
try{
const r=await(await api('/api/search?limit=100&q='+encodeURIComponent(q))).json();
if(my!==ftsSeq)return; // a newer keystroke superseded this response
const res=r.results||[];
if(!res.length){box.innerHTML='<div style="color:var(--muted);padding:12px">No matches.</div>';if(status)status.textContent='0 results';return}
box.innerHTML=res.map(h=>`<div onclick="ftsOpen('${esc(h.video_id)}')" style="padding:8px 10px;border-bottom:1px solid var(--border);cursor:pointer">
<div style="font-size:13px">${esc(h.title)}</div>
<div style="font-size:11px;color:var(--muted)">${esc(h.channel)} · ${esc(h.platform)}</div>
${h.snippet?`<div style="font-size:12px;color:var(--muted);margin-top:2px">${ftsSnippet(h.snippet)}</div>`:''}
</div>`).join('');
if(status)status.textContent=res.length+' result'+(res.length===1?'':'s');
}catch(e){box.innerHTML=`<div style="color:#f87171;padding:12px">Search failed: ${esc(e.message)}</div>`}
}
// FTS5 wraps matched terms in STX/ETX control chars; escape first, then mark.
function ftsSnippet(s){return esc(s).replace(/\x02/g,'<mark>').replace(/\x03/g,'</mark>')}
function ftsOpen(id){
document.getElementById('fts-modal')?.remove();
if(!findVideo(id)){setStatus('Video not in the loaded library — try Rescan');return}
selectVideo(id);
document.getElementById('details')?.scrollIntoView({behavior:'smooth',block:'nearest'});
}
/* ── Select / Details ───────────────────────────────────────────── */
function selectVideo(id){selectedId=selectedId===id?null:id;renderGrid();renderDetails()}
function closeDetails(){selectedId=null;renderGrid();renderDetails()}
@ -2253,6 +2302,10 @@ document.addEventListener('keydown',(e)=>{
document.getElementById('search')?.focus();
e.preventDefault();
break;
case 'f':
openSearch();
e.preventDefault();
break;
case 'r':
rescan();
e.preventDefault();
@ -2275,7 +2328,8 @@ function showShortcutsHelp(){
bg.innerHTML=`<div class="modal" style="max-width:380px">
<div class="modal-hdr"><h2>Keyboard shortcuts</h2><button onclick="this.closest('.modal-bg').remove()"></button></div>
<table><tbody>
${row('/','Focus search')}
${row('/','Focus filter box')}
${row('f','Search titles + descriptions')}
${row('r','Rescan library')}
${row('d','Open downloads')}
${row('Esc','Close modal / cancel')}