diff --git a/Cargo.toml b/Cargo.toml index db84823..73df74a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,4 +28,17 @@ ksni = { version = "0.3.4", features = ["tokio"] } libc = "0.2.186" [profile.release] -opt-level = 2 +opt-level = 3 +# Thin LTO does cross-crate inlining without the rust-lld + bundled-sqlite +# link breakage that full `lto = true` triggered (the previous !lto note +# was about full LTO; thin works with the current toolchain). Worth ~5-10% +# on hot paths. +lto = "thin" +# Codegen-units = 1 trades build time (~30s extra) for tighter inlining +# across the whole crate. Worth it for release binaries. +codegen-units = 1 +# Strip debuginfo from release binaries — cuts binary size by ~30 MB +# without affecting runtime. The crash log still captures backtraces at +# runtime; those are less informative without symbols but usable with +# addr2line against the unstripped build artifact. +strip = "debuginfo" diff --git a/PKGBUILD b/PKGBUILD index c4ead2c..d2e034c 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -18,7 +18,11 @@ optdepends=( 'libnotify: desktop notifications when downloads finish' ) makedepends=('rust' 'cargo') -options=('!lto') # rusqlite bundled sqlite cannot be LTO-linked with rust-lld +# Disable makepkg's environment-level LTO so we don't double up with the +# Cargo.toml profile (which sets lto = "thin"). Full LTO via rust-lld +# previously broke the rusqlite bundled-sqlite link; thin LTO in Cargo +# does the cross-crate inlining we actually want. +options=('!lto') source=("git+https://codeberg.org/anassaeneroi/yt-offline.git#branch=main") sha256sums=('SKIP') diff --git a/src/app.rs b/src/app.rs index d504b4a..cccdfad 100644 --- a/src/app.rs +++ b/src/app.rs @@ -250,16 +250,18 @@ impl App { let dir = platform::platform_root(&channels_root, p); let _ = std::fs::create_dir_all(&dir); } - let mut library = library::scan_channels(&channels_root); + // Open the DB first so the library scanner can use info_cache to + // skip re-parsing unchanged info.json sidecars. Cold start still + // pays full cost; every subsequent launch is faster. + let db_path = channels_root.join("yt-offline.db"); + let db = Database::open(&db_path) + .unwrap_or_else(|_| Database::open_in_memory().expect("in-memory db failed")); + let mut library = library::scan_channels_with_cache(&channels_root, Some(&db)); let status = format!( "{} channels, {} videos", library.len(), library.iter().map(|c| c.total_videos()).sum::() ); - - let db_path = channels_root.join("yt-offline.db"); - let db = Database::open(&db_path) - .unwrap_or_else(|_| Database::open_in_memory().expect("in-memory db failed")); // Hydrate per-channel download options + folder assignments from // SQLite onto the scanned library before publishing it to the UI. if let Ok(map) = db.get_all_channel_options() { @@ -294,16 +296,48 @@ impl App { let (thumb_request_tx, thumb_request_rx) = std::sync::mpsc::channel::(); let (thumb_result_tx, thumb_result_rx) = std::sync::mpsc::channel::<(PathBuf, Option)>(); - let ctx = cc.egui_ctx.clone(); - std::thread::spawn(move || { - while let Ok(path) = thumb_request_rx.recv() { - let img = decode_thumbnail_image(&path); - if thumb_result_tx.send((path, img)).is_err() { - break; - } - ctx.request_repaint(); - } - }); + // Decode pool: shared Receiver behind a Mutex (mpsc::Receiver is + // !Sync so we can't hand it directly to multiple threads). Each + // worker spends ~all its time in image::open + resize, so lock + // contention is negligible — there's at most one recv per thumb + // decode and the decode itself is milliseconds. + let thumb_request_rx = std::sync::Arc::new(std::sync::Mutex::new(thumb_request_rx)); + // Size the pool at half the available cores, clamped to [2, 8]. + // Thumbnail decoding is CPU-bound (image crate is pure Rust), so + // more workers help up to the core count; leaving half the cores + // free keeps the rest of the UI + downloads + scans responsive. + let n_workers = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) + .saturating_div(2) + .clamp(2, 8); + for worker_id in 0..n_workers { + let rx = thumb_request_rx.clone(); + let tx = thumb_result_tx.clone(); + let ctx = cc.egui_ctx.clone(); + std::thread::Builder::new() + .name(format!("yt-offline-thumb-{worker_id}")) + .spawn(move || { + loop { + // Hold the lock only for the recv itself; release + // before decoding so other workers can claim the + // next path while we work on this one. + let path = match rx.lock() { + Ok(guard) => match guard.recv() { + Ok(p) => p, + Err(_) => break, + }, + Err(_) => break, + }; + let img = decode_thumbnail_image(&path); + if tx.send((path, img)).is_err() { + break; + } + ctx.request_repaint(); + } + }) + .ok(); + } Self { config, @@ -437,7 +471,7 @@ impl App { } fn rescan(&mut self) { - let mut new_lib = library::scan_channels(&self.channels_root); + let mut new_lib = library::scan_channels_with_cache(&self.channels_root, Some(&self.db)); if let Ok(map) = self.db.get_all_channel_options() { library::apply_channel_options(&mut new_lib, &map); } diff --git a/src/database.rs b/src/database.rs index 2beb20d..55c43af 100644 --- a/src/database.rs +++ b/src/database.rs @@ -52,6 +52,11 @@ const FILE_POOL_SIZE: u32 = 4; /// Construction always returns a pool with at least one usable connection /// and the schema initialised. Subsequent method calls borrow a connection, /// run their query, and return it — no external `Mutex` is needed. +/// +/// `Clone` is cheap: the inner `r2d2::Pool` is an `Arc` so we hand out +/// new references, not new pools. This lets the library scanner take +/// its own handle for per-thread cache lookups during parallel scans. +#[derive(Clone)] pub struct Database { pool: Pool, } @@ -152,6 +157,19 @@ impl Database { folder_id INTEGER NOT NULL, PRIMARY KEY (platform, handle), FOREIGN KEY (folder_id) REFERENCES folders(id) ON DELETE CASCADE + ); + -- Cache of parsed info.json fields keyed by the file's absolute + -- path + mtime. Library scans hit this first; on miss they + -- parse the JSON and upsert here so the next scan is free. + -- The keyed-by-mtime invalidation means yt-dlp re-writing an + -- info.json (e.g. after a metadata refresh) auto-invalidates + -- without explicit eviction. + CREATE TABLE IF NOT EXISTS info_cache ( + path TEXT PRIMARY KEY, + mtime_unix INTEGER NOT NULL, + duration_secs REAL, + has_chapters INTEGER NOT NULL DEFAULT 0, + upload_date TEXT );", )?; Ok(()) @@ -424,6 +442,63 @@ impl Database { Ok(map) } + /// Look up the cached parse of an info.json sidecar. Returns + /// `(duration_secs, has_chapters, upload_date)` if the cache row's + /// `mtime_unix` matches the supplied value (cache hit), or `None` + /// when the row is missing or stale. + /// + /// Used by the library scan hot path. The lookup itself is two SQL + /// columns + an integer compare, dominated by the SQLite call + /// overhead (~microseconds) rather than JSON parsing (~hundreds of + /// microseconds), which is the savings we're harvesting. + pub fn info_cache_get( + &self, + path: &str, + mtime_unix: u64, + ) -> Option<(Option, bool, Option)> { + let conn = self.conn(); + let mut stmt = conn.prepare( + "SELECT mtime_unix, duration_secs, has_chapters, upload_date \ + FROM info_cache WHERE path = ?1", + ).ok()?; + let mut rows = stmt.query([path]).ok()?; + let row = rows.next().ok().flatten()?; + let stored_mtime: i64 = row.get(0).ok()?; + if stored_mtime != mtime_unix as i64 { + return None; + } + let dur: Option = row.get(1).ok()?; + let chap: i64 = row.get(2).ok()?; + let date: Option = row.get(3).ok()?; + Some((dur, chap != 0, date)) + } + + /// Upsert a parsed info.json result into the cache. Called on miss + /// by the library scanner. Errors are swallowed — a cache miss next + /// time costs the same as no cache at all. + pub fn info_cache_put( + &self, + path: &str, + mtime_unix: u64, + duration_secs: Option, + has_chapters: bool, + upload_date: Option<&str>, + ) { + let conn = self.conn(); + let _ = conn.execute( + "INSERT OR REPLACE INTO info_cache \ + (path, mtime_unix, duration_secs, has_chapters, upload_date) \ + VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![ + path, + mtime_unix as i64, + duration_secs, + has_chapters as i64, + upload_date, + ], + ); + } + /// Idempotently merge another yt-offline database into this one. /// /// Designed for "Import library backup…" — the user uploads a snapshot diff --git a/src/library.rs b/src/library.rs index b920559..74877f6 100644 --- a/src/library.rs +++ b/src/library.rs @@ -22,6 +22,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; +use crate::database::Database; use crate::download_options::DownloadOptions; use crate::platform::{self, Platform}; @@ -197,7 +198,18 @@ pub fn find_video<'a>(channels: &'a [Channel], id: &str) -> Option<(&'a Video, & /// Per-channel info.json reads are parallelised across the available CPUs /// because that's where ~all the time goes for large libraries (one fs read /// + one JSON parse per video, multiplied by thousands). -pub fn scan_channels(youtube_root: &Path) -> Vec { +/// Walk the channels directory and return the enriched library tree. +/// +/// `cache` is an optional handle to the SQLite database holding the +/// `info_cache` table; when present, the scanner skips re-parsing +/// info.json sidecars whose `mtime` is unchanged since the last scan. +/// The cache is best-effort; any SQLite error falls through to a fresh +/// parse without complaint. Each parallel scan worker checks out its +/// own r2d2 connection so per-file lookups don't serialise. +/// +/// Pass `None` from contexts that don't have a DB yet (early startup, +/// the in-memory fallback path). +pub fn scan_channels_with_cache(youtube_root: &Path, cache: Option<&Database>) -> Vec { // Gather (platform, channel-folder-name, full-path) across every platform. let mut dirs: Vec<(Platform, String, PathBuf)> = Vec::new(); for &platform in Platform::all() { @@ -216,8 +228,11 @@ pub fn scan_channels(youtube_root: &Path) -> Vec { .map(|n| n.get()) .unwrap_or(4) .min(dirs.len().max(1)); - let mut channels = parallel_map(dirs, n_workers, |(platform, name, path)| { - let (videos, playlists) = scan_channel_dir(&path); + // Hand each worker its own Database handle via .clone() (Arc inside, + // cheap). Workers without a cache get None. + let cache_handle: Option = cache.cloned(); + let mut channels = parallel_map(dirs, n_workers, move |(platform, name, path)| { + let (videos, playlists) = scan_channel_dir_with_cache(&path, cache_handle.as_ref()); if videos.is_empty() && playlists.is_empty() { return None; } let meta = load_channel_meta(&videos); let total_videos_cached = @@ -432,24 +447,47 @@ fn collect_raw_videos(entries: impl Iterator) -> Vec) -> Vec