From 1d720699134b25ca006903aa993ab6e3562396cb Mon Sep 17 00:00:00 2001 From: Luna Date: Thu, 21 May 2026 05:49:55 -0700 Subject: [PATCH] Add full-scan mode, persist password in DB, fix channel re-check URLs, SRT subs, cookie clear, UI fixes - Full-scan download toggle: omits --break-on-existing so all unarchived videos are fetched regardless of order, filling gaps in partial archives - Filter "Aborting remaining downloads" from job logs (break-on-existing noise) - Password hash moved from config.toml to SQLite settings table; survives git checkouts and rebuilds. config.toml and yt-offline.db added to .gitignore; config auto-generated on first run if absent - Channel re-check URLs derived from folder name (/@handle or /channel/UCxxx) instead of info.json's canonical channel_url, preventing duplicate UCxxx folders - SRT subtitles detected by library scanner and served via /api/sub-vtt/* endpoint that converts to WebVTT on the fly; browser element can now play them - Cookie clear button in both desktop and web UI (DELETE /api/cookies) - Per-job X close button on finished download jobs in desktop GUI - Fix widget ID clash when multiple download jobs are shown (push_id per job) - Watched videos excluded from Continue Watching list and badge count - settings table added to SQLite schema (key/value store) Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 6 ++ Cargo.lock | 106 +++++++++++++++++++++-- Cargo.toml | 3 + config.toml | 18 ---- src/app.rs | 209 +++++++++++++++++++++++++++++++--------------- src/config.rs | 4 - src/database.rs | 26 ++++++ src/downloader.rs | 37 +++++++- src/library.rs | 6 +- src/web.rs | 162 +++++++++++++++++++++++++++++------ 10 files changed, 450 insertions(+), 127 deletions(-) delete mode 100644 config.toml diff --git a/.gitignore b/.gitignore index 0adf20d..b9de2fb 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,9 @@ # Sensitive: YouTube session cookies — never commit cookies.txt + +# User-specific runtime config (bind address, password hash, etc.) — never commit +config.toml + +# Runtime database (watched status, positions, password hash) — never commit +yt-offline.db diff --git a/Cargo.lock b/Cargo.lock index 532b285..8b64056 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,6 +223,27 @@ dependencies = [ "libloading", ] +[[package]] +name = "ashpd" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f3f79755c74fd155000314eb349864caa787c6592eace6c6882dad873d9c39" +dependencies = [ + "enumflags2", + "futures-channel", + "futures-util", + "rand 0.9.4", + "raw-window-handle", + "serde", + "serde_repr", + "tokio", + "url", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "zbus 5.15.0", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -903,6 +924,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.11.1", + "block2 0.6.2", + "libc", "objc2 0.6.4", ] @@ -2330,6 +2353,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ "bitflags 2.11.1", + "block2 0.6.2", "objc2 0.6.4", "objc2-core-foundation", "objc2-core-graphics", @@ -2626,7 +2650,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ "base64ct", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -2718,6 +2742,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + [[package]] name = "potential_utf" version = "0.1.5" @@ -2850,8 +2880,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -2861,7 +2901,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -2873,6 +2923,15 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -2912,6 +2971,30 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" +[[package]] +name = "rfd" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2bee61e6cffa4635c72d7d81a84294e28f0930db0ddcb0f66d10244674ebed" +dependencies = [ + "ashpd", + "block2 0.6.2", + "dispatch2", + "js-sys", + "log", + "objc2 0.6.4", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-foundation 0.3.2", + "pollster", + "raw-window-handle", + "urlencoding", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-sys 0.59.0", +] + [[package]] name = "rusqlite" version = "0.31.0" @@ -3477,6 +3560,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.61.2", ] @@ -3736,8 +3820,15 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4848,7 +4939,8 @@ dependencies = [ "eframe", "image", "notify-rust", - "rand", + "rand 0.8.6", + "rfd", "rusqlite", "serde", "serde_json", @@ -4883,7 +4975,7 @@ dependencies = [ "hex", "nix", "ordered-stream", - "rand", + "rand 0.8.6", "serde", "serde_repr", "sha1", @@ -4922,6 +5014,7 @@ dependencies = [ "rustix 1.1.4", "serde", "serde_repr", + "tokio", "tracing", "uds_windows", "uuid", @@ -5136,6 +5229,7 @@ dependencies = [ "endi", "enumflags2", "serde", + "url", "winnow 1.0.2", "zvariant_derive 5.11.0", "zvariant_utils 3.3.1", diff --git a/Cargo.toml b/Cargo.toml index e009a5c..7288d1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,9 @@ tokio-stream = "0.1" tower-http = { version = "0.5", features = ["cors", "fs"] } argon2 = "0.5" rand = "0.8" +# File-picker dialog for the desktop GUI. xdg-portal backend keeps it pure-Rust +# (zbus, no GTK/libdbus build dep), consistent with notify-rust above. +rfd = { version = "0.15", default-features = false, features = ["xdg-portal", "tokio"] } [profile.release] opt-level = 2 diff --git a/config.toml b/config.toml deleted file mode 100644 index 43e629c..0000000 --- a/config.toml +++ /dev/null @@ -1,18 +0,0 @@ -[backup] -directory = "/mnt/InannaBeloved/youtube-backup/channels" - -[player] -command = "mpv" -browser = "firefox" - -[ui] -theme = "dark" - -[scheduler] -enabled = false -interval_hours = 24 - -[web] -port = 8081 -transcode = false -# source_url = "https://codeberg.org/anassaeneroi/yt-offline" # required for AGPL §13 diff --git a/src/app.rs b/src/app.rs index ebb7bc5..63f3007 100644 --- a/src/app.rs +++ b/src/app.rs @@ -70,6 +70,7 @@ pub struct App { show_downloads: bool, show_settings: bool, dl_url: String, + dl_full_scan: bool, textures: HashMap>, thumb_request_tx: Sender, thumb_result_rx: Receiver<(PathBuf, Option)>, @@ -103,6 +104,9 @@ pub struct App { settings_password_input: String, settings_cookies_input: String, settings_cookies_status: String, + // File picker → chosen cookies.txt path arrives here from a dialog thread. + cookies_pick_tx: Sender, + cookies_pick_rx: Receiver, // Maintenance (library health) window show_maintenance: bool, health_report: Option, @@ -141,7 +145,9 @@ impl App { let browser = config.player.browser.clone(); let config_bind = config.web.bind.clone(); - let password_set = config.web.download_password.is_some(); + let password_set = db.get_setting("password_hash").ok().flatten().is_some(); + + let (cookies_pick_tx, cookies_pick_rx) = std::sync::mpsc::channel::(); let (thumb_request_tx, thumb_request_rx) = std::sync::mpsc::channel::(); let (thumb_result_tx, thumb_result_rx) = @@ -169,6 +175,7 @@ impl App { show_downloads: false, show_settings: false, dl_url: String::new(), + dl_full_scan: false, textures: HashMap::new(), thumb_request_tx, thumb_result_rx, @@ -197,6 +204,8 @@ impl App { settings_password_input: String::new(), settings_cookies_input: String::new(), settings_cookies_status: String::new(), + cookies_pick_tx, + cookies_pick_rx, show_maintenance: false, health_report: None, } @@ -488,11 +497,11 @@ impl App { fn run_scheduled_check(&mut self) { let mut count = 0; let urls: Vec = self.library.iter() - .filter_map(|ch| ch.meta.as_ref()?.channel_url.clone()) + .map(|ch| crate::downloader::check_url_for_folder(&ch.name)) .collect(); for url in urls { let kind = detect_url_kind(&url); - self.downloader.start(url, &kind); + self.downloader.start(url, &kind, false); count += 1; } self.status = format!("Scheduled check: started {} channel downloads", count); @@ -576,7 +585,7 @@ impl App { self.settings_bind_mode = crate::web::bind_mode_of(&self.config.web.bind).to_string(); self.settings_password_enabled = - self.config.web.download_password.is_some(); + self.db.get_setting("password_hash").ok().flatten().is_some(); self.settings_password_input.clear(); self.settings_cookies_input.clear(); let (exists, n) = crate::web::cookies_status(); @@ -637,24 +646,17 @@ impl App { let mut pending_ch_download: Option<(String, String)> = None; // (url, channel_name) for i in 0..self.library.len() { - let (name, total, has_playlists, size_bytes, channel_url, url_inferred) = { + let (name, total, has_playlists, size_bytes, channel_url) = { let ch = &self.library[i]; - let meta_url = ch.meta.as_ref().and_then(|m| m.channel_url.clone()); - let (url, inferred) = if let Some(u) = meta_url { - (Some(u), false) - } else { - let folder_url = ch.path.file_name() - .and_then(|n| n.to_str()) - .map(|n| format!("https://www.youtube.com/@{n}")); - (folder_url, true) - }; + // Always derive the check URL from the folder name so yt-dlp + // writes to the existing folder, not a new UCxxx one. + let url = crate::downloader::check_url_for_folder(&ch.name); ( ch.name.clone(), ch.total_videos(), !ch.playlists.is_empty(), Self::channel_total_size(ch), url, - inferred, ) }; @@ -680,12 +682,9 @@ impl App { let url_for_menu = channel_url.clone(); let name_for_menu = name.clone(); resp.context_menu(|ui| { - if let Some(ref url) = url_for_menu { - let mut btn = ui.button("⬇ Check for new videos"); - if url_inferred { - btn = btn.on_hover_text(format!("URL inferred from folder name:\n{url}")); - } - if btn.clicked() { + { + let url = &url_for_menu; + if ui.button("⬇ Check for new videos").clicked() { pending_ch_download = Some((url.clone(), name_for_menu.clone())); ui.close_menu(); } @@ -719,7 +718,7 @@ impl App { // Process deferred right-click download action if let Some((url, ch_name)) = pending_ch_download { let kind = detect_url_kind(&url); - self.downloader.start(url, &kind); + self.downloader.start(url, &kind, self.dl_full_scan); self.status = format!("Checking {} for new videos…", ch_name); } }); @@ -760,11 +759,13 @@ impl App { } } + ui.checkbox(&mut self.dl_full_scan, "Full scan (check every video, fills gaps)"); + let ready = !self.dl_url.trim().is_empty(); if ui.add_enabled(ready, egui::Button::new("⬇ Start download")).clicked() { let url = self.dl_url.trim().to_string(); let dest = dest_preview.clone(); - self.downloader.start(url, &kind); + self.downloader.start(url, &kind, self.dl_full_scan); self.status = format!("Downloading: {dest}"); } @@ -782,40 +783,56 @@ impl App { if self.downloader.jobs.is_empty() { ui.label(egui::RichText::new("Nothing queued yet.").weak()); } + let mut remove_job: Option = None; egui::ScrollArea::vertical().show(ui, |ui| { - for job in self.downloader.jobs.iter().rev() { + let n = self.downloader.jobs.len(); + for i in (0..n).rev() { + let job = &self.downloader.jobs[i]; let (text, color) = match job.state { JobState::Running => ("running", egui::Color32::from_rgb(230, 200, 60)), JobState::Done => ("done", egui::Color32::from_rgb(110, 200, 110)), JobState::Failed => ("failed", egui::Color32::from_rgb(220, 110, 110)), }; - ui.group(|ui| { - ui.horizontal(|ui| { - ui.colored_label(color, text); - ui.label(&job.label); - }); - ui.label(egui::RichText::new(&job.url).small().weak()); - if job.state == JobState::Running { - ui.add(egui::ProgressBar::new(job.progress).show_percentage()); - } - let last = job.log.last().map(String::as_str).unwrap_or(""); - if !last.is_empty() { - ui.label(egui::RichText::new(last).small().monospace()); - } - ui.collapsing("output log", |ui| { - egui::ScrollArea::vertical() - .max_height(180.0) - .auto_shrink([false, true]) - .stick_to_bottom(true) - .show(ui, |ui| { - for line in &job.log { - ui.label(egui::RichText::new(line).small().monospace()); - } - }); + let finished = job.state != JobState::Running; + ui.push_id(i, |ui| { + ui.group(|ui| { + ui.horizontal(|ui| { + ui.colored_label(color, text); + ui.label(&job.label); + if finished { + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + if ui.small_button("✕").clicked() { + remove_job = Some(i); + } + }); + } + }); + ui.label(egui::RichText::new(&job.url).small().weak()); + if job.state == JobState::Running { + ui.add(egui::ProgressBar::new(job.progress).show_percentage()); + } + let last = job.log.last().map(String::as_str).unwrap_or(""); + if !last.is_empty() { + ui.label(egui::RichText::new(last).small().monospace()); + } + ui.collapsing("output log", |ui| { + egui::ScrollArea::vertical() + .max_height(180.0) + .auto_shrink([false, true]) + .stick_to_bottom(true) + .show(ui, |ui| { + for line in &self.downloader.jobs[i].log { + ui.label(egui::RichText::new(line).small().monospace()); + } + }); + }); }); }); } }); + if let Some(i) = remove_job { + self.downloader.remove_job(i); + } }); } @@ -1061,15 +1078,16 @@ impl App { ui.vertical(|ui| { ui.checkbox(&mut self.settings_password_enabled, "require for web downloads"); if self.settings_password_enabled { + let hint = if self.settings_password_input.is_empty() { + "leave blank to keep current" + } else { + "set a password" + }; ui.add( egui::TextEdit::singleline(&mut self.settings_password_input) .password(true) .desired_width(220.0) - .hint_text(if self.config.web.download_password.is_some() { - "leave blank to keep current" - } else { - "set a password" - }), + .hint_text(hint), ); } }); @@ -1094,22 +1112,56 @@ impl App { ui.label("Cookies:"); ui.vertical(|ui| { ui.label(egui::RichText::new(&self.settings_cookies_status).small().weak()); + if ui.button("📁 Choose cookies.txt…").clicked() { + // Run the file dialog off the UI thread; the chosen + // path comes back via cookies_pick_rx (polled in update()). + let tx = self.cookies_pick_tx.clone(); + std::thread::spawn(move || { + if let Ok(rt) = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + let picked = rt.block_on(async { + rfd::AsyncFileDialog::new() + .set_title("Select cookies.txt") + .add_filter("cookies", &["txt"]) + .pick_file() + .await + }); + if let Some(f) = picked { + let _ = tx.send(f.path().to_path_buf()); + } + } + }); + } + ui.label(egui::RichText::new("…or paste below").small().weak()); ui.add( egui::TextEdit::multiline(&mut self.settings_cookies_input) .desired_rows(3) .desired_width(300.0) .hint_text("paste Netscape cookies.txt…"), ); - if ui.button("Update cookies").clicked() { - match crate::web::write_cookies(&self.settings_cookies_input) { - Ok(n) => { - self.settings_cookies_status = format!("{n} cookie(s) loaded"); - self.settings_cookies_input.clear(); - self.status = format!("Cookies updated ({n} entries)"); + ui.horizontal(|ui| { + if ui.button("Update cookies").clicked() { + match crate::web::write_cookies(&self.settings_cookies_input) { + Ok(n) => { + self.settings_cookies_status = format!("{n} cookie(s) loaded"); + self.settings_cookies_input.clear(); + self.status = format!("Cookies updated ({n} entries)"); + } + Err(e) => self.status = format!("Cookies error: {e}"), } - Err(e) => self.status = format!("Cookies error: {e}"), } - } + if ui.button("Clear cookies").clicked() { + match crate::web::clear_cookies() { + Ok(()) => { + self.settings_cookies_status = "no cookies.txt".to_string(); + self.status = "Cookies cleared".to_string(); + } + Err(e) => self.status = format!("Error clearing cookies: {e}"), + } + } + }); ui.label( egui::RichText::new("Export via a browser extension, then paste.") .small() @@ -1134,15 +1186,26 @@ impl App { let bind_changed = new_bind != self.config.web.bind; self.config.web.bind = new_bind; - // Apply the download-password setting. - if !self.settings_password_enabled { - self.config.web.download_password = None; + // Apply the download-password setting to the database. + let pwd_result = if !self.settings_password_enabled { + self.db.set_setting("password_hash", None) } else if !self.settings_password_input.is_empty() { match crate::web::hash_password(&self.settings_password_input) { - Some(hash) => self.config.web.download_password = Some(hash), - None => self.status = "Error hashing password".to_string(), + Some(hash) => { + let r = self.db.set_setting("password_hash", Some(&hash)); + self.settings_password_input.clear(); + r + } + None => { + self.status = "Error hashing password".to_string(); + Ok(()) + } } - self.settings_password_input.clear(); + } else { + Ok(()) + }; + if let Err(e) = pwd_result { + self.status = format!("DB error: {e}"); } match self.config.save(&self.config_path) { @@ -1589,6 +1652,20 @@ impl eframe::App for App { self.textures.insert(path, handle); } + // A cookies.txt was chosen in the file dialog — validate and install it. + while let Ok(path) = self.cookies_pick_rx.try_recv() { + match std::fs::read_to_string(&path) { + Ok(content) => match crate::web::write_cookies(&content) { + Ok(n) => { + self.settings_cookies_status = format!("{n} cookie(s) loaded"); + self.status = format!("Cookies imported ({n} entries)"); + } + Err(e) => self.status = format!("Cookies error: {e}"), + }, + Err(e) => self.status = format!("Could not read {}: {e}", path.display()), + } + } + self.top_bar(ctx); self.channel_panel(ctx); if self.show_downloads { diff --git a/src/config.rs b/src/config.rs index 3936628..c020ae4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -88,9 +88,6 @@ pub struct WebSection { pub transcode: bool, /// Public URL to the source repository, shown in the web UI per AGPL §13. pub source_url: Option, - /// Optional plaintext password required for downloads via web UI. - /// If set, users must provide this password to queue downloads. - pub download_password: Option, } impl Default for WebSection { @@ -100,7 +97,6 @@ impl Default for WebSection { bind: default_web_bind(), transcode: false, source_url: None, - download_password: None, } } } diff --git a/src/database.rs b/src/database.rs index f5313d6..60bb0fc 100644 --- a/src/database.rs +++ b/src/database.rs @@ -8,6 +8,7 @@ //! |---|---|---| //! | `watched` | `video_id` (PK), `watched_at` | Records videos the user has marked watched | //! | `positions` | `video_id` (PK), `position_secs`, `updated_at` | Stores resume positions | +//! | `settings` | `key` (PK), `value` | Persistent app settings (password hash, etc.) | use rusqlite::{Connection, Result}; use std::collections::{HashMap, HashSet}; @@ -45,11 +46,36 @@ impl Database { video_id TEXT PRIMARY KEY, position_secs REAL NOT NULL, updated_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL );", )?; Ok(()) } + pub fn get_setting(&self, key: &str) -> Result> { + let mut stmt = self.conn.prepare("SELECT value FROM settings WHERE key = ?1")?; + let mut rows = stmt.query([key])?; + Ok(rows.next()?.map(|r| r.get(0)).transpose()?) + } + + pub fn set_setting(&self, key: &str, value: Option<&str>) -> Result<()> { + match value { + Some(v) => { + self.conn.execute( + "INSERT OR REPLACE INTO settings (key, value) VALUES (?1, ?2)", + [key, v], + )?; + } + None => { + self.conn.execute("DELETE FROM settings WHERE key = ?1", [key])?; + } + } + Ok(()) + } + pub fn set_watched(&self, video_id: &str, watched: bool) -> Result<()> { if watched { self.conn.execute( diff --git a/src/downloader.rs b/src/downloader.rs index c2f7469..de138c6 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -39,6 +39,21 @@ pub enum UrlKind { Unknown, } +/// Build a YouTube URL from a library folder name that yt-dlp will resolve to +/// the same folder it already downloaded to. +/// +/// Folder names that look like a channel ID (`UC` + 22 chars) use the +/// `/channel/` form; everything else is treated as a handle and gets `/@`. +/// This avoids the mismatch where info.json's canonical `channel_url` field +/// points to `/channel/UCxxx` and yt-dlp then creates a second folder. +pub fn check_url_for_folder(folder_name: &str) -> String { + if folder_name.starts_with("UC") && folder_name.len() == 24 { + format!("https://www.youtube.com/channel/{folder_name}") + } else { + format!("https://www.youtube.com/@{folder_name}") + } +} + /// Classify a YouTube URL into a [`UrlKind`] by inspecting its path. pub fn detect_url_kind(url: &str) -> UrlKind { if url.contains("playlist?list=") { @@ -132,7 +147,13 @@ impl Downloader { /// /// The output path template is derived from `kind` so that channels, /// playlists, and individual videos land in the right sub-directories. - pub fn start(&mut self, url: String, kind: &UrlKind) { + /// + /// When `full_scan` is false (default / incremental mode) `--break-on-existing` + /// is passed so yt-dlp stops as soon as it hits the first already-archived + /// video — fast for routine channel checks. When `full_scan` is true the + /// flag is omitted so every video is checked individually against the + /// download archive; slower, but correctly fills gaps in the history. + pub fn start(&mut self, url: String, kind: &UrlKind, full_scan: bool) { let archive_path = self.channels_root.join("archive.txt"); let (out_arg, label) = match kind { @@ -178,9 +199,11 @@ impl Downloader { .arg("all") .arg("--extractor-args") .arg("youtube:player_client=web") - .arg("--progress") - .arg("--break-on-existing") - .arg("--download-archive") + .arg("--progress"); + if !full_scan { + cmd.arg("--break-on-existing"); + } + cmd.arg("--download-archive") .arg(archive_path.display().to_string()) .arg("--impersonate") .arg("Chrome-146:Macos-26") @@ -254,6 +277,12 @@ impl Downloader { if let Some(stdout) = child.stdout.take() { for line in BufReader::new(stdout).lines().map_while(Result::ok) { + // Suppress the alarming "Aborting remaining downloads" line that + // yt-dlp emits for --break-on-existing; we replace it with a + // friendlier message when we detect exit code 101 below. + if line.trim() == "Aborting remaining downloads" { + continue; + } if let Some(p) = parse_progress(&line) { let _ = tx.send(Msg::Progress(p)); } diff --git a/src/library.rs b/src/library.rs index 42abaf0..8310c75 100644 --- a/src/library.rs +++ b/src/library.rs @@ -213,8 +213,10 @@ fn collect_raw_videos(entries: impl Iterator) -> Vec &'static str { // ── Cookies ───────────────────────────────────────────────────────────────────── +/// Convert SubRip (SRT) subtitle text to WebVTT. +/// +/// The only structural differences are the `WEBVTT` header and the timestamp +/// decimal separator (SRT uses `,`, VTT uses `.`). +fn srt_to_vtt(srt: &str) -> String { + let mut out = String::from("WEBVTT\n\n"); + for line in srt.lines() { + if line.contains("-->") { + out.push_str(&line.replace(',', ".")); + } else { + out.push_str(line); + } + out.push('\n'); + } + out +} + /// Path to the `cookies.txt` yt-dlp reads, resolved against the process working /// directory (the same place `config.toml` lives and where the downloader's /// relative `--cookies cookies.txt` resolves). @@ -463,7 +484,7 @@ fn is_authed(state: &WebState, headers: &HeaderMap) -> bool { /// Whether a download/access password is configured. fn password_required(state: &WebState) -> bool { - state.config.lock().unwrap().web.download_password.is_some() + state.db.lock().unwrap().get_setting("password_hash").ok().flatten().is_some() } #[derive(Deserialize)] @@ -476,7 +497,7 @@ async fn post_login( State(state): State>, Json(body): Json, ) -> Response { - let hash = state.config.lock().unwrap().web.download_password.clone(); + let hash = state.db.lock().unwrap().get_setting("password_hash").ok().flatten(); let Some(hash) = hash else { // No password configured; nothing to authenticate against. return (StatusCode::OK, "no password set").into_response(); @@ -558,7 +579,14 @@ async fn get_library(State(state): State>) -> impl IntoResponse { let thumb_url = v.thumb_path.as_deref().and_then(|p| file_url(root, p)); let subtitles: Vec = v.subtitles.iter() .filter_map(|s| { - let url = file_url(root, &s.path)?; + let is_srt = s.path.extension().and_then(|e| e.to_str()) == Some("srt"); + let url = if is_srt { + // Route SRT through the on-the-fly conversion endpoint. + let rel = s.path.strip_prefix(root).ok()?; + Some(format!("/api/sub-vtt/{}", rel.display())) + } else { + file_url(root, &s.path) + }?; Some(SubtitleInfo { lang: s.lang.clone(), label: lang_label(&s.lang), @@ -619,7 +647,7 @@ async fn post_download( return (StatusCode::BAD_REQUEST, "empty URL").into_response(); } let kind = detect_url_kind(&url); - state.downloader.lock().unwrap().start(url, &kind); + state.downloader.lock().unwrap().start(url, &kind, body.full_scan); (StatusCode::ACCEPTED, "ok").into_response() } @@ -643,7 +671,9 @@ async fn get_settings(State(state): State>) -> impl IntoResponse { let port = cfg.web.port; let current_bind = cfg.web.bind.clone(); let available_binds = get_available_binds(port); - let download_password_required = cfg.web.download_password.is_some(); + drop(cfg); + let download_password_required = + state.db.lock().unwrap().get_setting("password_hash").ok().flatten().is_some(); Json(SettingsPayload { transcode: state.transcode.load(Ordering::Relaxed), source_url, @@ -668,18 +698,6 @@ async fn post_settings( cfg.web.bind = new_addr; } - if let Some(new_pwd) = &body.new_download_password { - if new_pwd.is_empty() { - cfg.web.download_password = None; - } else if let Some(hashed) = hash_password(new_pwd) { - cfg.web.download_password = Some(hashed); - } else { - return (StatusCode::INTERNAL_SERVER_ERROR, "failed to hash password").into_response(); - } - // Password changed: drop all existing sessions so they must re-authenticate. - state.sessions.lock().unwrap().clear(); - } - if let Err(e) = cfg.save(&state.config_path) { return (StatusCode::INTERNAL_SERVER_ERROR, format!("save failed: {e}")).into_response(); } @@ -687,8 +705,26 @@ async fn post_settings( let current_bind = cfg.web.bind.clone(); let port = cfg.web.port; let available_binds = get_available_binds(port); - let download_password_required = cfg.web.download_password.is_some(); drop(cfg); + + if let Some(new_pwd) = &body.new_download_password { + let db = state.db.lock().unwrap(); + let result = if new_pwd.is_empty() { + db.set_setting("password_hash", None) + } else if let Some(hashed) = hash_password(new_pwd) { + db.set_setting("password_hash", Some(&hashed)) + } else { + return (StatusCode::INTERNAL_SERVER_ERROR, "failed to hash password").into_response(); + }; + if let Err(e) = result { + return (StatusCode::INTERNAL_SERVER_ERROR, format!("db error: {e}")).into_response(); + } + // Password changed: drop all existing sessions so they must re-authenticate. + state.sessions.lock().unwrap().clear(); + } + + let download_password_required = + state.db.lock().unwrap().get_setting("password_hash").ok().flatten().is_some(); Json(SettingsPayload { transcode: body.transcode, source_url, @@ -700,6 +736,31 @@ async fn post_settings( }).into_response() } +/// `GET /api/sub-vtt/*path` — serve an SRT subtitle file as WebVTT. +/// +/// The path is relative to the channels root. The file must be within the +/// channels root (path traversal is rejected with 403). +async fn get_sub_vtt( + State(state): State>, + Path(rel): Path, +) -> Response { + let path = state.channels_root.join(&rel); + // Reject path traversal outside the library. + let ok = match (state.channels_root.canonicalize(), path.canonicalize()) { + (Ok(root), Ok(p)) => p.starts_with(root), + _ => false, + }; + if !ok { + return StatusCode::FORBIDDEN.into_response(); + } + let content = match std::fs::read_to_string(&path) { + Ok(s) => s, + Err(_) => return StatusCode::NOT_FOUND.into_response(), + }; + let vtt = srt_to_vtt(&content); + ([(header::CONTENT_TYPE, "text/vtt; charset=utf-8")], vtt).into_response() +} + async fn get_transcode( State(state): State>, Path(id): Path, @@ -969,6 +1030,15 @@ async fn post_maintenance_repair( (StatusCode::ACCEPTED, "repair queued").into_response() } +/// Delete cookies.txt, removing all stored session cookies. +pub fn clear_cookies() -> Result<(), String> { + let p = cookies_path(); + if p.exists() { + std::fs::remove_file(&p).map_err(|e| e.to_string())?; + } + Ok(()) +} + /// `GET /api/cookies` — report whether a cookies file exists and its entry count. async fn get_cookies() -> impl IntoResponse { let (exists, count) = cookies_status(); @@ -988,6 +1058,14 @@ async fn post_cookies(Json(body): Json) -> impl IntoResponse { } } +/// `DELETE /api/cookies` — remove cookies.txt entirely. +async fn delete_cookies() -> impl IntoResponse { + match clear_cookies() { + Ok(()) => Json(serde_json::json!({ "ok": true })).into_response(), + Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, e).into_response(), + } +} + // ── Entry point ─────────────────────────────────────────────────────────────── pub fn run(config: Config) -> ! { @@ -1055,10 +1133,11 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) { .route("/api/metadata/:id", get(get_metadata)) .route("/api/settings", get(get_settings).post(post_settings)) .route("/api/transcode/:id", get(get_transcode)) + .route("/api/sub-vtt/*path", get(get_sub_vtt)) .route("/api/maintenance/scan", get(get_maintenance_scan)) .route("/api/maintenance/remove", post(post_maintenance_remove)) .route("/api/maintenance/repair/:id", post(post_maintenance_repair)) - .route("/api/cookies", get(get_cookies).post(post_cookies)) + .route("/api/cookies", get(get_cookies).post(post_cookies).delete(delete_cookies)) .route("/api/login", post(post_login)) .route("/api/logout", post(post_logout)) .nest_service("/files", ServeDir::new(&channels_root)) @@ -1274,6 +1353,7 @@ const HTML_UI: &str = r#"
+
@@ -1308,7 +1388,7 @@ function setStatus(s){document.getElementById('status').textContent=s} function renderSidebar(){ const el=document.getElementById('sidebar'); const allVids=library.flatMap(ch=>[...ch.videos,...ch.playlists.flatMap(p=>p.videos)]); - const contVids=allVids.filter(v=>v.resume_pos&&v.resume_pos>5); + const contVids=allVids.filter(v=>v.resume_pos&&v.resume_pos>5&&!v.watched); const total=library.reduce((s,c)=>s+c.total_videos,0); let h=`
Library
`; if(contVids.length)h+=`
▶ Continue (${contVids.length})
`; @@ -1340,7 +1420,7 @@ function currentVideos(){ if(showContinue){ for(const ch of library) for(const v of[...ch.videos,...ch.playlists.flatMap(p=>p.videos)]) - if(v.resume_pos&&v.resume_pos>5&&(!q||v.title.toLowerCase().includes(q)||v.id.includes(q))) + if(v.resume_pos&&v.resume_pos>5&&!v.watched&&(!q||v.title.toLowerCase().includes(q)||v.id.includes(q))) vids.push({...v,channel:ch.name}); vids.sort((a,b)=>(b.resume_pos||0)-(a.resume_pos||0)); return vids; @@ -1442,15 +1522,22 @@ async function previewDownload(){ setStatus(''); } } +function fullScan(){return document.getElementById('dl-full-scan')?.checked||false} async function confirmDownload(url,btn){ if(btn)btn.closest('.modal-bg').remove(); try{ - await api('/api/download',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})}); + await api('/api/download',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url,full_scan:fullScan()})}); document.getElementById('dl-url').value='';setStatus('Download queued…') }catch(e){setStatus('Error: '+e.message)} } -async function downloadChannel(url){try{await api('/api/download',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})});setStatus('Checking for new videos…')}catch(e){setStatus('Error: '+e.message)}} -async function downloadChannelByIdx(i){await downloadChannel(channelUrls[i]||'https://www.youtube.com/@'+library[i].name)} +async function downloadChannel(url){try{await api('/api/download',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url,full_scan:fullScan()})});setStatus('Checking for new videos…')}catch(e){setStatus('Error: '+e.message)}} +function checkUrlForChannel(name){ + // Mirror check_url_for_folder in downloader.rs: UC+22 chars = channel ID, else handle + return(/^UC.{22}$/.test(name)) + ?'https://www.youtube.com/channel/'+name + :'https://www.youtube.com/@'+name; +} +async function downloadChannelByIdx(i){await downloadChannel(checkUrlForChannel(library[i].name))} /* ── Rescan ─────────────────────────────────────────────────────── */ async function rescan(){try{await api('/api/rescan',{method:'POST'});await loadLibrary()}catch(e){setStatus('Error: '+e.message)}} @@ -1637,9 +1724,13 @@ async function openSettings(){
${cookiesStatus}
- - -
Export with a browser extension like "Get cookies.txt LOCALLY", then paste. Refresh when downloads start hitting captchas.
+ + +
+ + +
+
Choose a file or paste (e.g. from "Get cookies.txt LOCALLY"). Refresh when downloads start hitting captchas.
${srcRow}
@@ -1651,6 +1742,14 @@ async function openSettings(){ document.body.appendChild(bg); } +function loadCookieFile(input){ + const f=input.files&&input.files[0];if(!f)return; + const r=new FileReader(); + r.onload=()=>{document.getElementById('cf-cookies').value=r.result;setStatus('Loaded '+f.name+' — click Update cookies to save')}; + r.onerror=()=>setStatus('Could not read file'); + r.readAsText(f); +} + async function saveCookies(btn){ const t=document.getElementById('cf-cookies').value; if(!t.trim()){setStatus('Paste cookies first');return} @@ -1664,6 +1763,15 @@ async function saveCookies(btn){ setStatus('Cookies updated ('+d.cookies+' entries)'); }catch(e){setStatus('Cookies error: '+e.message)}finally{btn.disabled=false} } +async function clearCookies(btn){ + if(!confirm('Remove cookies.txt? Downloads requiring login will fail until you add new cookies.'))return; + btn.disabled=true; + try{ + await api('/api/cookies',{method:'DELETE'}); + document.getElementById('cookies-status').textContent='no cookies.txt'; + setStatus('Cookies cleared'); + }catch(e){setStatus('Error: '+e.message)}finally{btn.disabled=false} +} async function saveSettings(btn){ const transcode=document.getElementById('cf-transcode').checked;