diff --git a/Cargo.toml b/Cargo.toml index 0cfb1c7..3b046a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ axum = { version = "0.7", features = ["macros"] } tokio = { version = "1", features = ["full"] } tokio-util = { version = "0.7", features = ["io"] } tokio-stream = "0.1" -tower-http = { version = "0.5", features = ["cors", "fs"] } +tower-http = { version = "0.5", features = ["cors", "fs", "compression-gzip"] } argon2 = "0.5" rand = "0.8" # File-picker dialog for the desktop GUI. xdg-portal backend keeps it pure-Rust diff --git a/src/downloader.rs b/src/downloader.rs index 56d4589..57bf589 100644 --- a/src/downloader.rs +++ b/src/downloader.rs @@ -184,6 +184,16 @@ impl Downloader { } } + /// Append platform-specific extra flags. Currently only used to embed + /// album art into the audio file on music-first platforms (Bandcamp, + /// SoundCloud), where music players read embedded tags rather than + /// scanning for sidecar JPEGs. + fn apply_platform_extras(&self, platform: Platform, cmd: &mut Command) { + if platform.is_audio_first() { + cmd.arg("--embed-thumbnail"); + } + } + /// Append `--impersonate ` chosen per source platform. Both the /// bundled venv (which pip-installs `curl_cffi`) and a system yt-dlp /// with curl_cffi can satisfy this. Platforms that prefer no @@ -309,10 +319,19 @@ impl Downloader { // Remember the originating URL so re-checks don't have to // guess from the folder name. platform::write_source_url(&dir, &url); - ( - format!("{}/%(title)s [%(id)s]{live_suffix}.%(ext)s", dir.display()), - format!("{}/{}/{}", platform_label, handle, live_label), - ) + // Bandcamp at the bare artist URL is a whole discography. + // Organize each track into its album subfolder so the + // resulting tree mirrors how Bandcamp itself presents the + // catalog. Other platforms keep their flat per-creator layout. + let template = if info.platform == Platform::Bandcamp { + format!( + "{}/%(album|Unknown)s/%(title)s [%(id)s]{live_suffix}.%(ext)s", + dir.display() + ) + } else { + format!("{}/%(title)s [%(id)s]{live_suffix}.%(ext)s", dir.display()) + }; + (template, format!("{}/{}/{}", platform_label, handle, live_label)) } UrlKind::Playlist => ( format!( @@ -371,6 +390,7 @@ impl Downloader { cmd.arg("--download-archive") .arg(archive_path.display().to_string()); self.apply_impersonation(info.platform, &mut cmd); + self.apply_platform_extras(info.platform, &mut cmd); cmd.arg("-o").arg(&out_arg).arg(&url); Self::apply_retry_flags(&mut cmd); @@ -486,6 +506,15 @@ impl Downloader { cmd.env("PATH", new_path); } + // Absolute path that we want to redact out of any log line — yt-dlp + // sometimes echoes the cookie path in errors and that leaks the + // user's home directory into the UI / API responses. + let cookies_abs = std::env::current_dir() + .unwrap_or_else(|_| PathBuf::from(".")) + .join("cookies.txt") + .display() + .to_string(); + thread::spawn(move || { let mut child = match cmd.spawn() { Ok(child) => child, @@ -498,8 +527,10 @@ impl Downloader { if let Some(stderr) = child.stderr.take() { let tx = tx.clone(); + let cookies_abs = cookies_abs.clone(); thread::spawn(move || { for line in BufReader::new(stderr).lines().map_while(Result::ok) { + let line = redact_sensitive(&line, &cookies_abs); let _ = tx.send(Msg::Line(format!("[stderr] {line}"))); } }); @@ -513,6 +544,7 @@ impl Downloader { if line.trim() == "Aborting remaining downloads" { continue; } + let line = redact_sensitive(&line, &cookies_abs); if let Some(p) = parse_progress(&line) { let _ = tx.send(Msg::Progress(p)); } @@ -621,6 +653,18 @@ fn format_compact_utc(unix: u64) -> String { fn is_leap(y: u32) -> bool { (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 } +/// Strip the absolute on-disk path to `cookies.txt` from a log line, leaving +/// the bare filename. yt-dlp occasionally echoes the full path in error +/// messages ("Unable to load cookies from /home/user/.../cookies.txt"); the +/// user's home directory is not something we want to expose in the UI or +/// `/api/progress` responses. +fn redact_sensitive(line: &str, cookies_abs: &str) -> String { + if cookies_abs.is_empty() || !line.contains(cookies_abs) { + return line.to_string(); + } + line.replace(cookies_abs, "cookies.txt") +} + /// Parse a yt-dlp `[download] 42.7% …` line into a `[0.0, 1.0]` fraction. fn parse_progress(line: &str) -> Option { let rest = line.trim_start().strip_prefix("[download]")?.trim_start(); @@ -662,5 +706,20 @@ mod tests { let url = check_url_for_folder("LinusTechTips"); assert_eq!(url, "https://www.youtube.com/@LinusTechTips"); } + + #[test] + fn redact_sensitive_strips_cookie_path() { + let abs = "/home/luna/.config/yt-offline/cookies.txt"; + let input = format!("Unable to load cookies from {abs}"); + let out = redact_sensitive(&input, abs); + assert_eq!(out, "Unable to load cookies from cookies.txt"); + } + + #[test] + fn redact_sensitive_pass_through_when_not_present() { + let abs = "/home/luna/cookies.txt"; + let input = "[download] 47.2% of 100MiB"; + assert_eq!(redact_sensitive(input, abs), input); + } // URL classification tests live in `platform` now — see its tests module. } diff --git a/src/web.rs b/src/web.rs index f5cc5ec..65774b5 100644 --- a/src/web.rs +++ b/src/web.rs @@ -24,7 +24,7 @@ use std::collections::{HashMap, HashSet}; use std::path::{Path as StdPath, PathBuf}; use std::process::Stdio; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use axum::{ @@ -38,6 +38,7 @@ use axum::{ }; use std::net::SocketAddr; use serde::{Deserialize, Serialize}; +use tower_http::compression::CompressionLayer; use tower_http::services::ServeDir; use argon2::{Argon2, PasswordHash, PasswordHasher, PasswordVerifier}; use argon2::password_hash::SaltString; @@ -92,6 +93,12 @@ pub struct WebState { /// Cached "is password required" — refreshed when the password is changed. /// Avoids a DB hit on every request through `auth_middleware`. pub password_required_cache: AtomicBool, + /// Monotonically-incremented version counter; serves as the ETag for + /// `/api/library`. Bumped on any state change that would alter the + /// JSON response (rescan, watched toggle, resume position, maintenance + /// remove). Combined with `If-None-Match` short-circuits the megabytes + /// of library JSON when nothing has changed. + pub library_version: AtomicU64, /// Per-IP failure tracker for [`post_login`]. Each entry is the number of /// recent failures and the instant the lockout (if any) expires. pub login_attempts: Mutex>, @@ -520,7 +527,20 @@ pub fn write_cookies(text: &str) -> Result { if !content.ends_with('\n') { content.push('\n'); } - std::fs::write(cookies_path(), &content).map_err(|e| e.to_string())?; + let path = cookies_path(); + std::fs::write(&path, &content).map_err(|e| e.to_string())?; + // cookies.txt carries live session credentials — tighten the mode so it + // isn't world-readable on multi-user systems. Best-effort, like the + // similar guard on yt-offline.db. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Ok(meta) = std::fs::metadata(&path) { + let mut perms = meta.permissions(); + perms.set_mode(0o600); + let _ = std::fs::set_permissions(&path, perms); + } + } Ok(count_cookies(&content)) } @@ -648,6 +668,15 @@ fn password_required(state: &WebState) -> bool { state.password_required_cache.load(Ordering::Relaxed) } +/// Bump the library-version counter. Callers should invoke this after any +/// state change that would alter `/api/library`'s response (watched flip, +/// resume position, rescan, maintenance remove). The counter is consumed +/// as an `ETag` so well-behaved clients can short-circuit unchanged GETs +/// with `If-None-Match`. +fn bump_library_version(state: &WebState) { + state.library_version.fetch_add(1, Ordering::Relaxed); +} + /// Re-read the password setting from the DB and update the cache. Called /// after any change that could affect whether a password exists. fn refresh_password_cache(state: &WebState) { @@ -805,7 +834,34 @@ async fn get_index() -> impl IntoResponse { ) } -async fn get_library(State(state): State>) -> impl IntoResponse { +async fn get_library( + State(state): State>, + headers: HeaderMap, +) -> Response { + // Conditional GET: short-circuit with 304 when the client's cached + // ETag matches our current library_version. Saves megabytes for + // large libraries on every poll. + let version = state.library_version.load(Ordering::Relaxed); + let etag = format!("\"{}\"", version); + if let Some(client_etag) = headers.get(header::IF_NONE_MATCH).and_then(|v| v.to_str().ok()) { + if client_etag == etag { + return ([ + (header::ETAG, etag.clone()), + (header::CACHE_CONTROL, "no-cache".to_string()), + ], StatusCode::NOT_MODIFIED).into_response(); + } + } + let payload = build_library_payload(&state).await; + ( + [ + (header::ETAG, etag), + (header::CACHE_CONTROL, "no-cache".to_string()), + ], + Json(payload), + ).into_response() +} + +async fn build_library_payload(state: &Arc) -> LibraryResponse { let lib = state.library.lock().unwrap(); let watched = state.watched.lock().unwrap(); let positions = state.positions.lock().unwrap(); @@ -884,7 +940,7 @@ async fn get_library(State(state): State>) -> impl IntoResponse { } }).collect(); - Json(LibraryResponse { channels }) + LibraryResponse { channels } } async fn get_music(State(state): State>) -> impl IntoResponse { @@ -964,6 +1020,8 @@ async fn post_watched( return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(); } if now_watched { watched.insert(video_id); } else { watched.remove(&video_id); } + drop(watched); + bump_library_version(&state); (StatusCode::OK, if now_watched { "watched" } else { "unwatched" }).into_response() } @@ -1214,13 +1272,26 @@ async fn post_resume( Path(video_id): Path, Json(body): Json, ) -> impl IntoResponse { + // Track whether this video had a non-trivial resume position before/after + // so we only bump the library ETag on transitions (joins / leaves the + // "Continue watching" set). Plain position updates fire every few seconds + // during playback — invalidating the cache on each would defeat the + // ETag optimisation. + let mut positions = state.positions.lock().unwrap(); + let was_resumable = positions.get(&video_id).copied().is_some_and(|p| p > 3.0); let db = state.db.lock().unwrap(); if body.position > 3.0 { let _ = db.set_position(&video_id, body.position); - state.positions.lock().unwrap().insert(video_id, body.position); + positions.insert(video_id.clone(), body.position); } else { let _ = db.clear_position(&video_id); - state.positions.lock().unwrap().remove(&video_id); + positions.remove(&video_id); + } + drop(db); + drop(positions); + let now_resumable = body.position > 3.0; + if was_resumable != now_resumable { + bump_library_version(&state); } (StatusCode::OK, "ok") } @@ -1387,6 +1458,7 @@ async fn post_rescan(State(state): State>) -> impl IntoResponse { *state.watched.lock().unwrap() = w; } *state.library.lock().unwrap() = new_lib; + bump_library_version(&state); (StatusCode::OK, "rescanned") } @@ -1430,6 +1502,7 @@ async fn post_maintenance_remove( // Refresh the library so the removed copies disappear from the UI. let new_lib = library::scan_channels(&state.channels_root); *state.library.lock().unwrap() = new_lib; + bump_library_version(&state); Json(serde_json::json!({ "removed": removed, "errors": errors })) } @@ -1607,6 +1680,7 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) { sessions: Mutex::new(HashMap::new()), last_scheduled_check: Mutex::new(None), password_required_cache: AtomicBool::new(password_required_initial), + library_version: AtomicU64::new(1), login_attempts: Mutex::new(HashMap::new()), }); @@ -1684,6 +1758,12 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) { // malicious. Path-specific overrides aren't needed since we have no // legitimate large-upload endpoints. .layer(DefaultBodyLimit::max(4 * 1024 * 1024)) + // Compress JSON responses (gzip). `/api/library` in particular can + // be megabytes for large collections; gzip slices that ~10×. + // ServeDir output (video bytes) is already-compressed media, so the + // overhead would be wasted there — tower_http's compression layer + // skips already-compressed content types automatically. + .layer(CompressionLayer::new().gzip(true)) .layer(middleware::from_fn(security_headers)) .with_state(state); @@ -1937,9 +2017,21 @@ function closeSidebar(){document.getElementById('sidebar').classList.remove('ope async function api(path,opts){const r=await fetch(path,opts);if(!r.ok)throw new Error(await r.text());return r} /* ── Library ────────────────────────────────────────────────────── */ +// Cache the ETag of the most-recent library response so subsequent polls +// can short-circuit with 304 Not Modified when nothing has changed. Saves +// megabytes of JSON for large libraries on every periodic refresh. +let libraryEtag=null; async function loadLibrary(){ try{ - const data=(await(await api('/api/library')).json()); + const opts=libraryEtag?{headers:{'If-None-Match':libraryEtag}}:{}; + const r=await api('/api/library',opts); + if(r.status===304){ + // Nothing changed; keep our existing `library` array. + loadMusic(); + return; + } + libraryEtag=r.headers.get('ETag')||libraryEtag; + const data=await r.json(); library=data.channels; channelUrls=library.map(ch=>ch.channel_url||null); const total=library.reduce((s,c)=>s+c.size_bytes,0);