Bug fixes - Desktop auto-rescan: snapshot was_running before check_notifications updates prev_job_states, so the post-download rescan actually fires. - web::run shutdown: replace `let _ = tx;` (which dropped the sender immediately and short-circuited the shutdown select!) with std::mem::forget(tx); add an idle fallback so the `-> !` is honored. - /api/preview now resolves the bundled yt-dlp path and extends PATH for deno discovery, matching the rest of the downloader. - Scheduler interval clamped to >=1h on read so a manually-edited config.toml with 0 hours can't trigger every tick. - maintenance::is_within canonicalizes the parent + filename when the target is missing; remove_files treats NotFound as success. Security - Session tokens stored with issued-at Instant and pruned past 30 day TTL on every touch (was: unbounded HashSet). - Login rate-limited per source IP: 5 failures → 60s lockout (429). - Secure cookie flag added when X-Forwarded-Proto: https is present. - 4 MiB body size cap via DefaultBodyLimit. - Security headers middleware: CSP, X-Content-Type-Options, X-Frame-Options, Referrer-Policy. - JS safeUrl() defangs javascript:/vbscript:/data:text URLs before interpolation into src= attributes. - Bundled yt-dlp install verifies SHA-256 against the release's SHA2-256SUMS; deno hash printed for visual inspection. Install also reports byte counts every 3s so users see live progress. - yt-offline.db chmod 0600 at open time on Unix. New features - Bundled yt-dlp + deno mode: settings toggle (System/Bundled) plus one-click Install/Update from settings. Ensures executable bit on every launch in case the install chmod was missed. - Download queue with configurable max_concurrent (1–10); pending jobs surfaced in both UIs. - Music download mode: --extract-audio into music/<artist>/, separate sidebar entry, /api/music + /music-files/ endpoints, inline <audio>. - Download quality selector: Best / 1080p / 720p / 480p / 360p. - Sort videos by Newest / Oldest (upload_date from info.json with release_date fallback). - Plex metadata: per-episode .nfo (title, season/episode, aired, runtime, plot from .description), <stem>-thumb.jpg symlink, and show-level tvshow.nfo. - yt-dlp retry/throttle defaults: --retries 30 --fragment-retries 30 --retry-sleep linear=1:30:2 --sleep-requests 1 to ride out the "Connection reset by peer" failures. Performance - password_required cached in AtomicBool; refreshed only on password change. Eliminates a SQLite query per static-file fetch. - Job::log → VecDeque for O(1) front-pop instead of O(n) drain. - Library scan parallelized across cores via stdlib parallel_map. - Web UI poll: 600ms while active, 5s when idle, 2s after errors. - Music scan now recursive (Artist/Album/Track.opus). - percent_encode_segment uses write! to avoid per-byte String allocs. Code quality - 27 unit tests covering parse_progress, parse_stem, extract_after, detect_url_kind, srt_to_vtt, count_cookies, percent_encode_segment, bind_mode_of, hash_password, lang_label, is_within, sanitize, year_of, aired_date, xml_escape. - Unified library::find_video + Channel::all_videos() iterator, replacing three duplicated linear searches. - Downloader::browser wired to --cookies-from-browser as a fallback when no cookies.txt is present. - mpv detection now matches the binary basename, not substring. - Settings modal scroll fix; web settings expose all new fields. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
288 lines
10 KiB
Rust
288 lines
10 KiB
Rust
//! Library health scanning and repair.
|
|
//!
|
|
//! Two kinds of problems are detected over the scanned library:
|
|
//!
|
|
//! * **Duplicates** — the same YouTube video ID stored more than once (either
|
|
//! under different titles in one folder, or across folders). Each duplicate
|
|
//! group lists every copy with a "score" so the UI can recommend keeping the
|
|
//! most complete one and removing the rest.
|
|
//! * **Missing assets** — a downloaded video lacking its thumbnail, `info.json`,
|
|
//! or `.description` sidecar. These can be re-fetched from YouTube with yt-dlp
|
|
//! (subtitles are fetched alongside, since their absence isn't a reliable
|
|
//! signal — many videos legitimately have none).
|
|
//!
|
|
//! Deletion is never automatic: [`scan`] only reports, and [`remove_files`]
|
|
//! refuses any path outside the library root.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use serde::Serialize;
|
|
|
|
use crate::library::{Channel, Video};
|
|
|
|
/// One stored copy of a video (its on-disk files plus a completeness score).
|
|
#[derive(Serialize, Clone)]
|
|
pub struct DuplicateCopy {
|
|
/// Human-readable directory the copy lives in, relative to the library root.
|
|
pub location: String,
|
|
/// Every file belonging to this copy (video + all sidecars).
|
|
pub files: Vec<PathBuf>,
|
|
/// Size of the video file in bytes, if present.
|
|
pub file_size: Option<u64>,
|
|
/// Whether an actual video file (not just sidecars) is present.
|
|
pub has_video: bool,
|
|
/// True for the copy the UI recommends keeping (best score in the group).
|
|
pub recommended_keep: bool,
|
|
}
|
|
|
|
/// A set of copies that all share one video ID.
|
|
#[derive(Serialize, Clone)]
|
|
pub struct DuplicateGroup {
|
|
pub id: String,
|
|
pub title: String,
|
|
pub copies: Vec<DuplicateCopy>,
|
|
}
|
|
|
|
/// A video that is missing one or more sidecar assets.
|
|
#[derive(Serialize, Clone)]
|
|
pub struct MissingAssets {
|
|
pub id: String,
|
|
pub title: String,
|
|
/// Directory containing the video, relative to the library root.
|
|
pub location: String,
|
|
pub missing_thumbnail: bool,
|
|
pub missing_info: bool,
|
|
pub missing_description: bool,
|
|
}
|
|
|
|
/// The full result of a [`scan`].
|
|
#[derive(Serialize, Clone, Default)]
|
|
pub struct HealthReport {
|
|
pub duplicates: Vec<DuplicateGroup>,
|
|
pub missing: Vec<MissingAssets>,
|
|
}
|
|
|
|
/// The directory a video lives in, inferred from whichever path is known.
|
|
fn video_dir(v: &Video) -> Option<PathBuf> {
|
|
v.video_path
|
|
.as_ref()
|
|
.or(v.info_path.as_ref())
|
|
.or(v.thumb_path.as_ref())
|
|
.or(v.description_path.as_ref())
|
|
.or(v.subtitles.first().map(|s| &s.path))
|
|
.and_then(|p| p.parent().map(Path::to_path_buf))
|
|
}
|
|
|
|
/// Display a path relative to `root` (falls back to the full path).
|
|
fn rel(root: &Path, p: &Path) -> String {
|
|
p.strip_prefix(root).unwrap_or(p).display().to_string()
|
|
}
|
|
|
|
/// Every file on disk whose name begins with `<stem>.` in `dir`.
|
|
///
|
|
/// This captures the video plus all sidecars (thumbnail, info.json, description,
|
|
/// subtitles, live_chat.json, …) without listing each suffix explicitly. The
|
|
/// trailing dot prevents matching a different video whose stem is a prefix.
|
|
fn files_for_stem(dir: &Path, stem: &str) -> Vec<PathBuf> {
|
|
let prefix = format!("{stem}.");
|
|
let mut out = Vec::new();
|
|
if let Ok(entries) = std::fs::read_dir(dir) {
|
|
for e in entries.flatten() {
|
|
if e.file_name().to_string_lossy().starts_with(&prefix) {
|
|
out.push(e.path());
|
|
}
|
|
}
|
|
}
|
|
out.sort();
|
|
out
|
|
}
|
|
|
|
/// Scan the library for duplicates and missing assets.
|
|
pub fn scan(root: &Path, channels: &[Channel]) -> HealthReport {
|
|
// Collect every video together with its source channel name.
|
|
let mut all: Vec<&Video> = Vec::new();
|
|
for ch in channels {
|
|
all.extend(ch.videos.iter());
|
|
for pl in &ch.playlists {
|
|
all.extend(pl.videos.iter());
|
|
}
|
|
}
|
|
|
|
// ── Duplicates: group by video ID ──────────────────────────────────────
|
|
let mut by_id: BTreeMap<&str, Vec<&Video>> = BTreeMap::new();
|
|
for v in &all {
|
|
by_id.entry(v.id.as_str()).or_default().push(v);
|
|
}
|
|
|
|
let mut duplicates = Vec::new();
|
|
for (id, vids) in &by_id {
|
|
if vids.len() < 2 {
|
|
continue;
|
|
}
|
|
// Build a copy per video, then drop phantoms with no locatable files
|
|
// (e.g. a stem known only by a leftover live_chat.json).
|
|
let mut copies: Vec<DuplicateCopy> = vids
|
|
.iter()
|
|
.filter_map(|v| {
|
|
let dir = video_dir(v)?;
|
|
let files = files_for_stem(&dir, &v.stem);
|
|
if files.is_empty() {
|
|
return None;
|
|
}
|
|
Some(DuplicateCopy {
|
|
location: rel(root, &dir),
|
|
files,
|
|
file_size: v.file_size,
|
|
has_video: v.video_path.is_some(),
|
|
recommended_keep: false,
|
|
})
|
|
})
|
|
.collect();
|
|
if copies.len() < 2 {
|
|
continue; // not a real duplicate once phantoms are removed
|
|
}
|
|
// Score each copy: a present video file dominates, then file size,
|
|
// then the number of sidecar files. The highest score is kept.
|
|
let best_idx = copies
|
|
.iter()
|
|
.enumerate()
|
|
.max_by_key(|(_, c)| (c.has_video, c.file_size.unwrap_or(0), c.files.len()))
|
|
.map(|(i, _)| i)
|
|
.unwrap_or(0);
|
|
copies[best_idx].recommended_keep = true;
|
|
let title = vids
|
|
.iter()
|
|
.find(|v| v.video_path.is_some())
|
|
.unwrap_or(&vids[0])
|
|
.title
|
|
.clone();
|
|
duplicates.push(DuplicateGroup {
|
|
id: id.to_string(),
|
|
title,
|
|
copies,
|
|
});
|
|
}
|
|
|
|
// ── Missing assets: per video, only for ones with an actual video file ──
|
|
// Dedup by ID so a duplicate group isn't reported many times here.
|
|
let mut seen = std::collections::HashSet::new();
|
|
let mut missing = Vec::new();
|
|
for v in &all {
|
|
if v.video_path.is_none() || !seen.insert(v.id.as_str()) {
|
|
continue;
|
|
}
|
|
let missing_thumbnail = v.thumb_path.is_none();
|
|
let missing_info = v.info_path.is_none();
|
|
let missing_description = v.description_path.is_none();
|
|
if missing_thumbnail || missing_info || missing_description {
|
|
missing.push(MissingAssets {
|
|
id: v.id.clone(),
|
|
title: v.title.clone(),
|
|
location: video_dir(v).as_deref().map(|d| rel(root, d)).unwrap_or_default(),
|
|
missing_thumbnail,
|
|
missing_info,
|
|
missing_description,
|
|
});
|
|
}
|
|
}
|
|
|
|
HealthReport { duplicates, missing }
|
|
}
|
|
|
|
/// True if `target` resolves to a location inside `root`.
|
|
///
|
|
/// `target` may not exist (e.g. the caller is about to delete it). In that
|
|
/// case we canonicalise the parent directory and join the file name back on,
|
|
/// so a deleted file still gets a correct safety verdict instead of a false
|
|
/// "outside library" refusal.
|
|
fn is_within(root: &Path, target: &Path) -> bool {
|
|
let Ok(canon_root) = root.canonicalize() else { return false };
|
|
let canon_target = match target.canonicalize() {
|
|
Ok(p) => p,
|
|
Err(_) => {
|
|
let Some(parent) = target.parent() else { return false };
|
|
let Some(name) = target.file_name() else { return false };
|
|
let Ok(canon_parent) = parent.canonicalize() else { return false };
|
|
canon_parent.join(name)
|
|
}
|
|
};
|
|
canon_target.starts_with(canon_root)
|
|
}
|
|
|
|
/// Delete the given files, refusing any path that escapes `root`.
|
|
///
|
|
/// Returns the number of files removed and a list of human-readable errors
|
|
/// (including refusals for out-of-bounds paths).
|
|
pub fn remove_files(root: &Path, paths: &[PathBuf]) -> (usize, Vec<String>) {
|
|
let mut removed = 0;
|
|
let mut errors = Vec::new();
|
|
for p in paths {
|
|
if !is_within(root, p) {
|
|
errors.push(format!("refused (outside library): {}", p.display()));
|
|
continue;
|
|
}
|
|
match std::fs::remove_file(p) {
|
|
Ok(()) => removed += 1,
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
|
// Already gone — treat as success so a duplicate-delete from
|
|
// a stale UI doesn't look like an error.
|
|
removed += 1;
|
|
}
|
|
Err(e) => errors.push(format!("{}: {e}", p.display())),
|
|
}
|
|
}
|
|
(removed, errors)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::fs;
|
|
|
|
#[test]
|
|
fn is_within_accepts_inside_path() {
|
|
let tmp = std::env::temp_dir().join("yt-offline-iw-test-1");
|
|
let _ = fs::create_dir_all(&tmp);
|
|
let inside = tmp.join("foo.txt");
|
|
let _ = fs::write(&inside, "x");
|
|
assert!(is_within(&tmp, &inside));
|
|
let _ = fs::remove_file(&inside);
|
|
let _ = fs::remove_dir(&tmp);
|
|
}
|
|
|
|
#[test]
|
|
fn is_within_rejects_outside_path() {
|
|
let tmp = std::env::temp_dir().join("yt-offline-iw-test-2");
|
|
let _ = fs::create_dir_all(&tmp);
|
|
// The target's parent canonicalises to /tmp, which doesn't start with tmp.
|
|
let outside = std::env::temp_dir().join("not-our-dir-xyz.txt");
|
|
assert!(!is_within(&tmp, &outside));
|
|
let _ = fs::remove_dir(&tmp);
|
|
}
|
|
|
|
#[test]
|
|
fn is_within_handles_missing_target() {
|
|
// Target doesn't exist; parent dir does and is inside root.
|
|
let tmp = std::env::temp_dir().join("yt-offline-iw-test-3");
|
|
let _ = fs::create_dir_all(&tmp);
|
|
let ghost = tmp.join("does-not-exist.txt");
|
|
assert!(is_within(&tmp, &ghost));
|
|
let _ = fs::remove_dir(&tmp);
|
|
}
|
|
}
|
|
|
|
/// Look up a video's directory and filename stem by ID, for repair targeting.
|
|
/// Returns `(dir, stem)` of the first matching copy with a known location.
|
|
pub fn locate(channels: &[Channel], id: &str) -> Option<(PathBuf, String)> {
|
|
for ch in channels {
|
|
for v in ch.videos.iter().chain(ch.playlists.iter().flat_map(|p| p.videos.iter())) {
|
|
if v.id == id {
|
|
if let Some(dir) = video_dir(v) {
|
|
return Some((dir, v.stem.clone()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
None
|
|
}
|