Perceptual-hash dedup: web UI + background job (3.7, part 2)

Wires the fingerprint engine into the web Maintenance view.

- web.rs: a DedupState job (one at a time) on its own thread. POST
  /api/maintenance/dedup/scan snapshots the library, mtime-gates against
  stored fingerprints, fingerprints the new/changed videos in parallel
  (progress counter), prunes vanished entries, groups by visual
  similarity, and builds review rows (title/channel/size/files + a
  recommended-keep = largest copy). GET /api/maintenance/dedup/status
  polls progress + results. Deletion reuses /api/maintenance/remove.
- index.html: a "Similar content (perceptual)" section in the Maintenance
  modal — Scan button, live progress bar, grouped results with
  checkboxes (recommended-keep pre-unchecked), and a re-scan that drops
  deleted copies. Poller self-cancels when the modal closes.

Integration test (ffmpeg-gated): generates orig + CRF-38 downscaled
re-encode + an unrelated clip, runs the real scan end-to-end, and asserts
the first two group while the third stays out. 118 tests pass.

Desktop Maintenance UI next.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Luna 2026-06-07 06:17:58 -07:00
parent 6d5c1cae33
commit f30e32707e
3 changed files with 293 additions and 1 deletions

View file

@ -25,6 +25,13 @@ fn have_curl() -> bool {
.status().map(|s| s.success()).unwrap_or(false)
}
/// True if `ffmpeg` is usable — the perceptual-dedup test needs it to both
/// generate fixtures and fingerprint them; it skips otherwise.
fn have_ffmpeg() -> bool {
Command::new("ffmpeg").arg("-version").stdout(Stdio::null()).stderr(Stdio::null())
.status().map(|s| s.success()).unwrap_or(false)
}
/// A running `yt-offline --web` child against a scratch dir. Killed and
/// its dir removed on drop.
struct Server {
@ -351,3 +358,43 @@ fn full_text_search_indexes_titles_and_descriptions() {
assert!(!s.get("/api/search?q=quantumchromodynamics").1.contains("vid123"),
"unrelated query must not hit");
}
#[test]
fn perceptual_dedup_groups_reencodes() {
if !have_curl() { eprintln!("skip: no curl"); return; }
if !have_ffmpeg() { eprintln!("skip: no ffmpeg"); return; }
let s = Server::start();
let chan = s.dir.join("ch/channels/Demo");
std::fs::create_dir_all(&chan).unwrap();
let gen = |args: &[&str]| {
let ok = Command::new("ffmpeg").arg("-nostdin").arg("-y").arg("-loglevel").arg("error")
.args(args).status().map(|st| st.success()).unwrap_or(false);
assert!(ok, "ffmpeg gen failed: {args:?}");
};
let orig = chan.join("orig [aaa].mp4");
let reenc = chan.join("reenc [bbb].mp4");
let diff = chan.join("diff [ccc].mp4");
gen(&["-f","lavfi","-i","testsrc=duration=20:size=640x480:rate=10", orig.to_str().unwrap()]);
gen(&["-i", orig.to_str().unwrap(), "-vf","scale=320:240","-r","15","-c:v","libx264","-crf","38", reenc.to_str().unwrap()]);
gen(&["-f","lavfi","-i","testsrc2=duration=20:size=640x480:rate=10", diff.to_str().unwrap()]);
for stem in ["orig [aaa]", "reenc [bbb]", "diff [ccc]"] {
std::fs::write(chan.join(format!("{stem}.info.json")), r#"{"duration":20.0}"#).unwrap();
}
assert_eq!(s.post("/api/rescan", "").0, 200);
assert_eq!(s.post("/api/maintenance/dedup/scan", "").0, 200);
// Poll until the background job finishes (fingerprinting 3 short clips).
let mut body = String::new();
for _ in 0..160 {
let (_, b) = s.get("/api/maintenance/dedup/status");
if field(&b, "running") == Some("false") { body = b; break; }
std::thread::sleep(std::time::Duration::from_millis(250));
}
assert!(field(&body, "running") == Some("false"), "dedup job never finished: {body}");
// orig + its re-encode group together; the unrelated clip stays out.
assert!(body.contains("aaa"), "group should contain orig: {body}");
assert!(body.contains("bbb"), "group should contain the re-encode: {body}");
assert!(!body.contains("ccc"), "unrelated video must not be grouped: {body}");
}