luna/catacomb
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Perceptual-hash dedup: desktop UI + shared pipeline (3.7, part 3)

Browse source 
Extracts the dedup pipeline (mtime-gate -> parallel fingerprint -> prune
-> group) into `fingerprint::rebuild_and_group`, returning groups of file
paths. Both front-ends now call it and map paths to their own review
rows — web's run_dedup is refactored onto it (no behaviour change; the
integration test still passes).

Desktop: the 🩺 Maintenance screen gains a "Similar content (perceptual)"
section — a Scan button that runs the fingerprint job on a background
thread (live "N / total" progress via shared atomics + an mpsc result
channel), grouped results with a recommended-keep marker, and a per-group
"Remove non-recommended copies" that deletes via the existing
remove_files path and re-scans. Closes the desktop/web parity gap for
content-aware dedup.

118 tests pass.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Luna 2026-06-07 06:23:59 -07:00
parent f30e32707e
commit 4136a9468b
3 changed files with 243 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

58
src/web.rs
View file

@ -2090,47 +2090,23 @@ fn run_dedup(
by_path: HashMap<String, SimilarVideo>,
valid_paths: HashSet<String>,
) {
use crate::fingerprint;
let outcome: Result<Vec<SimilarGroup>, String> = (|| {
let known = db.fingerprint_mtimes().map_err(|e| e.to_string())?;
let todo: Vec<fingerprint::FpInput> = inputs
.into_iter()
.filter(|i| known.get(&i.path.display().to_string()) != Some(&i.mtime_unix))
.collect();
dedup.total.store(todo.len(), Ordering::Relaxed);
let workers = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4);
let computed = fingerprint::compute_batch(todo, workers, &dedup.done);
for c in &computed {
let _ = db.upsert_fingerprint(
&c.input.path.display().to_string(), c.input.mtime_unix,
&c.input.video_id, c.input.duration_secs, &c.hashes,
);
}
let _ = db.prune_fingerprints(&valid_paths);
let stored = db.load_fingerprints().map_err(|e| e.to_string())?;
let records: Vec<fingerprint::FpRecord> = stored.iter().map(|s| fingerprint::FpRecord {
video_id: s.video_id.clone(), duration_secs: s.duration_secs, hashes: s.hashes.clone(),
}).collect();
let groups_idx = fingerprint::group_similar(
&records, fingerprint::DEFAULT_DUR_TOL,
fingerprint::DEFAULT_MAX_HAM, fingerprint::DEFAULT_MIN_MATCH,
);
let mut groups = Vec::new();
for g in groups_idx {
let mut vids: Vec<SimilarVideo> =
g.iter().filter_map(|&i| by_path.get(&stored[i].path).cloned()).collect();
if vids.len() < 2 { continue; } // stale paths dropped out
// Recommend keeping the largest file (best quality) in each group.
let keep = vids.iter().enumerate()
.max_by_key(|(_, v)| v.file_size.unwrap_or(0)).map(|(i, _)| i);
for (i, v) in vids.iter_mut().enumerate() { v.recommended_keep = Some(i) == keep; }
groups.push(SimilarGroup { videos: vids });
}
Ok(groups)
})();
let workers = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4);
let outcome: Result<Vec<SimilarGroup>, String> =
crate::fingerprint::rebuild_and_group(&db, inputs, &valid_paths, workers, &dedup.done, &dedup.total)
.map(|path_groups| {
let mut groups = Vec::new();
for paths in path_groups {
let mut vids: Vec<SimilarVideo> =
paths.iter().filter_map(|p| by_path.get(p).cloned()).collect();
if vids.len() < 2 { continue; } // stale paths dropped out
// Recommend keeping the largest file (best quality).
let keep = vids.iter().enumerate()
.max_by_key(|(_, v)| v.file_size.unwrap_or(0)).map(|(i, _)| i);
for (i, v) in vids.iter_mut().enumerate() { v.recommended_keep = Some(i) == keep; }
groups.push(SimilarGroup { videos: vids });
}
groups
});
match outcome {
Ok(groups) => *dedup.result.lock_recover() = Some(groups),