Perceptual-hash dedup: web UI + background job (3.7, part 2)
Wires the fingerprint engine into the web Maintenance view. - web.rs: a DedupState job (one at a time) on its own thread. POST /api/maintenance/dedup/scan snapshots the library, mtime-gates against stored fingerprints, fingerprints the new/changed videos in parallel (progress counter), prunes vanished entries, groups by visual similarity, and builds review rows (title/channel/size/files + a recommended-keep = largest copy). GET /api/maintenance/dedup/status polls progress + results. Deletion reuses /api/maintenance/remove. - index.html: a "Similar content (perceptual)" section in the Maintenance modal — Scan button, live progress bar, grouped results with checkboxes (recommended-keep pre-unchecked), and a re-scan that drops deleted copies. Poller self-cancels when the modal closes. Integration test (ffmpeg-gated): generates orig + CRF-38 downscaled re-encode + an unrelated clip, runs the real scan end-to-end, and asserts the first two group while the third stays out. 118 tests pass. Desktop Maintenance UI next. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Luna
parent
6d5c1cae33
commit
f30e32707e
3 changed files with 293 additions and 1 deletions
168
src/web.rs
168
src/web.rs
|
|
@ -24,7 +24,7 @@
|
|||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::{Path as StdPath, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use axum::{
|
||||
|
|
@ -133,6 +133,43 @@ pub struct WebState {
|
|||
/// re-serializing. The Arc lets concurrent responses share one
|
||||
/// allocation. Cleared / replaced lazily on the next miss.
|
||||
pub library_body_cache: Mutex<Option<(u64, std::sync::Arc<String>)>>,
|
||||
/// Background perceptual-dedup job state. The expensive fingerprint pass
|
||||
/// runs on its own thread; this carries live progress + the last result
|
||||
/// so `/api/maintenance/dedup/status` can be polled. `Arc` so the worker
|
||||
/// thread can hold a handle past the request that started it.
|
||||
pub dedup: std::sync::Arc<DedupState>,
|
||||
}
|
||||
|
||||
/// Live state for the background perceptual-dedup job (see
|
||||
/// [`post_dedup_scan`]). One job runs at a time, guarded by `running`.
|
||||
#[derive(Default)]
|
||||
pub struct DedupState {
|
||||
running: AtomicBool,
|
||||
done: AtomicUsize,
|
||||
total: AtomicUsize,
|
||||
result: Mutex<Option<Vec<SimilarGroup>>>,
|
||||
error: Mutex<Option<String>>,
|
||||
}
|
||||
|
||||
/// One video in a perceptual-similarity group, with everything the review UI
|
||||
/// needs to show it and (optionally) delete its files.
|
||||
#[derive(serde::Serialize, Clone)]
|
||||
pub struct SimilarVideo {
|
||||
video_id: String,
|
||||
title: String,
|
||||
channel: String,
|
||||
location: String,
|
||||
file_size: Option<u64>,
|
||||
/// Absolute paths of the video + its sidecars, for the delete action.
|
||||
files: Vec<String>,
|
||||
/// The copy the UI recommends keeping (largest file in the group).
|
||||
recommended_keep: bool,
|
||||
}
|
||||
|
||||
/// A cluster of videos that share visual content across different IDs.
|
||||
#[derive(serde::Serialize, Clone)]
|
||||
pub struct SimilarGroup {
|
||||
videos: Vec<SimilarVideo>,
|
||||
}
|
||||
|
||||
/// Failed-login tracking entry. After [`LOGIN_LOCKOUT_AFTER`] failures from
|
||||
|
|
@ -1976,6 +2013,132 @@ struct RemoveRequest {
|
|||
paths: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
/// `POST /api/maintenance/dedup/scan` — start (or no-op if already running)
|
||||
/// the background perceptual-dedup pass. Fingerprints any new/changed videos
|
||||
/// (cached by mtime), then groups by visual similarity. Returns immediately;
|
||||
/// poll `/api/maintenance/dedup/status` for progress + results.
|
||||
async fn post_dedup_scan(State(state): State<Arc<WebState>>) -> impl IntoResponse {
|
||||
if state.dedup.running.swap(true, Ordering::SeqCst) {
|
||||
return Json(serde_json::json!({ "started": false, "running": true }));
|
||||
}
|
||||
state.dedup.done.store(0, Ordering::Relaxed);
|
||||
state.dedup.total.store(0, Ordering::Relaxed);
|
||||
*state.dedup.error.lock_recover() = None;
|
||||
|
||||
// Snapshot what the worker needs from the library (cheap clones of paths +
|
||||
// display strings) so it doesn't hold the lock during the ffmpeg pass.
|
||||
let (inputs, by_path, valid_paths) = {
|
||||
let lib = state.library.lock_recover();
|
||||
let root = state.library_root.clone();
|
||||
let mut inputs: Vec<crate::fingerprint::FpInput> = Vec::new();
|
||||
let mut by_path: HashMap<String, SimilarVideo> = HashMap::new();
|
||||
let mut valid_paths: HashSet<String> = HashSet::new();
|
||||
for ch in lib.iter() {
|
||||
let channel = ch.name.clone();
|
||||
let videos = ch.videos.iter().chain(ch.playlists.iter().flat_map(|p| p.videos.iter()));
|
||||
for v in videos {
|
||||
let Some(vp) = &v.video_path else { continue }; // need a real file
|
||||
let path_str = vp.display().to_string();
|
||||
valid_paths.insert(path_str.clone());
|
||||
inputs.push(crate::fingerprint::FpInput {
|
||||
path: vp.clone(),
|
||||
mtime_unix: v.mtime_unix.map(|m| m as i64).unwrap_or(0),
|
||||
video_id: v.id.clone(),
|
||||
duration_secs: v.duration_secs.unwrap_or(0.0),
|
||||
});
|
||||
let mut files = vec![path_str.clone()];
|
||||
for p in [v.thumb_path.as_ref(), v.info_path.as_ref(), v.description_path.as_ref()]
|
||||
.into_iter().flatten() { files.push(p.display().to_string()); }
|
||||
for s in &v.subtitles { files.push(s.path.display().to_string()); }
|
||||
let location = vp.parent()
|
||||
.map(|d| d.strip_prefix(&root).unwrap_or(d).display().to_string())
|
||||
.unwrap_or_default();
|
||||
by_path.insert(path_str, SimilarVideo {
|
||||
video_id: v.id.clone(), title: v.title.clone(), channel: channel.clone(),
|
||||
location, file_size: v.file_size, files, recommended_keep: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
(inputs, by_path, valid_paths)
|
||||
};
|
||||
|
||||
let db = state.db.clone();
|
||||
let dedup = state.dedup.clone();
|
||||
std::thread::spawn(move || run_dedup(db, dedup, inputs, by_path, valid_paths));
|
||||
Json(serde_json::json!({ "started": true }))
|
||||
}
|
||||
|
||||
/// `GET /api/maintenance/dedup/status` — progress + results of the dedup job.
|
||||
async fn get_dedup_status(State(state): State<Arc<WebState>>) -> impl IntoResponse {
|
||||
let d = &state.dedup;
|
||||
Json(serde_json::json!({
|
||||
"running": d.running.load(Ordering::Relaxed),
|
||||
"done": d.done.load(Ordering::Relaxed),
|
||||
"total": d.total.load(Ordering::Relaxed),
|
||||
"groups": d.result.lock_recover().clone(),
|
||||
"error": d.error.lock_recover().clone(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Worker body for the dedup job: mtime-gate, fingerprint the missing videos
|
||||
/// in parallel, prune vanished entries, then group by visual similarity and
|
||||
/// stash the result. Always clears `running` on exit.
|
||||
fn run_dedup(
|
||||
db: Database,
|
||||
dedup: std::sync::Arc<DedupState>,
|
||||
inputs: Vec<crate::fingerprint::FpInput>,
|
||||
by_path: HashMap<String, SimilarVideo>,
|
||||
valid_paths: HashSet<String>,
|
||||
) {
|
||||
use crate::fingerprint;
|
||||
let outcome: Result<Vec<SimilarGroup>, String> = (|| {
|
||||
let known = db.fingerprint_mtimes().map_err(|e| e.to_string())?;
|
||||
let todo: Vec<fingerprint::FpInput> = inputs
|
||||
.into_iter()
|
||||
.filter(|i| known.get(&i.path.display().to_string()) != Some(&i.mtime_unix))
|
||||
.collect();
|
||||
dedup.total.store(todo.len(), Ordering::Relaxed);
|
||||
|
||||
let workers = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4);
|
||||
let computed = fingerprint::compute_batch(todo, workers, &dedup.done);
|
||||
for c in &computed {
|
||||
let _ = db.upsert_fingerprint(
|
||||
&c.input.path.display().to_string(), c.input.mtime_unix,
|
||||
&c.input.video_id, c.input.duration_secs, &c.hashes,
|
||||
);
|
||||
}
|
||||
let _ = db.prune_fingerprints(&valid_paths);
|
||||
|
||||
let stored = db.load_fingerprints().map_err(|e| e.to_string())?;
|
||||
let records: Vec<fingerprint::FpRecord> = stored.iter().map(|s| fingerprint::FpRecord {
|
||||
video_id: s.video_id.clone(), duration_secs: s.duration_secs, hashes: s.hashes.clone(),
|
||||
}).collect();
|
||||
let groups_idx = fingerprint::group_similar(
|
||||
&records, fingerprint::DEFAULT_DUR_TOL,
|
||||
fingerprint::DEFAULT_MAX_HAM, fingerprint::DEFAULT_MIN_MATCH,
|
||||
);
|
||||
|
||||
let mut groups = Vec::new();
|
||||
for g in groups_idx {
|
||||
let mut vids: Vec<SimilarVideo> =
|
||||
g.iter().filter_map(|&i| by_path.get(&stored[i].path).cloned()).collect();
|
||||
if vids.len() < 2 { continue; } // stale paths dropped out
|
||||
// Recommend keeping the largest file (best quality) in each group.
|
||||
let keep = vids.iter().enumerate()
|
||||
.max_by_key(|(_, v)| v.file_size.unwrap_or(0)).map(|(i, _)| i);
|
||||
for (i, v) in vids.iter_mut().enumerate() { v.recommended_keep = Some(i) == keep; }
|
||||
groups.push(SimilarGroup { videos: vids });
|
||||
}
|
||||
Ok(groups)
|
||||
})();
|
||||
|
||||
match outcome {
|
||||
Ok(groups) => *dedup.result.lock_recover() = Some(groups),
|
||||
Err(e) => *dedup.error.lock_recover() = Some(e),
|
||||
}
|
||||
dedup.running.store(false, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// `POST /api/maintenance/remove` — delete the listed duplicate files.
|
||||
/// Paths outside the library root are refused.
|
||||
async fn post_maintenance_remove(
|
||||
|
|
@ -2611,6 +2774,7 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) {
|
|||
progress_tx,
|
||||
login_attempts: Mutex::new(HashMap::new()),
|
||||
library_body_cache: Mutex::new(None),
|
||||
dedup: std::sync::Arc::new(DedupState::default()),
|
||||
});
|
||||
|
||||
// Broadcast progress snapshots to WebSocket subscribers. Ticks fast
|
||||
|
|
@ -2721,6 +2885,8 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) {
|
|||
.route("/api/plex/generate", post(post_plex_generate))
|
||||
.route("/api/maintenance/scan", get(get_maintenance_scan))
|
||||
.route("/api/maintenance/remove", post(post_maintenance_remove))
|
||||
.route("/api/maintenance/dedup/scan", post(post_dedup_scan))
|
||||
.route("/api/maintenance/dedup/status", get(get_dedup_status))
|
||||
.route("/api/maintenance/repair/:id", post(post_maintenance_repair))
|
||||
.route("/api/cookies", get(get_cookies).post(post_cookies).delete(delete_cookies))
|
||||
.route("/api/scheduler/run", post(post_scheduler_run))
|
||||
|
|
|
|||
|
|
@ -2162,7 +2162,86 @@ function renderMaintenance(r){
|
|||
</div>`;
|
||||
}
|
||||
}
|
||||
h+=`<h3 style="font-size:13px;margin:16px 0 8px">Similar content (perceptual)</h3>
|
||||
<div class="settings-hint" style="margin-bottom:8px">Finds the same video re-uploaded under a different ID — mirrors, re-encodes, resolution changes — by comparing sampled frames. The first scan fingerprints your library (a few minutes); after that it's cached, so re-scans are quick.</div>
|
||||
<div id="dedup-area"></div>`;
|
||||
body.innerHTML=h;
|
||||
initDedupArea();
|
||||
}
|
||||
|
||||
/* ── Perceptual dedup (similar content) ─────────────────────────── */
|
||||
let dedupTimer=null;
|
||||
async function initDedupArea(){
|
||||
if(dedupTimer){clearInterval(dedupTimer);dedupTimer=null}
|
||||
try{
|
||||
const s=await(await api('/api/maintenance/dedup/status')).json();
|
||||
if(s.running){renderDedupProgress(s);startDedupPoll();}
|
||||
else if(s.groups){renderSimilarGroups(s.groups);}
|
||||
else dedupIdle(s.error);
|
||||
}catch(e){const a=document.getElementById('dedup-area');if(a)a.innerHTML=`<div style="color:#f87171;font-size:12px">${esc(e.message)}</div>`}
|
||||
}
|
||||
function dedupIdle(err){
|
||||
const a=document.getElementById('dedup-area');if(!a)return;
|
||||
a.innerHTML=`${err?`<div style="color:#f87171;font-size:12px;margin-bottom:6px">Last scan error: ${esc(err)}</div>`:''}
|
||||
<button class="primary" onclick="startDedup(this)">🔍 Scan for similar content</button>`;
|
||||
}
|
||||
async function startDedup(btn){
|
||||
if(btn)btn.disabled=true;
|
||||
try{await api('/api/maintenance/dedup/scan',{method:'POST'});startDedupPoll();}
|
||||
catch(e){setStatus('Error: '+e.message);if(btn)btn.disabled=false}
|
||||
}
|
||||
function startDedupPoll(){
|
||||
if(dedupTimer)clearInterval(dedupTimer);
|
||||
dedupTimer=setInterval(async()=>{
|
||||
if(!document.getElementById('dedup-area')){clearInterval(dedupTimer);dedupTimer=null;return} // modal closed
|
||||
let s;try{s=await(await api('/api/maintenance/dedup/status')).json()}catch{return}
|
||||
if(s.running){renderDedupProgress(s);return}
|
||||
clearInterval(dedupTimer);dedupTimer=null;
|
||||
if(s.error)dedupIdle(s.error); else renderSimilarGroups(s.groups||[]);
|
||||
},800);
|
||||
}
|
||||
function renderDedupProgress(s){
|
||||
const a=document.getElementById('dedup-area');if(!a)return;
|
||||
const pct=s.total?Math.round(100*s.done/s.total):0;
|
||||
a.innerHTML=`<div style="font-size:12px;color:var(--muted);margin-bottom:6px">Fingerprinting ${s.done} / ${s.total||'…'} new videos…</div>
|
||||
<div style="height:6px;background:var(--bg);border-radius:3px;overflow:hidden"><div style="height:100%;width:${pct}%;background:var(--accent);transition:width .3s"></div></div>`;
|
||||
}
|
||||
function renderSimilarGroups(groups){
|
||||
const a=document.getElementById('dedup-area');if(!a)return;
|
||||
let h=`<div style="display:flex;gap:8px;align-items:center;margin-bottom:8px">
|
||||
<span style="font-size:12px;color:var(--muted)">${groups.length} similar group${groups.length===1?'':'s'}</span>
|
||||
<button onclick="startDedup(this)">⟳ Re-scan</button></div>`;
|
||||
if(!groups.length){h+='<div style="color:var(--muted);font-size:12px">No visually-similar duplicates found across different IDs. 🎉</div>';a.innerHTML=h;return}
|
||||
groups.forEach((g,gi)=>{
|
||||
h+=`<div style="border:1px solid var(--border);border-radius:6px;padding:8px;margin-bottom:8px">
|
||||
<div style="font-weight:600;font-size:12px;margin-bottom:6px">Group ${gi+1} — ${g.videos.length} copies</div>`;
|
||||
g.videos.forEach(v=>{
|
||||
const tag=v.recommended_keep?'<span style="color:#4ade80">keep</span>':'<span style="color:#f87171">remove</span>';
|
||||
const size=v.file_size?fmtSize(v.file_size):'no video';
|
||||
h+=`<label style="display:flex;align-items:center;gap:8px;font-size:12px;padding:3px 0">
|
||||
<input type="checkbox" class="sim-chk" data-files='${esc(JSON.stringify(v.files))}' ${v.recommended_keep?'':'checked'}>
|
||||
<span style="flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap" title="${esc(v.location)}">${esc(v.title)} <span style="color:var(--muted)">· ${esc(v.channel)} · ${size} · [${esc(v.video_id)}]</span></span>
|
||||
${tag}
|
||||
</label>`;
|
||||
});
|
||||
h+=`</div>`;
|
||||
});
|
||||
h+=`<button class="primary" onclick="removeSimilar(this)">🗑 Remove checked copies</button>`;
|
||||
a.innerHTML=h;
|
||||
}
|
||||
async function removeSimilar(btn){
|
||||
const chks=[...document.querySelectorAll('.sim-chk:checked')];
|
||||
let paths=[];
|
||||
for(const c of chks){try{paths=paths.concat(JSON.parse(c.dataset.files))}catch{}}
|
||||
if(!paths.length){setStatus('Nothing selected.');return}
|
||||
if(!confirm(`Delete ${paths.length} file(s) across the checked copies? This cannot be undone.`))return;
|
||||
btn.disabled=true;
|
||||
try{
|
||||
const r=await(await api('/api/maintenance/remove',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({paths})})).json();
|
||||
setStatus(`Removed ${r.removed} file(s)`+(r.errors&&r.errors.length?`, ${r.errors.length} error(s)`:''));
|
||||
await loadLibrary();
|
||||
startDedup(); // re-scan (fingerprints cached) so deleted copies drop out
|
||||
}catch(e){setStatus('Error: '+e.message);btn.disabled=false}
|
||||
}
|
||||
|
||||
async function removeDuplicates(btn){
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue