Perceptual-hash dedup: fingerprint engine + storage (3.7, part 1)
The core of content-aware duplicate detection — finding the same video under a different ID (reupload / mirror / re-encode), which the ID-based maintenance scan can't. - src/fingerprint.rs: samples 6 frames per video via ffmpeg keyframe seek (fast — never full-decodes), downscales to a 9x8 grayscale grid, and dHashes each to a 64-bit frame hash. `group_similar` clusters videos whose frame hashes match within a Hamming threshold, comparing only within a duration-tolerance window (sorted sliding window + union-find) so it stays near-linear instead of O(n²). Parallel `compute_batch` over a hand-rolled scoped thread pool (no rayon), with a progress counter. - src/database.rs: `video_fingerprint` table keyed by (path, mtime) like info_cache — hash once, reuse forever; new downloads hashed as they arrive. Store / load / mtime-gate / prune helpers. Unit tests cover dHash, Hamming, duration-bucketed + transitive grouping, and DB round-trip/replace/prune. An opt-in `real_ffmpeg_groups_reencodes` test (run with --ignored) generates a video + a downscaled CRF-38 re-encode + an unrelated clip and asserts the first two group and the third doesn't — measured ~0.5s/video serial (≈65ms/video across 8 cores). UI wiring (background build + review screen in both front-ends) follows. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Luna
parent
fcd202fde0
commit
6d5c1cae33
3 changed files with 489 additions and 0 deletions
119
src/database.rs
119
src/database.rs
|
|
@ -59,6 +59,15 @@ pub struct SearchHit {
|
|||
pub snippet: String,
|
||||
}
|
||||
|
||||
/// One video's stored perceptual fingerprint, loaded for dedup grouping.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct StoredFingerprint {
|
||||
pub path: String,
|
||||
pub video_id: String,
|
||||
pub duration_secs: f64,
|
||||
pub hashes: Vec<u64>,
|
||||
}
|
||||
|
||||
/// Build a safe FTS5 MATCH expression from free-form user input: each
|
||||
/// whitespace token becomes a quoted prefix term, AND-ed together. Quoting
|
||||
/// neutralizes FTS5 operators in the input; the trailing `*` gives
|
||||
|
|
@ -243,6 +252,17 @@ impl Database {
|
|||
CREATE TABLE IF NOT EXISTS search_meta (
|
||||
video_id TEXT PRIMARY KEY,
|
||||
mtime_unix INTEGER NOT NULL
|
||||
);
|
||||
-- Perceptual fingerprints for content-aware dedup (see
|
||||
-- [`crate::fingerprint`]). Keyed by file path + mtime like
|
||||
-- info_cache, so each video is hashed once and reused. `hashes`
|
||||
-- is a comma-separated list of decimal u64 frame dHashes.
|
||||
CREATE TABLE IF NOT EXISTS video_fingerprint (
|
||||
path TEXT PRIMARY KEY,
|
||||
mtime_unix INTEGER NOT NULL,
|
||||
video_id TEXT NOT NULL,
|
||||
duration_secs REAL,
|
||||
hashes TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
|
||||
|
|
@ -1033,6 +1053,74 @@ impl Database {
|
|||
})?;
|
||||
rows.collect()
|
||||
}
|
||||
|
||||
/// `path -> mtime` for every fingerprinted video, so a caller can skip
|
||||
/// re-hashing files that haven't changed.
|
||||
pub fn fingerprint_mtimes(&self) -> Result<HashMap<String, i64>> {
|
||||
let conn = self.conn();
|
||||
let mut stmt = conn.prepare("SELECT path, mtime_unix FROM video_fingerprint")?;
|
||||
let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?)))?;
|
||||
let mut map = HashMap::new();
|
||||
for row in rows { let (p, m) = row?; map.insert(p, m); }
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
/// Store (or replace) one video's perceptual fingerprint. `hashes` is
|
||||
/// serialized as a comma-separated decimal list.
|
||||
pub fn upsert_fingerprint(
|
||||
&self, path: &str, mtime_unix: i64, video_id: &str,
|
||||
duration_secs: f64, hashes: &[u64],
|
||||
) -> Result<()> {
|
||||
let blob = hashes.iter().map(|h| h.to_string()).collect::<Vec<_>>().join(",");
|
||||
self.conn().execute(
|
||||
"INSERT OR REPLACE INTO video_fingerprint
|
||||
(path, mtime_unix, video_id, duration_secs, hashes)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
rusqlite::params![path, mtime_unix, video_id, duration_secs, blob],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load every stored fingerprint for grouping. Rows with unparsable or
|
||||
/// empty hash lists are skipped (a failed fingerprint shouldn't poison the
|
||||
/// scan).
|
||||
pub fn load_fingerprints(&self) -> Result<Vec<StoredFingerprint>> {
|
||||
let conn = self.conn();
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT path, video_id, duration_secs, hashes FROM video_fingerprint")?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
let hashes: String = r.get(3)?;
|
||||
let hashes = hashes.split(',').filter_map(|s| s.parse::<u64>().ok()).collect::<Vec<_>>();
|
||||
Ok(StoredFingerprint {
|
||||
path: r.get(0)?,
|
||||
video_id: r.get(1)?,
|
||||
duration_secs: r.get::<_, Option<f64>>(2)?.unwrap_or(0.0),
|
||||
hashes,
|
||||
})
|
||||
})?;
|
||||
let mut out = Vec::new();
|
||||
for row in rows { let f = row?; if !f.hashes.is_empty() { out.push(f); } }
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Drop fingerprints whose file path is no longer in `valid_paths`.
|
||||
/// Returns the number evicted.
|
||||
pub fn prune_fingerprints(&self, valid_paths: &HashSet<String>) -> Result<usize> {
|
||||
let conn = self.conn();
|
||||
let existing: Vec<String> = {
|
||||
let mut stmt = conn.prepare("SELECT path FROM video_fingerprint")?;
|
||||
let rows = stmt.query_map([], |r| r.get::<_, String>(0))?;
|
||||
rows.collect::<Result<Vec<_>>>()?
|
||||
};
|
||||
let mut n = 0;
|
||||
for p in existing {
|
||||
if !valid_paths.contains(&p) {
|
||||
conn.execute("DELETE FROM video_fingerprint WHERE path = ?1", [&p])?;
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-table row counts that landed in the live DB during a restore.
|
||||
|
|
@ -1108,6 +1196,37 @@ mod search_tests {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod fingerprint_db_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn store_load_replace_prune() {
|
||||
let db = Database::open_in_memory().unwrap();
|
||||
db.upsert_fingerprint("/a.mkv", 10, "vid-a", 120.0, &[1, 2, 3]).unwrap();
|
||||
db.upsert_fingerprint("/b.mkv", 20, "vid-b", 121.0, &[4, 5, 6]).unwrap();
|
||||
|
||||
let m = db.fingerprint_mtimes().unwrap();
|
||||
assert_eq!(m.get("/a.mkv"), Some(&10));
|
||||
assert_eq!(m.len(), 2);
|
||||
|
||||
let fps = db.load_fingerprints().unwrap();
|
||||
assert_eq!(fps.len(), 2);
|
||||
let a = fps.iter().find(|f| f.video_id == "vid-a").unwrap();
|
||||
assert_eq!(a.hashes, vec![1, 2, 3]);
|
||||
assert_eq!(a.duration_secs, 120.0);
|
||||
|
||||
// Re-upsert replaces (same path PK) and updates the mtime.
|
||||
db.upsert_fingerprint("/a.mkv", 11, "vid-a", 120.0, &[9]).unwrap();
|
||||
assert_eq!(db.fingerprint_mtimes().unwrap().get("/a.mkv"), Some(&11));
|
||||
|
||||
// Prune drops anything not in the keep set.
|
||||
let keep: HashSet<String> = ["/a.mkv".to_string()].into_iter().collect();
|
||||
assert_eq!(db.prune_fingerprints(&keep).unwrap(), 1);
|
||||
assert_eq!(db.load_fingerprints().unwrap().len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod restore_tests {
|
||||
use super::*;
|
||||
|
|
|
|||
369
src/fingerprint.rs
Normal file
369
src/fingerprint.rs
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
//! Perceptual-hash fingerprinting for content-aware duplicate detection.
|
||||
//!
|
||||
//! The library's `maintenance` scan finds duplicates only by yt-dlp video ID
|
||||
//! — it catches "you downloaded this twice", but not the *same content* under
|
||||
//! a different ID (a reupload, a cross-platform mirror, a re-encode). This
|
||||
//! module fingerprints the actual pixels so those can be grouped.
|
||||
//!
|
||||
//! # How it stays fast
|
||||
//!
|
||||
//! - **Sampled frames, fast seek.** Per video we grab [`FRAMES`] frames at
|
||||
//! percentage offsets using ffmpeg *keyframe seek* (`-ss` before `-i`) and
|
||||
//! downscale to a 9×8 grayscale grid during decode — so we never full-decode
|
||||
//! a video, just touch a few keyframes.
|
||||
//! - **dHash.** Each frame becomes a 64-bit difference hash (compare each pixel
|
||||
//! to its right neighbour). Robust to re-encoding, scaling, and minor quality
|
||||
//! changes; sensitive to actually-different content.
|
||||
//! - **Compute once, cache.** Fingerprints are stored in SQLite keyed by
|
||||
//! `(path, mtime)` (see [`crate::database`]), so a video is hashed once and
|
||||
//! skipped forever after — new downloads are hashed as they arrive.
|
||||
//! - **Duration bucketing.** Re-uploads share ~the same runtime, so
|
||||
//! [`group_similar`] only compares videos within a duration tolerance,
|
||||
//! turning an O(n²) all-pairs compare into a near-linear sliding window.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
/// Frames sampled per video. Six spread-out frames is enough to distinguish
|
||||
/// content while keeping the per-video cost to a few fast ffmpeg seeks.
|
||||
pub const FRAMES: usize = 6;
|
||||
|
||||
/// dHash grid width/height. A `(W+1)×H` grayscale image yields `W*H` bits;
|
||||
/// 9×8 → 64 bits, one frame hash per `u64`.
|
||||
const GW: usize = 9;
|
||||
const GH: usize = 8;
|
||||
|
||||
/// Fraction-of-duration offsets at which frames are sampled. Avoids the very
|
||||
/// start/end (intros, outros, black frames) where unrelated videos look alike.
|
||||
const OFFSETS: [f64; FRAMES] = [0.10, 0.25, 0.40, 0.55, 0.70, 0.85];
|
||||
|
||||
/// Compute a difference hash from a `GW*GH` grayscale buffer (row-major).
|
||||
/// Bit *k* is set when a pixel is brighter than its right-hand neighbour.
|
||||
pub fn dhash(gray: &[u8]) -> u64 {
|
||||
debug_assert_eq!(gray.len(), GW * GH);
|
||||
let mut hash = 0u64;
|
||||
let mut bit = 0u32;
|
||||
for y in 0..GH {
|
||||
for x in 0..GW - 1 {
|
||||
if gray[y * GW + x] > gray[y * GW + x + 1] {
|
||||
hash |= 1u64 << bit;
|
||||
}
|
||||
bit += 1;
|
||||
}
|
||||
}
|
||||
hash
|
||||
}
|
||||
|
||||
/// Hamming distance between two frame hashes (number of differing bits).
|
||||
#[inline]
|
||||
pub fn hamming(a: u64, b: u64) -> u32 {
|
||||
(a ^ b).count_ones()
|
||||
}
|
||||
|
||||
/// Extract one frame at `at_secs` as a `GW×GH` grayscale buffer via ffmpeg.
|
||||
/// Returns `None` if ffmpeg is missing, the seek failed, or the output was the
|
||||
/// wrong size. Uses keyframe seek (`-ss` before `-i`) for speed.
|
||||
fn extract_frame_gray(path: &Path, at_secs: f64) -> Option<Vec<u8>> {
|
||||
let out = Command::new("ffmpeg")
|
||||
.arg("-nostdin")
|
||||
.arg("-loglevel").arg("error")
|
||||
.arg("-ss").arg(format!("{at_secs:.3}"))
|
||||
.arg("-i").arg(path)
|
||||
.arg("-frames:v").arg("1")
|
||||
.arg("-an").arg("-sn")
|
||||
.arg("-vf").arg(format!("scale={GW}:{GH}:flags=area,format=gray"))
|
||||
.arg("-f").arg("rawvideo")
|
||||
.arg("-")
|
||||
.stdin(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.output()
|
||||
.ok()?;
|
||||
if out.status.success() && out.stdout.len() >= GW * GH {
|
||||
Some(out.stdout[..GW * GH].to_vec())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Fingerprint a single video: sample [`FRAMES`] frames and hash each. Returns
|
||||
/// the per-frame hashes (possibly fewer than `FRAMES` if some seeks failed, or
|
||||
/// empty when the duration is unusable / ffmpeg can't read the file).
|
||||
pub fn fingerprint(path: &Path, duration_secs: f64) -> Vec<u64> {
|
||||
if !(duration_secs.is_finite() && duration_secs > 1.0) {
|
||||
return Vec::new();
|
||||
}
|
||||
OFFSETS
|
||||
.iter()
|
||||
.filter_map(|f| extract_frame_gray(path, duration_secs * f).map(|g| dhash(&g)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// One video to fingerprint.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FpInput {
|
||||
pub path: PathBuf,
|
||||
pub mtime_unix: i64,
|
||||
pub video_id: String,
|
||||
pub duration_secs: f64,
|
||||
}
|
||||
|
||||
/// Result of fingerprinting one [`FpInput`].
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FpComputed {
|
||||
pub input: FpInput,
|
||||
pub hashes: Vec<u64>,
|
||||
}
|
||||
|
||||
/// Fingerprint many videos in parallel, bumping `progress` after each one so a
|
||||
/// UI can show "N / total". Mirrors the library scanner's hand-rolled worker
|
||||
/// pool (no rayon dependency).
|
||||
pub fn compute_batch(
|
||||
inputs: Vec<FpInput>,
|
||||
n_workers: usize,
|
||||
progress: &std::sync::atomic::AtomicUsize,
|
||||
) -> Vec<FpComputed> {
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
let len = inputs.len();
|
||||
if len == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
let workers = n_workers.max(1).min(len);
|
||||
let inputs: Arc<Vec<Mutex<Option<FpInput>>>> =
|
||||
Arc::new(inputs.into_iter().map(|v| Mutex::new(Some(v))).collect());
|
||||
let results: Arc<Vec<Mutex<Option<FpComputed>>>> =
|
||||
Arc::new((0..len).map(|_| Mutex::new(None)).collect());
|
||||
let next = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..workers {
|
||||
let inputs = inputs.clone();
|
||||
let results = results.clone();
|
||||
let next = next.clone();
|
||||
scope.spawn(move || loop {
|
||||
let i = next.fetch_add(1, Ordering::Relaxed);
|
||||
if i >= len {
|
||||
break;
|
||||
}
|
||||
let input = inputs[i].lock().unwrap().take().unwrap();
|
||||
let hashes = fingerprint(&input.path, input.duration_secs);
|
||||
*results[i].lock().unwrap() = Some(FpComputed { input, hashes });
|
||||
progress.fetch_add(1, Ordering::Relaxed);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
Arc::try_unwrap(results)
|
||||
.unwrap_or_else(|_| unreachable!("scope joined all workers"))
|
||||
.into_iter()
|
||||
.map(|m| m.into_inner().unwrap().unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A stored fingerprint, ready for grouping.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FpRecord {
|
||||
pub video_id: String,
|
||||
pub duration_secs: f64,
|
||||
pub hashes: Vec<u64>,
|
||||
}
|
||||
|
||||
/// Two videos are "similar" when at least `min_match` of their frame hashes
|
||||
/// each find a partner in the other within `max_ham` bits.
|
||||
fn similar(a: &FpRecord, b: &FpRecord, max_ham: u32, min_match: usize) -> bool {
|
||||
if a.hashes.is_empty() || b.hashes.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let need = min_match.min(a.hashes.len()).min(b.hashes.len());
|
||||
let mut matches = 0usize;
|
||||
for &ha in &a.hashes {
|
||||
if b.hashes.iter().any(|&hb| hamming(ha, hb) <= max_ham) {
|
||||
matches += 1;
|
||||
if matches >= need {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Disjoint-set (union-find) for clustering similar videos transitively.
|
||||
struct UnionFind {
|
||||
parent: Vec<usize>,
|
||||
}
|
||||
impl UnionFind {
|
||||
fn new(n: usize) -> Self {
|
||||
Self { parent: (0..n).collect() }
|
||||
}
|
||||
fn find(&mut self, mut x: usize) -> usize {
|
||||
while self.parent[x] != x {
|
||||
self.parent[x] = self.parent[self.parent[x]];
|
||||
x = self.parent[x];
|
||||
}
|
||||
x
|
||||
}
|
||||
fn union(&mut self, a: usize, b: usize) {
|
||||
let (ra, rb) = (self.find(a), self.find(b));
|
||||
if ra != rb {
|
||||
self.parent[ra] = rb;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Group videos that share visual content. Only videos whose durations are
|
||||
/// within `dur_tol` seconds are ever compared (sorted + sliding window), so
|
||||
/// this is near-linear unless a library is full of identical-length clips.
|
||||
/// Returns groups (indices into `records`) of size ≥ 2, largest first.
|
||||
pub fn group_similar(
|
||||
records: &[FpRecord],
|
||||
dur_tol: f64,
|
||||
max_ham: u32,
|
||||
min_match: usize,
|
||||
) -> Vec<Vec<usize>> {
|
||||
let n = records.len();
|
||||
if n < 2 {
|
||||
return Vec::new();
|
||||
}
|
||||
// Index order sorted by duration so we can stop comparing once the next
|
||||
// candidate is outside the tolerance window.
|
||||
let mut order: Vec<usize> = (0..n).collect();
|
||||
order.sort_by(|&a, &b| {
|
||||
records[a]
|
||||
.duration_secs
|
||||
.partial_cmp(&records[b].duration_secs)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
|
||||
let mut uf = UnionFind::new(n);
|
||||
for oi in 0..order.len() {
|
||||
let i = order[oi];
|
||||
for &j in &order[oi + 1..] {
|
||||
if records[j].duration_secs - records[i].duration_secs > dur_tol {
|
||||
break; // sorted: no later candidate can be in range either
|
||||
}
|
||||
if similar(&records[i], &records[j], max_ham, min_match) {
|
||||
uf.union(i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Collect clusters of size ≥ 2.
|
||||
let mut by_root: std::collections::HashMap<usize, Vec<usize>> = std::collections::HashMap::new();
|
||||
for idx in 0..n {
|
||||
let r = uf.find(idx);
|
||||
by_root.entry(r).or_default().push(idx);
|
||||
}
|
||||
let mut groups: Vec<Vec<usize>> = by_root.into_values().filter(|g| g.len() >= 2).collect();
|
||||
groups.sort_by(|a, b| b.len().cmp(&a.len()));
|
||||
groups
|
||||
}
|
||||
|
||||
/// Default tolerances for [`group_similar`]: durations within 3 s, frame
|
||||
/// hashes within 8/64 bits, and at least 3 of 6 frames matching.
|
||||
pub const DEFAULT_DUR_TOL: f64 = 3.0;
|
||||
pub const DEFAULT_MAX_HAM: u32 = 8;
|
||||
pub const DEFAULT_MIN_MATCH: usize = 3;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn dhash_gradients() {
|
||||
// Strictly increasing rows → every pixel < its right neighbour → no
|
||||
// bits set. Strictly decreasing → all 64 bits set.
|
||||
let mut inc = [0u8; GW * GH];
|
||||
let mut dec = [0u8; GW * GH];
|
||||
for y in 0..GH {
|
||||
for x in 0..GW {
|
||||
inc[y * GW + x] = (x * 20) as u8;
|
||||
dec[y * GW + x] = ((GW - x) * 20) as u8;
|
||||
}
|
||||
}
|
||||
assert_eq!(dhash(&inc), 0);
|
||||
assert_eq!(dhash(&dec), u64::MAX >> (64 - (GW - 1) * GH));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hamming_basic() {
|
||||
assert_eq!(hamming(0b1010, 0b1010), 0);
|
||||
assert_eq!(hamming(0b1010, 0b0000), 2);
|
||||
}
|
||||
|
||||
fn rec(id: &str, dur: f64, hashes: &[u64]) -> FpRecord {
|
||||
FpRecord { video_id: id.into(), duration_secs: dur, hashes: hashes.to_vec() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn groups_identical_content() {
|
||||
let h = [1u64, 2, 3, 4, 5, 6];
|
||||
let recs = vec![
|
||||
rec("a", 100.0, &h),
|
||||
rec("b", 101.0, &h), // same hashes, near duration
|
||||
rec("c", 100.5, &[!1, !2, !3, !4]),// very different hashes
|
||||
];
|
||||
let groups = group_similar(&recs, DEFAULT_DUR_TOL, DEFAULT_MAX_HAM, DEFAULT_MIN_MATCH);
|
||||
assert_eq!(groups.len(), 1);
|
||||
let g: Vec<&str> = groups[0].iter().map(|&i| recs[i].video_id.as_str()).collect();
|
||||
assert!(g.contains(&"a") && g.contains(&"b") && !g.contains(&"c"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duration_tolerance_excludes_far_matches() {
|
||||
let h = [10u64, 20, 30, 40, 50, 60];
|
||||
// Identical hashes but durations 100s apart → must not group.
|
||||
let recs = vec![rec("a", 60.0, &h), rec("b", 600.0, &h)];
|
||||
assert!(group_similar(&recs, DEFAULT_DUR_TOL, DEFAULT_MAX_HAM, DEFAULT_MIN_MATCH).is_empty());
|
||||
}
|
||||
|
||||
/// End-to-end check against real ffmpeg: a video and a re-encoded,
|
||||
/// downscaled, quality-degraded copy must group together, while an
|
||||
/// unrelated video must not. Opt-in (needs ffmpeg) — run with
|
||||
/// `cargo test --release real_ffmpeg -- --ignored --nocapture`.
|
||||
#[test]
|
||||
#[ignore = "requires ffmpeg; generates test videos"]
|
||||
fn real_ffmpeg_groups_reencodes() {
|
||||
let dir = std::env::temp_dir().join(format!("ytoff-fp-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let gen = |args: &[&str]| {
|
||||
let ok = Command::new("ffmpeg").arg("-nostdin").arg("-y").arg("-loglevel").arg("error")
|
||||
.args(args).status().map(|s| s.success()).unwrap_or(false);
|
||||
assert!(ok, "ffmpeg gen failed for {args:?}");
|
||||
};
|
||||
let orig = dir.join("orig.mp4");
|
||||
let reenc = dir.join("reenc.mp4");
|
||||
let diff = dir.join("diff.mp4");
|
||||
gen(&["-f","lavfi","-i","testsrc=duration=20:size=640x480:rate=10", orig.to_str().unwrap()]);
|
||||
// Re-encode: downscale + heavy compression + different container framerate.
|
||||
gen(&["-i", orig.to_str().unwrap(), "-vf","scale=320:240","-r","15","-c:v","libx264","-crf","38", reenc.to_str().unwrap()]);
|
||||
gen(&["-f","lavfi","-i","testsrc2=duration=20:size=640x480:rate=10", diff.to_str().unwrap()]);
|
||||
|
||||
let t0 = std::time::Instant::now();
|
||||
let recs = vec![
|
||||
FpRecord { video_id: "orig".into(), duration_secs: 20.0, hashes: fingerprint(&orig, 20.0) },
|
||||
FpRecord { video_id: "reenc".into(), duration_secs: 20.0, hashes: fingerprint(&reenc, 20.0) },
|
||||
FpRecord { video_id: "diff".into(), duration_secs: 20.0, hashes: fingerprint(&diff, 20.0) },
|
||||
];
|
||||
eprintln!("fingerprinted 3 videos in {:?} ({:?}/video)", t0.elapsed(), t0.elapsed() / 3);
|
||||
for r in &recs { assert_eq!(r.hashes.len(), FRAMES, "{} got {} frames", r.video_id, r.hashes.len()); }
|
||||
|
||||
let groups = group_similar(&recs, DEFAULT_DUR_TOL, DEFAULT_MAX_HAM, DEFAULT_MIN_MATCH);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
assert_eq!(groups.len(), 1, "exactly one duplicate group expected, got {groups:?}");
|
||||
let ids: Vec<&str> = groups[0].iter().map(|&i| recs[i].video_id.as_str()).collect();
|
||||
assert!(ids.contains(&"orig") && ids.contains(&"reenc"), "orig+reenc should group: {ids:?}");
|
||||
assert!(!ids.contains(&"diff"), "unrelated video must not be in the group: {ids:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transitive_clustering() {
|
||||
// a~b and b~c (each pair shares ≥3 frames) → one group {a,b,c}.
|
||||
let a = rec("a", 100.0, &[1, 2, 3, 4, 5, 6]);
|
||||
let b = rec("b", 100.0, &[1, 2, 3, 99, 98, 97]);
|
||||
let c = rec("c", 100.0, &[99, 98, 97, 4, 5, 6]);
|
||||
let groups = group_similar(&[a, b, c], DEFAULT_DUR_TOL, 0, 3);
|
||||
assert_eq!(groups.len(), 1);
|
||||
assert_eq!(groups[0].len(), 3);
|
||||
}
|
||||
}
|
||||
|
|
@ -21,6 +21,7 @@ mod disk_space;
|
|||
mod download_options;
|
||||
mod downloader;
|
||||
mod error_class;
|
||||
mod fingerprint;
|
||||
mod library;
|
||||
mod maintenance;
|
||||
mod platform;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue