feat(db): persist library snapshot as JSON blob keyed by root
Add serde derives to the library structs and a library_snapshot table with save_library_snapshot/load_library_snapshot. Load returns None on a missing row or unparsable JSON (with #[serde(default)] fields tolerating struct evolution). Groundwork for desktop instant startup. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
cfed8118f7
commit
5a55c15209
2 changed files with 154 additions and 5 deletions
122
src/database.rs
122
src/database.rs
|
|
@ -313,6 +313,11 @@ impl Database {
|
||||||
video_id TEXT NOT NULL,
|
video_id TEXT NOT NULL,
|
||||||
duration_secs REAL,
|
duration_secs REAL,
|
||||||
hashes TEXT NOT NULL
|
hashes TEXT NOT NULL
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS library_snapshot (
|
||||||
|
root TEXT PRIMARY KEY,
|
||||||
|
json TEXT NOT NULL,
|
||||||
|
saved_at INTEGER NOT NULL
|
||||||
);",
|
);",
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
|
|
@ -827,6 +832,40 @@ impl Database {
|
||||||
let _ = tx.commit();
|
let _ = tx.commit();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Persist the scanned library as a JSON blob keyed by its root path, so the
|
||||||
|
/// desktop GUI can render it instantly on the next launch before the (slow,
|
||||||
|
/// disk-bound) rescan finishes. Error-swallowing: a failure just means no
|
||||||
|
/// fresh snapshot next launch — non-fatal, like `info_cache_put_many`.
|
||||||
|
pub fn save_library_snapshot(&self, root: &std::path::Path, library: &[crate::library::Channel]) {
|
||||||
|
let Ok(json) = serde_json::to_string(library) else { return };
|
||||||
|
let root = root.display().to_string();
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs() as i64)
|
||||||
|
.unwrap_or(0);
|
||||||
|
let conn = self.conn();
|
||||||
|
let _ = conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO library_snapshot (root, json, saved_at) VALUES (?1, ?2, ?3)",
|
||||||
|
rusqlite::params![root, json, now],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load the last-persisted library for `root`. Returns `None` if there is no
|
||||||
|
/// snapshot row or the stored JSON no longer deserializes (e.g. after a
|
||||||
|
/// struct change) — callers fall back to the scanning state.
|
||||||
|
pub fn load_library_snapshot(&self, root: &std::path::Path) -> Option<Vec<crate::library::Channel>> {
|
||||||
|
let root = root.display().to_string();
|
||||||
|
let conn = self.conn();
|
||||||
|
let json: String = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT json FROM library_snapshot WHERE root = ?1",
|
||||||
|
rusqlite::params![root],
|
||||||
|
|r| r.get(0),
|
||||||
|
)
|
||||||
|
.ok()?;
|
||||||
|
serde_json::from_str(&json).ok()
|
||||||
|
}
|
||||||
|
|
||||||
/// Idempotently merge another catacomb database into this one.
|
/// Idempotently merge another catacomb database into this one.
|
||||||
///
|
///
|
||||||
/// Designed for "Import library backup…" — the user uploads a snapshot
|
/// Designed for "Import library backup…" — the user uploads a snapshot
|
||||||
|
|
@ -1657,3 +1696,86 @@ fn pool_init_to_rusqlite(e: r2d2::Error) -> rusqlite::Error {
|
||||||
// unused-import warning when no caller references it directly.
|
// unused-import warning when no caller references it directly.
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
type _SilenceConnectionImport = Connection;
|
type _SilenceConnectionImport = Connection;
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod library_snapshot_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn library_snapshot_round_trips() {
|
||||||
|
use crate::library::{Channel, Playlist, Video};
|
||||||
|
use crate::platform::Platform;
|
||||||
|
let db = Database::open_in_memory().unwrap();
|
||||||
|
let root = std::path::Path::new("/tmp/lib-root");
|
||||||
|
|
||||||
|
let video = Video {
|
||||||
|
id: "abc123".into(),
|
||||||
|
title: "Test Video".into(),
|
||||||
|
stem: "Test Video [abc123]".into(),
|
||||||
|
video_path: Some("/tmp/lib-root/channels/Chan/Test [abc123].mp4".into()),
|
||||||
|
thumb_path: None,
|
||||||
|
description_path: None,
|
||||||
|
info_path: None,
|
||||||
|
subtitles: vec![],
|
||||||
|
has_live_chat: false,
|
||||||
|
duration_secs: Some(42.0),
|
||||||
|
has_chapters: true,
|
||||||
|
file_size: Some(1024),
|
||||||
|
mtime_unix: Some(1_700_000_000),
|
||||||
|
upload_date: Some("20250101".into()),
|
||||||
|
};
|
||||||
|
let chan = Channel {
|
||||||
|
name: "Chan".into(),
|
||||||
|
path: "/tmp/lib-root/channels/Chan".into(),
|
||||||
|
platform: Platform::YouTube,
|
||||||
|
source_url: Some("https://youtube.com/@chan".into()),
|
||||||
|
videos: vec![video.clone()],
|
||||||
|
playlists: vec![Playlist {
|
||||||
|
name: "PL".into(),
|
||||||
|
path: "/tmp/lib-root/channels/Chan/PL".into(),
|
||||||
|
videos: vec![video.clone()],
|
||||||
|
}],
|
||||||
|
meta: None,
|
||||||
|
total_videos_cached: 2,
|
||||||
|
total_size_cached: 2048,
|
||||||
|
download_options: Default::default(),
|
||||||
|
folder_id: Some(7),
|
||||||
|
};
|
||||||
|
|
||||||
|
let library = vec![chan];
|
||||||
|
db.save_library_snapshot(root, &library);
|
||||||
|
let loaded = db.load_library_snapshot(root).expect("snapshot present");
|
||||||
|
assert_eq!(loaded.len(), 1);
|
||||||
|
assert_eq!(loaded[0].name, "Chan");
|
||||||
|
assert_eq!(loaded[0].videos.len(), 1);
|
||||||
|
assert_eq!(loaded[0].videos[0].id, "abc123");
|
||||||
|
assert_eq!(loaded[0].playlists[0].videos[0].duration_secs, Some(42.0));
|
||||||
|
assert_eq!(loaded[0].folder_id, Some(7));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn library_snapshot_missing_root_is_none() {
|
||||||
|
let db = Database::open_in_memory().unwrap();
|
||||||
|
assert!(db
|
||||||
|
.load_library_snapshot(std::path::Path::new("/nope"))
|
||||||
|
.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn library_snapshot_garbage_json_is_none() {
|
||||||
|
let db = Database::open_in_memory().unwrap();
|
||||||
|
{
|
||||||
|
// The in-memory pool is size 1; release this connection before
|
||||||
|
// load_library_snapshot checks one out, or it deadlocks the pool.
|
||||||
|
let conn = db.conn();
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO library_snapshot (root, json, saved_at) VALUES (?1, ?2, ?3)",
|
||||||
|
rusqlite::params!["/tmp/bad", "{not valid json", 0i64],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
assert!(db
|
||||||
|
.load_library_snapshot(std::path::Path::new("/tmp/bad"))
|
||||||
|
.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,8 @@
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::database::Database;
|
use crate::database::Database;
|
||||||
use crate::download_options::DownloadOptions;
|
use crate::download_options::DownloadOptions;
|
||||||
use crate::platform::{self, Platform};
|
use crate::platform::{self, Platform};
|
||||||
|
|
@ -31,7 +33,7 @@ const AUDIO_EXTS: &[&str] = &["mp3", "m4a", "opus", "flac", "ogg", "wav", "aac"]
|
||||||
const THUMB_EXTS: &[&str] = &["webp", "jpg", "jpeg", "png"];
|
const THUMB_EXTS: &[&str] = &["webp", "jpg", "jpeg", "png"];
|
||||||
|
|
||||||
/// A single WebVTT subtitle track discovered alongside a video file.
|
/// A single WebVTT subtitle track discovered alongside a video file.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Subtitle {
|
pub struct Subtitle {
|
||||||
/// ISO 639-1/2 language code extracted from the `.lang.vtt` filename suffix.
|
/// ISO 639-1/2 language code extracted from the `.lang.vtt` filename suffix.
|
||||||
pub lang: String,
|
pub lang: String,
|
||||||
|
|
@ -53,38 +55,52 @@ pub struct Track {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A fully enriched video entry, ready to serve to the UI.
|
/// A fully enriched video entry, ready to serve to the UI.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Video {
|
pub struct Video {
|
||||||
/// yt-dlp video ID (the part inside `[…]` in the filename).
|
/// yt-dlp video ID (the part inside `[…]` in the filename).
|
||||||
|
#[serde(default)]
|
||||||
pub id: String,
|
pub id: String,
|
||||||
|
#[serde(default)]
|
||||||
pub title: String,
|
pub title: String,
|
||||||
|
#[serde(default)]
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
pub stem: String,
|
pub stem: String,
|
||||||
|
#[serde(default)]
|
||||||
pub video_path: Option<PathBuf>,
|
pub video_path: Option<PathBuf>,
|
||||||
|
#[serde(default)]
|
||||||
pub thumb_path: Option<PathBuf>,
|
pub thumb_path: Option<PathBuf>,
|
||||||
|
#[serde(default)]
|
||||||
pub description_path: Option<PathBuf>,
|
pub description_path: Option<PathBuf>,
|
||||||
/// Path to the `.info.json` sidecar — used to read duration, chapters, etc.
|
/// Path to the `.info.json` sidecar — used to read duration, chapters, etc.
|
||||||
|
#[serde(default)]
|
||||||
pub info_path: Option<PathBuf>,
|
pub info_path: Option<PathBuf>,
|
||||||
|
#[serde(default)]
|
||||||
pub subtitles: Vec<Subtitle>,
|
pub subtitles: Vec<Subtitle>,
|
||||||
|
#[serde(default)]
|
||||||
pub has_live_chat: bool,
|
pub has_live_chat: bool,
|
||||||
/// Duration read from `info.json`; `None` if the sidecar is missing.
|
/// Duration read from `info.json`; `None` if the sidecar is missing.
|
||||||
|
#[serde(default)]
|
||||||
pub duration_secs: Option<f64>,
|
pub duration_secs: Option<f64>,
|
||||||
/// Whether `info.json` lists a non-empty `chapters` array. Cached at scan
|
/// Whether `info.json` lists a non-empty `chapters` array. Cached at scan
|
||||||
/// time so the web layer needn't re-read and parse the sidecar per request.
|
/// time so the web layer needn't re-read and parse the sidecar per request.
|
||||||
|
#[serde(default)]
|
||||||
pub has_chapters: bool,
|
pub has_chapters: bool,
|
||||||
/// Size of the video file on disk; `None` if the video file is missing.
|
/// Size of the video file on disk; `None` if the video file is missing.
|
||||||
|
#[serde(default)]
|
||||||
pub file_size: Option<u64>,
|
pub file_size: Option<u64>,
|
||||||
/// Filesystem mtime of the video file as a UNIX timestamp (seconds).
|
/// Filesystem mtime of the video file as a UNIX timestamp (seconds).
|
||||||
/// Used by the activity feed to surface recent additions. `None` if the
|
/// Used by the activity feed to surface recent additions. `None` if the
|
||||||
/// video file is missing or the system clock returned an error.
|
/// video file is missing or the system clock returned an error.
|
||||||
|
#[serde(default)]
|
||||||
pub mtime_unix: Option<u64>,
|
pub mtime_unix: Option<u64>,
|
||||||
/// Upload date as `YYYYMMDD` (yt-dlp's native format from info.json).
|
/// Upload date as `YYYYMMDD` (yt-dlp's native format from info.json).
|
||||||
/// `None` if the info.json sidecar is missing or lacks the field.
|
/// `None` if the info.json sidecar is missing or lacks the field.
|
||||||
|
#[serde(default)]
|
||||||
pub upload_date: Option<String>,
|
pub upload_date: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A sub-directory inside a channel that contains videos (treated as a playlist).
|
/// A sub-directory inside a channel that contains videos (treated as a playlist).
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Playlist {
|
pub struct Playlist {
|
||||||
pub name: String,
|
pub name: String,
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
|
|
@ -93,7 +109,7 @@ pub struct Playlist {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Channel-level metadata pulled from the first available `info.json`.
|
/// Channel-level metadata pulled from the first available `info.json`.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct ChannelMeta {
|
pub struct ChannelMeta {
|
||||||
pub subscriber_count: Option<u64>,
|
pub subscriber_count: Option<u64>,
|
||||||
pub channel_url: Option<String>,
|
pub channel_url: Option<String>,
|
||||||
|
|
@ -101,34 +117,45 @@ pub struct ChannelMeta {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A top-level channel directory with all its videos and playlists.
|
/// A top-level channel directory with all its videos and playlists.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
pub struct Channel {
|
pub struct Channel {
|
||||||
|
#[serde(default)]
|
||||||
pub name: String,
|
pub name: String,
|
||||||
|
#[serde(default)]
|
||||||
pub path: PathBuf,
|
pub path: PathBuf,
|
||||||
/// Source platform — drives sidebar grouping and the re-check URL.
|
/// Source platform — drives sidebar grouping and the re-check URL.
|
||||||
|
#[serde(default)]
|
||||||
pub platform: Platform,
|
pub platform: Platform,
|
||||||
/// Originating URL read from the `.source-url` sidecar. Falls back to a
|
/// Originating URL read from the `.source-url` sidecar. Falls back to a
|
||||||
/// folder-name heuristic for legacy YouTube libraries that predate it.
|
/// folder-name heuristic for legacy YouTube libraries that predate it.
|
||||||
|
#[serde(default)]
|
||||||
pub source_url: Option<String>,
|
pub source_url: Option<String>,
|
||||||
/// Videos stored directly inside the channel directory (not in a sub-folder).
|
/// Videos stored directly inside the channel directory (not in a sub-folder).
|
||||||
|
#[serde(default)]
|
||||||
pub videos: Vec<Video>,
|
pub videos: Vec<Video>,
|
||||||
/// Sub-directories that contain at least one video.
|
/// Sub-directories that contain at least one video.
|
||||||
|
#[serde(default)]
|
||||||
pub playlists: Vec<Playlist>,
|
pub playlists: Vec<Playlist>,
|
||||||
|
#[serde(default)]
|
||||||
pub meta: Option<ChannelMeta>,
|
pub meta: Option<ChannelMeta>,
|
||||||
/// Cached sum of `videos.len() + playlists[*].videos.len()`.
|
/// Cached sum of `videos.len() + playlists[*].videos.len()`.
|
||||||
|
#[serde(default)]
|
||||||
pub total_videos_cached: usize,
|
pub total_videos_cached: usize,
|
||||||
/// Cached sum of all video file sizes.
|
/// Cached sum of all video file sizes.
|
||||||
|
#[serde(default)]
|
||||||
pub total_size_cached: u64,
|
pub total_size_cached: u64,
|
||||||
/// Per-channel download-option overrides loaded from the SQLite
|
/// Per-channel download-option overrides loaded from the SQLite
|
||||||
/// `channel_options` table. Empty by default, meaning "use globals".
|
/// `channel_options` table. Empty by default, meaning "use globals".
|
||||||
/// The scanner leaves this as `default()`; callers populate it after the
|
/// The scanner leaves this as `default()`; callers populate it after the
|
||||||
/// scan by reading the DB once via
|
/// scan by reading the DB once via
|
||||||
/// [`crate::database::Database::get_all_channel_options`].
|
/// [`crate::database::Database::get_all_channel_options`].
|
||||||
|
#[serde(default)]
|
||||||
pub download_options: DownloadOptions,
|
pub download_options: DownloadOptions,
|
||||||
/// Optional folder grouping. `None` means the channel is "Unfiled" and
|
/// Optional folder grouping. `None` means the channel is "Unfiled" and
|
||||||
/// appears under its platform's heading. Populated post-scan by
|
/// appears under its platform's heading. Populated post-scan by
|
||||||
/// [`apply_channel_folders`] from the
|
/// [`apply_channel_folders`] from the
|
||||||
/// [`crate::database::Database::get_all_channel_assignments`] map.
|
/// [`crate::database::Database::get_all_channel_assignments`] map.
|
||||||
|
#[serde(default)]
|
||||||
pub folder_id: Option<i64>,
|
pub folder_id: Option<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue