Live-stream recording, recent-additions feed, per-platform impersonate profiles
Three roadmap items together since they touch overlapping code: Live-stream recording - New `live: bool` argument on Downloader::start. When true, attaches `--live-from-start --wait-for-video 30`, suppresses `--break-on-existing` (every recording is a unique file), and prepends a UTC timestamp suffix to the output filename so re-recordings of the same broadcaster don't collide. - The job label gains a "🔴 LIVE" marker so a long-running record is obviously different from a VOD pull. - Both UIs gain a "🔴 Live stream" checkbox in the download panel, hidden when Music mode is selected. Works for Twitch *and* YouTube Live since yt-dlp recognizes both from the URL. - Web `POST /api/download` accepts a `live: bool` body field; all other callers (scheduled re-check, right-click "Check for new videos") pass false. Recent-additions feed - `library::Video` gains `mtime_unix: Option<u64>` — populated from the metadata we already read for `file_size`, so no extra fs hit. - New `SidebarView::Recent` (desktop) + `showRecent` (web) sort the whole library by mtime descending and cap at 100 entries. - Sidebar entry "🕒 Recent additions" appears only when the library has any dated content, so a fresh install doesn't show an empty view. Per-platform impersonation - New `Platform::impersonate_target()`: - Twitch → None (skip impersonation; their OAuth/Helix dislikes mismatched TLS fingerprints). - TikTok → "Chrome-Android-131" (mobile profile matches their first-party app surface). - Everything else → "Chrome-146:Macos-26". - `Downloader::apply_impersonation` now takes the platform and routes through the override; `repair()` (always YouTube-implicit) and `start_music()` (uses classify_url on the URL) wire through too. - `/api/preview` does the same classification before dispatching to yt-dlp. - 3 new tests confirm Twitch → None, TikTok → mobile, YouTube → desktop. 44 unit tests pass. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
c0be4bb533
commit
b24ef4be67
5 changed files with 245 additions and 42 deletions
52
src/app.rs
52
src/app.rs
|
|
@ -47,6 +47,8 @@ enum SidebarView {
|
||||||
Channel(usize),
|
Channel(usize),
|
||||||
Playlist(usize, usize),
|
Playlist(usize, usize),
|
||||||
ContinueWatching,
|
ContinueWatching,
|
||||||
|
/// Activity feed — recent additions across all channels, sorted by mtime.
|
||||||
|
Recent,
|
||||||
Music,
|
Music,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -60,6 +62,7 @@ struct Card {
|
||||||
duration_secs: Option<f64>,
|
duration_secs: Option<f64>,
|
||||||
file_size: Option<u64>,
|
file_size: Option<u64>,
|
||||||
upload_date: Option<String>,
|
upload_date: Option<String>,
|
||||||
|
mtime_unix: Option<u64>,
|
||||||
watched: bool,
|
watched: bool,
|
||||||
resume_pos: Option<f64>,
|
resume_pos: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
@ -83,6 +86,8 @@ pub struct App {
|
||||||
dl_full_scan: bool,
|
dl_full_scan: bool,
|
||||||
dl_quality: DownloadQuality,
|
dl_quality: DownloadQuality,
|
||||||
dl_music_mode: bool,
|
dl_music_mode: bool,
|
||||||
|
/// Record an ongoing live stream from the start instead of joining live.
|
||||||
|
dl_live: bool,
|
||||||
textures: HashMap<PathBuf, Option<egui::TextureHandle>>,
|
textures: HashMap<PathBuf, Option<egui::TextureHandle>>,
|
||||||
thumb_request_tx: Sender<PathBuf>,
|
thumb_request_tx: Sender<PathBuf>,
|
||||||
thumb_result_rx: Receiver<(PathBuf, Option<egui::ColorImage>)>,
|
thumb_result_rx: Receiver<(PathBuf, Option<egui::ColorImage>)>,
|
||||||
|
|
@ -219,6 +224,7 @@ impl App {
|
||||||
dl_full_scan: true,
|
dl_full_scan: true,
|
||||||
dl_quality: DownloadQuality::Best,
|
dl_quality: DownloadQuality::Best,
|
||||||
dl_music_mode: false,
|
dl_music_mode: false,
|
||||||
|
dl_live: false,
|
||||||
textures: HashMap::new(),
|
textures: HashMap::new(),
|
||||||
thumb_request_tx,
|
thumb_request_tx,
|
||||||
thumb_result_rx,
|
thumb_result_rx,
|
||||||
|
|
@ -325,6 +331,7 @@ impl App {
|
||||||
duration_secs: v.duration_secs,
|
duration_secs: v.duration_secs,
|
||||||
file_size: v.file_size,
|
file_size: v.file_size,
|
||||||
upload_date: v.upload_date.clone(),
|
upload_date: v.upload_date.clone(),
|
||||||
|
mtime_unix: v.mtime_unix,
|
||||||
watched: self.watched.contains(&v.id),
|
watched: self.watched.contains(&v.id),
|
||||||
resume_pos,
|
resume_pos,
|
||||||
});
|
});
|
||||||
|
|
@ -347,6 +354,21 @@ impl App {
|
||||||
});
|
});
|
||||||
return cards;
|
return cards;
|
||||||
}
|
}
|
||||||
|
SidebarView::Recent => {
|
||||||
|
// Most-recently-modified videos across the whole library.
|
||||||
|
// Cap at 100 entries so the grid stays responsive for users
|
||||||
|
// with thousands of files.
|
||||||
|
for ch in &self.library {
|
||||||
|
for v in ch.videos.iter().chain(ch.playlists.iter().flat_map(|p| p.videos.iter())) {
|
||||||
|
if v.mtime_unix.is_some() {
|
||||||
|
add_video(&mut cards, &ch.name, v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cards.sort_by(|a, b| b.mtime_unix.unwrap_or(0).cmp(&a.mtime_unix.unwrap_or(0)));
|
||||||
|
cards.truncate(100);
|
||||||
|
return cards;
|
||||||
|
}
|
||||||
SidebarView::All => {
|
SidebarView::All => {
|
||||||
for ch in &self.library {
|
for ch in &self.library {
|
||||||
for v in ch.videos.iter().chain(ch.playlists.iter().flat_map(|p| p.videos.iter())) {
|
for v in ch.videos.iter().chain(ch.playlists.iter().flat_map(|p| p.videos.iter())) {
|
||||||
|
|
@ -567,7 +589,8 @@ impl App {
|
||||||
.collect();
|
.collect();
|
||||||
for url in urls {
|
for url in urls {
|
||||||
let info = classify_url(&url);
|
let info = classify_url(&url);
|
||||||
self.downloader.start(url, &info, true, DownloadQuality::Best);
|
// Scheduled re-check: never treat as live.
|
||||||
|
self.downloader.start(url, &info, true, DownloadQuality::Best, false);
|
||||||
count += 1;
|
count += 1;
|
||||||
}
|
}
|
||||||
self.status = format!("Scheduled check: started {} channel downloads", count);
|
self.status = format!("Scheduled check: started {} channel downloads", count);
|
||||||
|
|
@ -732,6 +755,23 @@ impl App {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Recent additions across the whole library — capped at 100
|
||||||
|
// in `compute_cards`. Only shown when the library has
|
||||||
|
// anything dated; on a brand-new install this stays hidden.
|
||||||
|
let has_dated = self.library.iter()
|
||||||
|
.flat_map(|c| c.all_videos())
|
||||||
|
.any(|v| v.mtime_unix.is_some());
|
||||||
|
if has_dated && ui
|
||||||
|
.selectable_label(
|
||||||
|
self.sidebar_view == SidebarView::Recent,
|
||||||
|
"🕒 Recent additions",
|
||||||
|
)
|
||||||
|
.clicked()
|
||||||
|
{
|
||||||
|
self.sidebar_view = SidebarView::Recent;
|
||||||
|
self.selected_video = None;
|
||||||
|
}
|
||||||
|
|
||||||
let music_count = self.music_library.len();
|
let music_count = self.music_library.len();
|
||||||
if ui
|
if ui
|
||||||
.selectable_label(
|
.selectable_label(
|
||||||
|
|
@ -836,7 +876,7 @@ impl App {
|
||||||
// Process deferred right-click download action
|
// Process deferred right-click download action
|
||||||
if let Some((url, ch_name)) = pending_ch_download {
|
if let Some((url, ch_name)) = pending_ch_download {
|
||||||
let info = classify_url(&url);
|
let info = classify_url(&url);
|
||||||
self.downloader.start(url, &info, !self.dl_full_scan, DownloadQuality::Best);
|
self.downloader.start(url, &info, !self.dl_full_scan, DownloadQuality::Best, false);
|
||||||
self.status = format!("Checking {} for new videos…", ch_name);
|
self.status = format!("Checking {} for new videos…", ch_name);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
@ -901,6 +941,12 @@ impl App {
|
||||||
});
|
});
|
||||||
ui.checkbox(&mut self.dl_full_scan, "Fast mode (stop at first already-downloaded video)")
|
ui.checkbox(&mut self.dl_full_scan, "Fast mode (stop at first already-downloaded video)")
|
||||||
.on_hover_text("Faster for large channels but may miss new videos if gaps exist in the archive. Leave off to check every video.");
|
.on_hover_text("Faster for large channels but may miss new videos if gaps exist in the archive. Leave off to check every video.");
|
||||||
|
ui.checkbox(&mut self.dl_live, "🔴 Live stream (record from start)")
|
||||||
|
.on_hover_text(
|
||||||
|
"Use for Twitch/YouTube Live broadcasts. Adds --live-from-start \
|
||||||
|
so yt-dlp records from the beginning instead of joining mid-stream. \
|
||||||
|
Also waits if the stream hasn't begun yet.",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let ready = !self.dl_url.trim().is_empty();
|
let ready = !self.dl_url.trim().is_empty();
|
||||||
|
|
@ -911,7 +957,7 @@ impl App {
|
||||||
self.downloader.start_music(url);
|
self.downloader.start_music(url);
|
||||||
self.status = "Downloading music…".to_string();
|
self.status = "Downloading music…".to_string();
|
||||||
} else {
|
} else {
|
||||||
self.downloader.start(url, &info, !self.dl_full_scan, self.dl_quality);
|
self.downloader.start(url, &info, !self.dl_full_scan, self.dl_quality, self.dl_live);
|
||||||
self.status = format!("Downloading: {dest}");
|
self.status = format!("Downloading: {dest}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@
|
||||||
//! | `--xattrs` | Store metadata in filesystem extended attributes |
|
//! | `--xattrs` | Store metadata in filesystem extended attributes |
|
||||||
//! | `--sponsorblock-mark all` | Mark (but don't remove) SponsorBlock segments |
|
//! | `--sponsorblock-mark all` | Mark (but don't remove) SponsorBlock segments |
|
||||||
//! | `--extractor-args youtube:player_client=web` | Use the web player API to avoid throttling |
|
//! | `--extractor-args youtube:player_client=web` | Use the web player API to avoid throttling |
|
||||||
//! | `--impersonate Chrome-146:Macos-26` | Impersonate a real browser for bot detection |
|
//! | `--impersonate <target>` | Browser TLS fingerprint per source platform (see [`crate::platform::Platform::impersonate_target`]) |
|
||||||
//! | `--break-on-existing` | Stop when the archive file records the video as already downloaded |
|
//! | `--break-on-existing` | Stop when the archive file records the video as already downloaded |
|
||||||
//! | `--download-archive archive.txt` | Record downloaded IDs to avoid re-downloading |
|
//! | `--download-archive archive.txt` | Record downloaded IDs to avoid re-downloading |
|
||||||
|
|
||||||
|
|
@ -30,7 +30,7 @@ use std::process::{Command, Stdio};
|
||||||
use std::sync::mpsc::{channel, Receiver};
|
use std::sync::mpsc::{channel, Receiver};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
|
|
||||||
use crate::platform::{self, UrlInfo, UrlKind};
|
use crate::platform::{self, Platform, UrlInfo, UrlKind};
|
||||||
use crate::ytdlp_bin;
|
use crate::ytdlp_bin;
|
||||||
|
|
||||||
/// Video quality level passed as a `-f` format selector to yt-dlp.
|
/// Video quality level passed as a `-f` format selector to yt-dlp.
|
||||||
|
|
@ -184,14 +184,15 @@ impl Downloader {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Append `--impersonate Chrome-146:Macos-26` to bypass YouTube's bot
|
/// Append `--impersonate <target>` chosen per source platform. Both the
|
||||||
/// detection. Both the bundled venv (which pip-installs `curl_cffi` via
|
/// bundled venv (which pip-installs `curl_cffi`) and a system yt-dlp
|
||||||
/// [`ytdlp_bin::install_command`]) and a system yt-dlp with curl_cffi
|
/// with curl_cffi can satisfy this. Platforms that prefer no
|
||||||
/// support this. We leave it on unconditionally now; yt-dlp's
|
/// impersonation (e.g. Twitch's OAuth) return `None` from
|
||||||
/// `--list-impersonate-targets` warning surfaces in the install log if
|
/// [`Platform::impersonate_target`] and the flag is omitted.
|
||||||
/// curl_cffi is unavailable, so the user has a clear path forward.
|
fn apply_impersonation(&self, platform: Platform, cmd: &mut Command) {
|
||||||
fn apply_impersonation(&self, cmd: &mut Command) {
|
if let Some(target) = platform.impersonate_target() {
|
||||||
cmd.arg("--impersonate").arg("Chrome-146:Macos-26");
|
cmd.arg("--impersonate").arg(target);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Append the cookie-source flags. Prefers `cookies.txt` in the working
|
/// Append the cookie-source flags. Prefers `cookies.txt` in the working
|
||||||
|
|
@ -277,7 +278,13 @@ impl Downloader {
|
||||||
/// video — fast for routine channel checks. When `full_scan` is true the
|
/// video — fast for routine channel checks. When `full_scan` is true the
|
||||||
/// flag is omitted so every video is checked individually against the
|
/// flag is omitted so every video is checked individually against the
|
||||||
/// download archive; slower, but correctly fills gaps in the history.
|
/// download archive; slower, but correctly fills gaps in the history.
|
||||||
pub fn start(&mut self, url: String, info: &UrlInfo, full_scan: bool, quality: DownloadQuality) {
|
///
|
||||||
|
/// When `live` is true, the invocation is configured to record a live
|
||||||
|
/// stream from the start: `--live-from-start --wait-for-video 30` is
|
||||||
|
/// added, `--break-on-existing` is suppressed (each recording is unique),
|
||||||
|
/// and the output filename gains a UTC timestamp suffix so re-recordings
|
||||||
|
/// of the same channel don't collide.
|
||||||
|
pub fn start(&mut self, url: String, info: &UrlInfo, full_scan: bool, quality: DownloadQuality, live: bool) {
|
||||||
let platform_dir = platform::platform_root(&self.channels_root, info.platform);
|
let platform_dir = platform::platform_root(&self.channels_root, info.platform);
|
||||||
// Per-platform download archive keeps cross-platform IDs from colliding
|
// Per-platform download archive keeps cross-platform IDs from colliding
|
||||||
// (TikTok IDs are numeric, YouTube IDs are 11-char base64, etc.).
|
// (TikTok IDs are numeric, YouTube IDs are 11-char base64, etc.).
|
||||||
|
|
@ -285,6 +292,16 @@ impl Downloader {
|
||||||
let archive_path = platform_dir.join("archive.txt");
|
let archive_path = platform_dir.join("archive.txt");
|
||||||
let platform_label = info.platform.dir_name();
|
let platform_label = info.platform.dir_name();
|
||||||
|
|
||||||
|
// Live recordings get a UTC timestamp suffix in the filename so a
|
||||||
|
// re-recording of the same stream doesn't overwrite the prior one.
|
||||||
|
// VOD downloads rely on yt-dlp's stable `%(id)s` for uniqueness.
|
||||||
|
let live_suffix = if live {
|
||||||
|
format!(" [{}]", format_compact_utc(now_unix()))
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
};
|
||||||
|
let live_label = if live { " 🔴 LIVE" } else { "" };
|
||||||
|
|
||||||
let (out_arg, label) = match &info.kind {
|
let (out_arg, label) = match &info.kind {
|
||||||
UrlKind::Channel { handle } => {
|
UrlKind::Channel { handle } => {
|
||||||
let dir = platform_dir.join(handle);
|
let dir = platform_dir.join(handle);
|
||||||
|
|
@ -293,23 +310,23 @@ impl Downloader {
|
||||||
// guess from the folder name.
|
// guess from the folder name.
|
||||||
platform::write_source_url(&dir, &url);
|
platform::write_source_url(&dir, &url);
|
||||||
(
|
(
|
||||||
format!("{}/%(title)s [%(id)s].%(ext)s", dir.display()),
|
format!("{}/%(title)s [%(id)s]{live_suffix}.%(ext)s", dir.display()),
|
||||||
format!("{}/{}/", platform_label, handle),
|
format!("{}/{}/{}", platform_label, handle, live_label),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
UrlKind::Playlist => (
|
UrlKind::Playlist => (
|
||||||
format!(
|
format!(
|
||||||
"{}/%(uploader,channel,creator|Unknown)s/%(playlist_title)s/%(title)s [%(id)s].%(ext)s",
|
"{}/%(uploader,channel,creator|Unknown)s/%(playlist_title)s/%(title)s [%(id)s]{live_suffix}.%(ext)s",
|
||||||
platform_dir.display()
|
platform_dir.display()
|
||||||
),
|
),
|
||||||
format!("{}/<creator>/<playlist>/", platform_label),
|
format!("{}/<creator>/<playlist>/{}", platform_label, live_label),
|
||||||
),
|
),
|
||||||
UrlKind::Video | UrlKind::Unknown => (
|
UrlKind::Video | UrlKind::Unknown => (
|
||||||
format!(
|
format!(
|
||||||
"{}/%(uploader,channel,creator|Unknown)s/%(title)s [%(id)s].%(ext)s",
|
"{}/%(uploader,channel,creator|Unknown)s/%(title)s [%(id)s]{live_suffix}.%(ext)s",
|
||||||
platform_dir.display()
|
platform_dir.display()
|
||||||
),
|
),
|
||||||
format!("{}/<creator>/", platform_label),
|
format!("{}/<creator>/{}", platform_label, live_label),
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -339,12 +356,21 @@ impl Downloader {
|
||||||
if let Some(fmt) = quality.format_spec() {
|
if let Some(fmt) = quality.format_spec() {
|
||||||
cmd.arg("-f").arg(fmt);
|
cmd.arg("-f").arg(fmt);
|
||||||
}
|
}
|
||||||
if !full_scan {
|
if live {
|
||||||
|
// Record the broadcast from the start instead of joining live.
|
||||||
|
// `--wait-for-video` polls the URL every 30 s until a stream
|
||||||
|
// is actually live, so scheduling a recording before the
|
||||||
|
// stream begins works naturally.
|
||||||
|
cmd.arg("--live-from-start").arg("--wait-for-video").arg("30");
|
||||||
|
} else if !full_scan {
|
||||||
|
// Live recordings should never short-circuit on existing archive
|
||||||
|
// entries — every recording is its own file. Only honor
|
||||||
|
// `--break-on-existing` for VOD/channel-check downloads.
|
||||||
cmd.arg("--break-on-existing");
|
cmd.arg("--break-on-existing");
|
||||||
}
|
}
|
||||||
cmd.arg("--download-archive")
|
cmd.arg("--download-archive")
|
||||||
.arg(archive_path.display().to_string());
|
.arg(archive_path.display().to_string());
|
||||||
self.apply_impersonation(&mut cmd);
|
self.apply_impersonation(info.platform, &mut cmd);
|
||||||
cmd.arg("-o").arg(&out_arg).arg(&url);
|
cmd.arg("-o").arg(&out_arg).arg(&url);
|
||||||
Self::apply_retry_flags(&mut cmd);
|
Self::apply_retry_flags(&mut cmd);
|
||||||
|
|
||||||
|
|
@ -374,7 +400,10 @@ impl Downloader {
|
||||||
.arg("--write-auto-subs")
|
.arg("--write-auto-subs")
|
||||||
.arg("--extractor-args")
|
.arg("--extractor-args")
|
||||||
.arg("youtube:player_client=web");
|
.arg("youtube:player_client=web");
|
||||||
self.apply_impersonation(&mut cmd);
|
// `repair()` rebuilds a YouTube watch URL from a stored video ID, so
|
||||||
|
// the source platform is always YouTube here regardless of where the
|
||||||
|
// original video lives on disk.
|
||||||
|
self.apply_impersonation(Platform::YouTube, &mut cmd);
|
||||||
cmd.arg("-o").arg(&out_arg).arg(&url);
|
cmd.arg("-o").arg(&out_arg).arg(&url);
|
||||||
Self::apply_retry_flags(&mut cmd);
|
Self::apply_retry_flags(&mut cmd);
|
||||||
|
|
||||||
|
|
@ -411,7 +440,11 @@ impl Downloader {
|
||||||
.arg("--xattrs")
|
.arg("--xattrs")
|
||||||
.arg("--extractor-args")
|
.arg("--extractor-args")
|
||||||
.arg("youtube:player_client=web");
|
.arg("youtube:player_client=web");
|
||||||
self.apply_impersonation(&mut cmd);
|
// Music downloads can come from any audio-first platform — classify
|
||||||
|
// the URL once so SoundCloud/Bandcamp pulls get their appropriate
|
||||||
|
// (typically no-op) impersonation profile.
|
||||||
|
let platform = platform::classify_url(&url).platform;
|
||||||
|
self.apply_impersonation(platform, &mut cmd);
|
||||||
cmd.arg("--progress")
|
cmd.arg("--progress")
|
||||||
.arg("--download-archive")
|
.arg("--download-archive")
|
||||||
.arg(archive_path.display().to_string())
|
.arg(archive_path.display().to_string())
|
||||||
|
|
@ -541,6 +574,53 @@ impl Downloader {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Current UNIX timestamp in seconds. Used to disambiguate live-recording
|
||||||
|
/// filenames at job-start time.
|
||||||
|
fn now_unix() -> u64 {
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Format a UNIX timestamp as `YYYYMMDD-HHMMSS` (UTC) for embedding in
|
||||||
|
/// filenames. No `chrono` dep — short manual calendar walk; good enough
|
||||||
|
/// for human-readable filename suffixes.
|
||||||
|
fn format_compact_utc(unix: u64) -> String {
|
||||||
|
let day = unix / 86_400;
|
||||||
|
let day_secs = unix % 86_400;
|
||||||
|
let hour = day_secs / 3600;
|
||||||
|
let minute = (day_secs % 3600) / 60;
|
||||||
|
let second = day_secs % 60;
|
||||||
|
|
||||||
|
let mut year = 1970u32;
|
||||||
|
let mut remaining_days = day;
|
||||||
|
loop {
|
||||||
|
let leap = is_leap(year);
|
||||||
|
let yd = if leap { 366 } else { 365 };
|
||||||
|
if remaining_days < yd as u64 { break; }
|
||||||
|
remaining_days -= yd as u64;
|
||||||
|
year += 1;
|
||||||
|
}
|
||||||
|
let months: [u8; 12] = if is_leap(year) {
|
||||||
|
[31,29,31,30,31,30,31,31,30,31,30,31]
|
||||||
|
} else {
|
||||||
|
[31,28,31,30,31,30,31,31,30,31,30,31]
|
||||||
|
};
|
||||||
|
let mut month = 0usize;
|
||||||
|
while month < 12 && remaining_days >= months[month] as u64 {
|
||||||
|
remaining_days -= months[month] as u64;
|
||||||
|
month += 1;
|
||||||
|
}
|
||||||
|
let day_of_month = remaining_days as u32 + 1;
|
||||||
|
format!(
|
||||||
|
"{:04}{:02}{:02}-{:02}{:02}{:02}",
|
||||||
|
year, month as u32 + 1, day_of_month, hour, minute, second
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_leap(y: u32) -> bool { (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 }
|
||||||
|
|
||||||
/// Parse a yt-dlp `[download] 42.7% …` line into a `[0.0, 1.0]` fraction.
|
/// Parse a yt-dlp `[download] 42.7% …` line into a `[0.0, 1.0]` fraction.
|
||||||
fn parse_progress(line: &str) -> Option<f32> {
|
fn parse_progress(line: &str) -> Option<f32> {
|
||||||
let rest = line.trim_start().strip_prefix("[download]")?.trim_start();
|
let rest = line.trim_start().strip_prefix("[download]")?.trim_start();
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,10 @@ pub struct Video {
|
||||||
pub has_chapters: bool,
|
pub has_chapters: bool,
|
||||||
/// Size of the video file on disk; `None` if the video file is missing.
|
/// Size of the video file on disk; `None` if the video file is missing.
|
||||||
pub file_size: Option<u64>,
|
pub file_size: Option<u64>,
|
||||||
|
/// Filesystem mtime of the video file as a UNIX timestamp (seconds).
|
||||||
|
/// Used by the activity feed to surface recent additions. `None` if the
|
||||||
|
/// video file is missing or the system clock returned an error.
|
||||||
|
pub mtime_unix: Option<u64>,
|
||||||
/// Upload date as `YYYYMMDD` (yt-dlp's native format from info.json).
|
/// Upload date as `YYYYMMDD` (yt-dlp's native format from info.json).
|
||||||
/// `None` if the info.json sidecar is missing or lacks the field.
|
/// `None` if the info.json sidecar is missing or lacks the field.
|
||||||
pub upload_date: Option<String>,
|
pub upload_date: Option<String>,
|
||||||
|
|
@ -404,9 +408,14 @@ fn enrich(raws: Vec<RawVideo>) -> Vec<Video> {
|
||||||
(dur, chap, date)
|
(dur, chap, date)
|
||||||
})
|
})
|
||||||
.unwrap_or((None, false, None));
|
.unwrap_or((None, false, None));
|
||||||
let file_size = raw.video_path.as_ref()
|
let metadata = raw.video_path.as_ref()
|
||||||
.and_then(|p| std::fs::metadata(p).ok())
|
.and_then(|p| std::fs::metadata(p).ok());
|
||||||
.map(|m| m.len());
|
let file_size = metadata.as_ref().map(|m| m.len());
|
||||||
|
let mtime_unix = metadata
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|m| m.modified().ok())
|
||||||
|
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||||
|
.map(|d| d.as_secs());
|
||||||
Video {
|
Video {
|
||||||
id: raw.id,
|
id: raw.id,
|
||||||
title: raw.title,
|
title: raw.title,
|
||||||
|
|
@ -420,6 +429,7 @@ fn enrich(raws: Vec<RawVideo>) -> Vec<Video> {
|
||||||
duration_secs,
|
duration_secs,
|
||||||
has_chapters,
|
has_chapters,
|
||||||
file_size,
|
file_size,
|
||||||
|
mtime_unix,
|
||||||
upload_date,
|
upload_date,
|
||||||
}
|
}
|
||||||
}).collect();
|
}).collect();
|
||||||
|
|
|
||||||
|
|
@ -142,6 +142,26 @@ impl Platform {
|
||||||
pub fn is_audio_first(self) -> bool {
|
pub fn is_audio_first(self) -> bool {
|
||||||
matches!(self, Self::Bandcamp | Self::SoundCloud)
|
matches!(self, Self::Bandcamp | Self::SoundCloud)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// yt-dlp `--impersonate` target tuned for each source. Returns `None`
|
||||||
|
/// when impersonation should be skipped entirely (e.g. Twitch's OAuth
|
||||||
|
/// flow can object to mismatched TLS fingerprints).
|
||||||
|
///
|
||||||
|
/// Targets follow yt-dlp's format. TikTok matches the patterns it
|
||||||
|
/// expects from its first-party mobile app so a desktop fingerprint
|
||||||
|
/// doesn't trip its bot scoring; everything else gets a recent desktop
|
||||||
|
/// Chrome.
|
||||||
|
pub fn impersonate_target(self) -> Option<&'static str> {
|
||||||
|
match self {
|
||||||
|
// Twitch's auth surface dislikes TLS-fingerprint impersonation;
|
||||||
|
// omit the flag so curl_cffi doesn't break OAuth/Helix calls.
|
||||||
|
Self::Twitch => None,
|
||||||
|
// TikTok's API is mobile-first — pretend to be Chrome on Android.
|
||||||
|
Self::TikTok => Some("Chrome-Android-131"),
|
||||||
|
// Default: recent desktop Chrome on macOS.
|
||||||
|
_ => Some("Chrome-146:Macos-26"),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// What kind of URL we're looking at — drives the yt-dlp `-o` template.
|
/// What kind of URL we're looking at — drives the yt-dlp `-o` template.
|
||||||
|
|
@ -460,4 +480,22 @@ mod tests {
|
||||||
assert_eq!(platform_root(cr, Platform::TikTok), Path::new("/foo/tiktok"));
|
assert_eq!(platform_root(cr, Platform::TikTok), Path::new("/foo/tiktok"));
|
||||||
assert_eq!(platform_root(cr, Platform::Twitch), Path::new("/foo/twitch"));
|
assert_eq!(platform_root(cr, Platform::Twitch), Path::new("/foo/twitch"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn twitch_skips_impersonation() {
|
||||||
|
// TLS fingerprint impersonation can interfere with Twitch's OAuth/Helix.
|
||||||
|
assert!(Platform::Twitch.impersonate_target().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tiktok_uses_mobile_profile() {
|
||||||
|
let t = Platform::TikTok.impersonate_target().unwrap();
|
||||||
|
assert!(t.to_lowercase().contains("android"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn youtube_uses_desktop_chrome() {
|
||||||
|
let t = Platform::YouTube.impersonate_target().unwrap();
|
||||||
|
assert!(t.starts_with("Chrome-"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
59
src/web.rs
59
src/web.rs
|
|
@ -184,6 +184,8 @@ struct VideoInfo {
|
||||||
file_size: Option<u64>,
|
file_size: Option<u64>,
|
||||||
/// Upload date as `YYYYMMDD` (yt-dlp's native format).
|
/// Upload date as `YYYYMMDD` (yt-dlp's native format).
|
||||||
upload_date: Option<String>,
|
upload_date: Option<String>,
|
||||||
|
/// Filesystem mtime as a UNIX timestamp. Drives the Recent-additions feed.
|
||||||
|
mtime_unix: Option<u64>,
|
||||||
has_video: bool,
|
has_video: bool,
|
||||||
has_live_chat: bool,
|
has_live_chat: bool,
|
||||||
watched: bool,
|
watched: bool,
|
||||||
|
|
@ -227,6 +229,11 @@ struct StartDownloadRequest {
|
||||||
/// When "music", audio-only mode is used regardless of other settings.
|
/// When "music", audio-only mode is used regardless of other settings.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
quality: String,
|
quality: String,
|
||||||
|
/// Treat the URL as an ongoing live broadcast and record from the start.
|
||||||
|
/// Adds `--live-from-start --wait-for-video 30` and timestamps the
|
||||||
|
/// output filename so re-recordings don't collide.
|
||||||
|
#[serde(default)]
|
||||||
|
live: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Response body for `GET /api/progress`.
|
/// Response body for `GET /api/progress`.
|
||||||
|
|
@ -841,6 +848,7 @@ async fn get_library(State(state): State<Arc<WebState>>) -> impl IntoResponse {
|
||||||
duration_secs: v.duration_secs,
|
duration_secs: v.duration_secs,
|
||||||
file_size: v.file_size,
|
file_size: v.file_size,
|
||||||
upload_date: v.upload_date.clone(),
|
upload_date: v.upload_date.clone(),
|
||||||
|
mtime_unix: v.mtime_unix,
|
||||||
has_video: v.video_path.is_some(),
|
has_video: v.video_path.is_some(),
|
||||||
has_live_chat: v.has_live_chat,
|
has_live_chat: v.has_live_chat,
|
||||||
watched: watched.contains(&v.id),
|
watched: watched.contains(&v.id),
|
||||||
|
|
@ -940,7 +948,7 @@ async fn post_download(
|
||||||
_ => DownloadQuality::Best,
|
_ => DownloadQuality::Best,
|
||||||
};
|
};
|
||||||
let info = classify_url(&url);
|
let info = classify_url(&url);
|
||||||
dl.start(url, &info, body.full_scan, quality);
|
dl.start(url, &info, body.full_scan, quality, body.live);
|
||||||
}
|
}
|
||||||
(StatusCode::ACCEPTED, "ok").into_response()
|
(StatusCode::ACCEPTED, "ok").into_response()
|
||||||
}
|
}
|
||||||
|
|
@ -1255,11 +1263,13 @@ async fn get_preview(
|
||||||
} else if !browser.is_empty() && browser != "none" {
|
} else if !browser.is_empty() && browser != "none" {
|
||||||
cmd.arg("--cookies-from-browser").arg(&browser);
|
cmd.arg("--cookies-from-browser").arg(&browser);
|
||||||
}
|
}
|
||||||
// The bundled venv install now ships `curl_cffi` (see
|
// The bundled venv install ships `curl_cffi`, so --impersonate works
|
||||||
// `crate::ytdlp_bin::install_command`), so --impersonate works in both
|
// in both bundled and system modes. Pick the target per source so
|
||||||
// bundled and system modes.
|
// TikTok previews use the mobile profile and Twitch skips it entirely.
|
||||||
let _ = use_bundled;
|
let _ = use_bundled;
|
||||||
cmd.arg("--impersonate").arg("Chrome-146:Macos-26");
|
if let Some(target) = crate::platform::Platform::from_url(&url).impersonate_target() {
|
||||||
|
cmd.arg("--impersonate").arg(target);
|
||||||
|
}
|
||||||
let output = cmd
|
let output = cmd
|
||||||
.arg(&url)
|
.arg(&url)
|
||||||
.stdin(Stdio::null())
|
.stdin(Stdio::null())
|
||||||
|
|
@ -1455,7 +1465,7 @@ async fn post_scheduler_run(State(state): State<Arc<WebState>>) -> impl IntoResp
|
||||||
let mut dl = state.downloader.lock().unwrap();
|
let mut dl = state.downloader.lock().unwrap();
|
||||||
for url in urls {
|
for url in urls {
|
||||||
let info = classify_url(&url);
|
let info = classify_url(&url);
|
||||||
dl.start(url, &info, true, DownloadQuality::Best);
|
dl.start(url, &info, true, DownloadQuality::Best, false);
|
||||||
}
|
}
|
||||||
*state.last_scheduled_check.lock().unwrap() = Some(std::time::Instant::now());
|
*state.last_scheduled_check.lock().unwrap() = Some(std::time::Instant::now());
|
||||||
(StatusCode::ACCEPTED, format!("started {count} channel checks")).into_response()
|
(StatusCode::ACCEPTED, format!("started {count} channel checks")).into_response()
|
||||||
|
|
@ -1629,7 +1639,7 @@ async fn serve(config: Config, shutdown_rx: std::sync::mpsc::Receiver<()>) {
|
||||||
let mut dl = sched_state.downloader.lock().unwrap();
|
let mut dl = sched_state.downloader.lock().unwrap();
|
||||||
for url in urls {
|
for url in urls {
|
||||||
let info = classify_url(&url);
|
let info = classify_url(&url);
|
||||||
dl.start(url, &info, true, DownloadQuality::Best);
|
dl.start(url, &info, true, DownloadQuality::Best, false);
|
||||||
}
|
}
|
||||||
*sched_state.last_scheduled_check.lock().unwrap() = Some(std::time::Instant::now());
|
*sched_state.last_scheduled_check.lock().unwrap() = Some(std::time::Instant::now());
|
||||||
}
|
}
|
||||||
|
|
@ -1908,12 +1918,13 @@ const HTML_UI: &str = r#"<!DOCTYPE html>
|
||||||
<option value="music">🎵 Music</option>
|
<option value="music">🎵 Music</option>
|
||||||
</select>
|
</select>
|
||||||
<label id="fast-mode-label" style="display:flex;align-items:center;gap:4px;font-size:12px;white-space:nowrap;cursor:pointer" title="Stop at the first already-downloaded video. Faster for large channels but may miss new videos if gaps exist in the archive."><input type="checkbox" id="dl-full-scan"> Fast mode</label>
|
<label id="fast-mode-label" style="display:flex;align-items:center;gap:4px;font-size:12px;white-space:nowrap;cursor:pointer" title="Stop at the first already-downloaded video. Faster for large channels but may miss new videos if gaps exist in the archive."><input type="checkbox" id="dl-full-scan"> Fast mode</label>
|
||||||
|
<label id="live-mode-label" style="display:flex;align-items:center;gap:4px;font-size:12px;white-space:nowrap;cursor:pointer" title="Record an ongoing live broadcast (Twitch/YouTube Live) from the start instead of joining live. Waits if the stream has not begun yet."><input type="checkbox" id="dl-live"> 🔴 Live</label>
|
||||||
<span id="agpl-notice" style="font-size:10px;color:var(--muted);margin-left:auto;white-space:nowrap;overflow:hidden;text-overflow:ellipsis"></span>
|
<span id="agpl-notice" style="font-size:10px;color:var(--muted);margin-left:auto;white-space:nowrap;overflow:hidden;text-overflow:ellipsis"></span>
|
||||||
</footer>
|
</footer>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
'use strict';
|
'use strict';
|
||||||
let library=[], channelUrls=[], activeChannelIdx=null, activePlaylistIdx=null, showContinue=false, showChannels=false, showMusic=false;
|
let library=[], channelUrls=[], activeChannelIdx=null, activePlaylistIdx=null, showContinue=false, showChannels=false, showMusic=false, showRecent=false;
|
||||||
let musicTracks=[];
|
let musicTracks=[];
|
||||||
let bulkMode=false, selected=new Set(), selectedId=null;
|
let bulkMode=false, selected=new Set(), selectedId=null;
|
||||||
let currentPlayingId=null, saveTimer=null;
|
let currentPlayingId=null, saveTimer=null;
|
||||||
|
|
@ -1946,10 +1957,12 @@ function renderSidebar(){
|
||||||
const allVids=library.flatMap(ch=>[...ch.videos,...ch.playlists.flatMap(p=>p.videos)]);
|
const allVids=library.flatMap(ch=>[...ch.videos,...ch.playlists.flatMap(p=>p.videos)]);
|
||||||
const contVids=allVids.filter(v=>v.resume_pos&&v.resume_pos>5&&!v.watched);
|
const contVids=allVids.filter(v=>v.resume_pos&&v.resume_pos>5&&!v.watched);
|
||||||
const total=library.reduce((s,c)=>s+c.total_videos,0);
|
const total=library.reduce((s,c)=>s+c.total_videos,0);
|
||||||
|
const hasDated=allVids.some(v=>v.mtime_unix);
|
||||||
let h=`<div class="sidebar-label">Library</div>`;
|
let h=`<div class="sidebar-label">Library</div>`;
|
||||||
if(contVids.length)h+=`<div class="ch-item${showContinue?' active':''}" onclick="setContinue()">▶ Continue (${contVids.length})</div>`;
|
if(contVids.length)h+=`<div class="ch-item${showContinue?' active':''}" onclick="setContinue()">▶ Continue (${contVids.length})</div>`;
|
||||||
|
if(hasDated)h+=`<div class="ch-item${showRecent?' active':''}" onclick="setRecent()">🕒 Recent additions</div>`;
|
||||||
h+=`<div class="ch-item${showChannels?' active':''}" onclick="setChannels()">⊟ Channels (${library.length})</div>`;
|
h+=`<div class="ch-item${showChannels?' active':''}" onclick="setChannels()">⊟ Channels (${library.length})</div>`;
|
||||||
h+=`<div class="ch-item${!showContinue&&!showChannels&&activeChannelIdx===null?' active':''}" onclick="setView(null,null)">⊞ All (${total})</div>`;
|
h+=`<div class="ch-item${!showContinue&&!showChannels&&!showRecent&&activeChannelIdx===null&&!showMusic?' active':''}" onclick="setView(null,null)">⊞ All (${total})</div>`;
|
||||||
h+=`<div class="ch-item${showMusic?' active':''}" onclick="setMusic()">♫ Music (${musicTracks.length})</div>`;
|
h+=`<div class="ch-item${showMusic?' active':''}" onclick="setMusic()">♫ Music (${musicTracks.length})</div>`;
|
||||||
// Group sidebar entries by platform so a multi-platform library reads as
|
// Group sidebar entries by platform so a multi-platform library reads as
|
||||||
// distinct sections rather than one flat list.
|
// distinct sections rather than one flat list.
|
||||||
|
|
@ -1973,10 +1986,11 @@ function renderSidebar(){
|
||||||
}
|
}
|
||||||
el.innerHTML=h;
|
el.innerHTML=h;
|
||||||
}
|
}
|
||||||
function setContinue(){showContinue=true;showChannels=false;showMusic=false;activeChannelIdx=null;activePlaylistIdx=null;selected.clear();closeSidebar();renderSidebar();renderGrid()}
|
function setContinue(){showContinue=true;showChannels=false;showMusic=false;showRecent=false;activeChannelIdx=null;activePlaylistIdx=null;selected.clear();closeSidebar();renderSidebar();renderGrid()}
|
||||||
function setChannels(){showChannels=true;showContinue=false;showMusic=false;activeChannelIdx=null;activePlaylistIdx=null;selected.clear();closeSidebar();renderSidebar();renderGrid()}
|
function setRecent(){showRecent=true;showContinue=false;showChannels=false;showMusic=false;activeChannelIdx=null;activePlaylistIdx=null;selected.clear();closeSidebar();renderSidebar();renderGrid()}
|
||||||
function setView(ci,pi){showContinue=false;showChannels=false;showMusic=false;activeChannelIdx=ci;activePlaylistIdx=pi;selected.clear();closeSidebar();renderSidebar();renderGrid()}
|
function setChannels(){showChannels=true;showContinue=false;showMusic=false;showRecent=false;activeChannelIdx=null;activePlaylistIdx=null;selected.clear();closeSidebar();renderSidebar();renderGrid()}
|
||||||
function setMusic(){showMusic=true;showContinue=false;showChannels=false;activeChannelIdx=null;activePlaylistIdx=null;selected.clear();closeSidebar();renderSidebar();renderGrid()}
|
function setView(ci,pi){showContinue=false;showChannels=false;showMusic=false;showRecent=false;activeChannelIdx=ci;activePlaylistIdx=pi;selected.clear();closeSidebar();renderSidebar();renderGrid()}
|
||||||
|
function setMusic(){showMusic=true;showContinue=false;showChannels=false;showRecent=false;activeChannelIdx=null;activePlaylistIdx=null;selected.clear();closeSidebar();renderSidebar();renderGrid()}
|
||||||
|
|
||||||
/* ── Grid ───────────────────────────────────────────────────────── */
|
/* ── Grid ───────────────────────────────────────────────────────── */
|
||||||
function currentVideos(){
|
function currentVideos(){
|
||||||
|
|
@ -1991,6 +2005,15 @@ function currentVideos(){
|
||||||
vids.sort((a,b)=>(b.resume_pos||0)-(a.resume_pos||0));
|
vids.sort((a,b)=>(b.resume_pos||0)-(a.resume_pos||0));
|
||||||
return vids;
|
return vids;
|
||||||
}
|
}
|
||||||
|
if(showRecent){
|
||||||
|
// Most-recently-modified across the whole library, capped at 100.
|
||||||
|
for(const ch of library)
|
||||||
|
for(const v of[...ch.videos,...ch.playlists.flatMap(p=>p.videos)])
|
||||||
|
if(v.mtime_unix&&(!q||v.title.toLowerCase().includes(q)||v.id.includes(q)))
|
||||||
|
vids.push({...v,channel:ch.name});
|
||||||
|
vids.sort((a,b)=>(b.mtime_unix||0)-(a.mtime_unix||0));
|
||||||
|
return vids.slice(0,100);
|
||||||
|
}
|
||||||
for(let i=0;i<library.length;i++){
|
for(let i=0;i<library.length;i++){
|
||||||
const ch=library[i];
|
const ch=library[i];
|
||||||
if(activeChannelIdx!==null&&i!==activeChannelIdx)continue;
|
if(activeChannelIdx!==null&&i!==activeChannelIdx)continue;
|
||||||
|
|
@ -2149,13 +2172,19 @@ async function previewDownload(){
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
function fullScan(){return !(document.getElementById('dl-full-scan')?.checked||false)}
|
function fullScan(){return !(document.getElementById('dl-full-scan')?.checked||false)}
|
||||||
|
function dlLive(){return document.getElementById('dl-live')?.checked||false}
|
||||||
function dlQuality(){return document.getElementById('dl-quality')?.value||'best'}
|
function dlQuality(){return document.getElementById('dl-quality')?.value||'best'}
|
||||||
function updateDlMode(){const isMusic=dlQuality()==='music';document.getElementById('fast-mode-label').style.display=isMusic?'none':'flex'}
|
function updateDlMode(){
|
||||||
|
const isMusic=dlQuality()==='music';
|
||||||
|
document.getElementById('fast-mode-label').style.display=isMusic?'none':'flex';
|
||||||
|
// Live recording makes no sense for music-only mode.
|
||||||
|
document.getElementById('live-mode-label').style.display=isMusic?'none':'flex';
|
||||||
|
}
|
||||||
async function confirmDownload(url,btn){
|
async function confirmDownload(url,btn){
|
||||||
if(btn)btn.closest('.modal-bg').remove();
|
if(btn)btn.closest('.modal-bg').remove();
|
||||||
try{
|
try{
|
||||||
const quality=dlQuality();
|
const quality=dlQuality();
|
||||||
const body={url,full_scan:fullScan(),quality};
|
const body={url,full_scan:fullScan(),quality,live:dlLive()};
|
||||||
await api('/api/download',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});
|
await api('/api/download',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)});
|
||||||
document.getElementById('dl-url').value='';setStatus('Download queued…')
|
document.getElementById('dl-url').value='';setStatus('Download queued…')
|
||||||
}catch(e){setStatus('Error: '+e.message)}
|
}catch(e){setStatus('Error: '+e.message)}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue