Add bundled yt-dlp, music library, Plex metadata, sort by date; security + perf hardening
Bug fixes - Desktop auto-rescan: snapshot was_running before check_notifications updates prev_job_states, so the post-download rescan actually fires. - web::run shutdown: replace `let _ = tx;` (which dropped the sender immediately and short-circuited the shutdown select!) with std::mem::forget(tx); add an idle fallback so the `-> !` is honored. - /api/preview now resolves the bundled yt-dlp path and extends PATH for deno discovery, matching the rest of the downloader. - Scheduler interval clamped to >=1h on read so a manually-edited config.toml with 0 hours can't trigger every tick. - maintenance::is_within canonicalizes the parent + filename when the target is missing; remove_files treats NotFound as success. Security - Session tokens stored with issued-at Instant and pruned past 30 day TTL on every touch (was: unbounded HashSet). - Login rate-limited per source IP: 5 failures → 60s lockout (429). - Secure cookie flag added when X-Forwarded-Proto: https is present. - 4 MiB body size cap via DefaultBodyLimit. - Security headers middleware: CSP, X-Content-Type-Options, X-Frame-Options, Referrer-Policy. - JS safeUrl() defangs javascript:/vbscript:/data:text URLs before interpolation into src= attributes. - Bundled yt-dlp install verifies SHA-256 against the release's SHA2-256SUMS; deno hash printed for visual inspection. Install also reports byte counts every 3s so users see live progress. - yt-offline.db chmod 0600 at open time on Unix. New features - Bundled yt-dlp + deno mode: settings toggle (System/Bundled) plus one-click Install/Update from settings. Ensures executable bit on every launch in case the install chmod was missed. - Download queue with configurable max_concurrent (1–10); pending jobs surfaced in both UIs. - Music download mode: --extract-audio into music/<artist>/, separate sidebar entry, /api/music + /music-files/ endpoints, inline <audio>. - Download quality selector: Best / 1080p / 720p / 480p / 360p. - Sort videos by Newest / Oldest (upload_date from info.json with release_date fallback). - Plex metadata: per-episode .nfo (title, season/episode, aired, runtime, plot from .description), <stem>-thumb.jpg symlink, and show-level tvshow.nfo. - yt-dlp retry/throttle defaults: --retries 30 --fragment-retries 30 --retry-sleep linear=1:30:2 --sleep-requests 1 to ride out the "Connection reset by peer" failures. Performance - password_required cached in AtomicBool; refreshed only on password change. Eliminates a SQLite query per static-file fetch. - Job::log → VecDeque for O(1) front-pop instead of O(n) drain. - Library scan parallelized across cores via stdlib parallel_map. - Web UI poll: 600ms while active, 5s when idle, 2s after errors. - Music scan now recursive (Artist/Album/Track.opus). - percent_encode_segment uses write! to avoid per-byte String allocs. Code quality - 27 unit tests covering parse_progress, parse_stem, extract_after, detect_url_kind, srt_to_vtt, count_cookies, percent_encode_segment, bind_mode_of, hash_password, lang_label, is_within, sanitize, year_of, aired_date, xml_escape. - Unified library::find_video + Channel::all_videos() iterator, replacing three duplicated linear searches. - Downloader::browser wired to --cookies-from-browser as a fallback when no cookies.txt is present. - mpv detection now matches the binary basename, not substring. - Settings modal scroll fix; web settings expose all new fields. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Luna
parent
1d72069913
commit
59996735f3
10 changed files with 2300 additions and 156 deletions
226
src/ytdlp_bin.rs
Normal file
226
src/ytdlp_bin.rs
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
//! Management of bundled yt-dlp + deno binaries.
|
||||
//!
|
||||
//! When [`crate::config::BackupSection::use_bundled_ytdlp`] is enabled, all
|
||||
//! yt-dlp invocations use a binary under `~/.local/share/yt-offline/bin/`
|
||||
//! instead of the system PATH. A bundled `deno` lives alongside so yt-dlp can
|
||||
//! evaluate the JavaScript signature-deciphering code YouTube serves with the
|
||||
//! player — without depending on a system-wide JS runtime.
|
||||
//!
|
||||
//! Both binaries are downloaded on demand from the official GitHub releases by
|
||||
//! [`install_command`], which returns a shell pipeline that curls and unpacks
|
||||
//! them. The pipeline is run as a regular yt-dlp [`crate::downloader::Job`] so
|
||||
//! the user sees progress in the same UI.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
|
||||
/// Directory holding the bundled binaries (`yt-dlp`, `deno`).
|
||||
pub fn bundled_dir() -> PathBuf {
|
||||
let home = std::env::var_os("HOME")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("."));
|
||||
home.join(".local").join("share").join("yt-offline").join("bin")
|
||||
}
|
||||
|
||||
/// File name of the yt-dlp executable on this platform.
|
||||
fn ytdlp_filename() -> &'static str {
|
||||
if cfg!(windows) { "yt-dlp.exe" } else { "yt-dlp" }
|
||||
}
|
||||
|
||||
/// Full path the bundled yt-dlp binary should live at.
|
||||
pub fn bundled_ytdlp_path() -> PathBuf {
|
||||
bundled_dir().join(ytdlp_filename())
|
||||
}
|
||||
|
||||
/// Returns the yt-dlp invocation target for [`std::process::Command::new`].
|
||||
///
|
||||
/// With `use_bundled = true` returns the absolute path to the bundled binary
|
||||
/// (even if it does not yet exist — yt-dlp simply fails to launch and the
|
||||
/// error surfaces in the job log, prompting the user to click Update).
|
||||
/// Otherwise returns the bare `"yt-dlp"` string so the system PATH is used.
|
||||
pub fn ytdlp_invocation(use_bundled: bool) -> PathBuf {
|
||||
if use_bundled {
|
||||
bundled_ytdlp_path()
|
||||
} else {
|
||||
PathBuf::from("yt-dlp")
|
||||
}
|
||||
}
|
||||
|
||||
/// True if the bundled yt-dlp binary currently exists on disk.
|
||||
pub fn bundled_installed() -> bool {
|
||||
bundled_ytdlp_path().exists()
|
||||
}
|
||||
|
||||
/// Defensively re-apply `+x` to every regular file inside [`bundled_dir`].
|
||||
///
|
||||
/// Called before each bundled-mode invocation to guard against the install
|
||||
/// script's `chmod` step having been skipped (e.g. partial install) or the
|
||||
/// executable bit having been stripped by some other process.
|
||||
///
|
||||
/// No-op on Windows (executability is determined by the `.exe` suffix there).
|
||||
pub fn ensure_bundled_executable() {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let dir = bundled_dir();
|
||||
let Ok(entries) = std::fs::read_dir(&dir) else { return };
|
||||
for entry in entries.flatten() {
|
||||
let p = entry.path();
|
||||
if !p.is_file() { continue; }
|
||||
if let Ok(meta) = entry.metadata() {
|
||||
let mut perms = meta.permissions();
|
||||
let mode = perms.mode();
|
||||
let want = mode | 0o111;
|
||||
if mode != want {
|
||||
perms.set_mode(want);
|
||||
let _ = std::fs::set_permissions(&p, perms);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// GitHub release asset name for yt-dlp on this platform.
|
||||
fn ytdlp_asset() -> &'static str {
|
||||
if cfg!(target_os = "windows") {
|
||||
"yt-dlp.exe"
|
||||
} else if cfg!(target_os = "macos") {
|
||||
"yt-dlp_macos"
|
||||
} else {
|
||||
"yt-dlp_linux"
|
||||
}
|
||||
}
|
||||
|
||||
/// GitHub release asset name for deno on this platform.
|
||||
fn deno_asset() -> &'static str {
|
||||
match (std::env::consts::OS, std::env::consts::ARCH) {
|
||||
("linux", "x86_64") => "deno-x86_64-unknown-linux-gnu.zip",
|
||||
("linux", "aarch64") => "deno-aarch64-unknown-linux-gnu.zip",
|
||||
("macos", "x86_64") => "deno-x86_64-apple-darwin.zip",
|
||||
("macos", "aarch64") => "deno-aarch64-apple-darwin.zip",
|
||||
("windows", _) => "deno-x86_64-pc-windows-msvc.zip",
|
||||
_ => "deno-x86_64-unknown-linux-gnu.zip",
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a [`Command`] that installs or updates the bundled yt-dlp + deno
|
||||
/// binaries by running a curl/unzip pipeline through `bash -c` (or PowerShell
|
||||
/// on Windows). On success, both binaries are present in [`bundled_dir`] with
|
||||
/// executable bit set.
|
||||
pub fn install_command() -> Command {
|
||||
let dir = bundled_dir();
|
||||
let dir_str = dir.display().to_string();
|
||||
let ytdlp_url = format!(
|
||||
"https://github.com/yt-dlp/yt-dlp/releases/latest/download/{}",
|
||||
ytdlp_asset()
|
||||
);
|
||||
let deno_url = format!(
|
||||
"https://github.com/denoland/deno/releases/latest/download/{}",
|
||||
deno_asset()
|
||||
);
|
||||
let ytdlp_name = ytdlp_filename();
|
||||
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let script = format!(
|
||||
"$ErrorActionPreference='Stop'; \
|
||||
New-Item -ItemType Directory -Force -Path '{dir}' | Out-Null; \
|
||||
Write-Host '==> downloading yt-dlp'; \
|
||||
Invoke-WebRequest -Uri '{yurl}' -OutFile '{dir}\\{ybin}'; \
|
||||
Write-Host '==> downloading deno'; \
|
||||
Invoke-WebRequest -Uri '{durl}' -OutFile '{dir}\\deno.zip'; \
|
||||
Write-Host '==> extracting deno'; \
|
||||
Expand-Archive -Force '{dir}\\deno.zip' '{dir}'; \
|
||||
Remove-Item '{dir}\\deno.zip'; \
|
||||
Write-Host '==> versions'; & '{dir}\\{ybin}' --version; & '{dir}\\deno.exe' --version",
|
||||
dir = dir_str, yurl = ytdlp_url, durl = deno_url, ybin = ytdlp_name,
|
||||
);
|
||||
let mut cmd = Command::new("powershell");
|
||||
cmd.arg("-NoProfile").arg("-Command").arg(script);
|
||||
cmd
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
// Download each file in the background and poll the growing file size
|
||||
// every 3 s so the job log shows visible progress during the transfer
|
||||
// instead of going silent for minutes.
|
||||
//
|
||||
// For yt-dlp we additionally fetch its published `SHA2-256SUMS` file
|
||||
// and verify the binary against it — that file is signed-by-presence
|
||||
// in the same GitHub release, so any tampering would have to compromise
|
||||
// both URLs to slip a bad binary through. Deno doesn't publish a
|
||||
// similarly convenient single-file checksum manifest, so we just print
|
||||
// its SHA-256 for visual inspection.
|
||||
let sums_url = "https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-256SUMS";
|
||||
let ytdlp_asset_name = ytdlp_asset();
|
||||
let script = format!(
|
||||
r#"set -e
|
||||
command -v curl >/dev/null || {{ echo 'error: curl not installed'; exit 1; }}
|
||||
command -v unzip >/dev/null || {{ echo 'error: unzip not installed'; exit 1; }}
|
||||
command -v sha256sum >/dev/null || {{ echo 'error: sha256sum not installed'; exit 1; }}
|
||||
mkdir -p '{dir}'
|
||||
|
||||
# ── yt-dlp ──────────────────────────────────────────────────────────────────
|
||||
echo '==> downloading yt-dlp'
|
||||
curl -fL --no-progress-meter --connect-timeout 30 --max-time 600 --retry 3 \
|
||||
-o '{dir}/{ybin}' '{yurl}' &
|
||||
DLPID=$!
|
||||
while kill -0 $DLPID 2>/dev/null; do
|
||||
sleep 3
|
||||
SZ=$(wc -c < '{dir}/{ybin}' 2>/dev/null || echo 0)
|
||||
echo " yt-dlp: $SZ bytes received..."
|
||||
done
|
||||
wait $DLPID
|
||||
echo " done: $(wc -c < '{dir}/{ybin}') bytes"
|
||||
|
||||
echo '==> verifying yt-dlp SHA-256'
|
||||
curl -fL --no-progress-meter --connect-timeout 30 --max-time 60 \
|
||||
-o '{dir}/yt-dlp.sums' '{sums}'
|
||||
EXPECTED=$(awk -v n="{yasset}" '$2==n {{ print $1 }}' '{dir}/yt-dlp.sums' | head -n1)
|
||||
ACTUAL=$(sha256sum '{dir}/{ybin}' | awk '{{ print $1 }}')
|
||||
rm '{dir}/yt-dlp.sums'
|
||||
if [ -z "$EXPECTED" ]; then
|
||||
echo " warn: no checksum entry for {yasset} in SHA2-256SUMS"
|
||||
echo " actual SHA-256: $ACTUAL"
|
||||
elif [ "$EXPECTED" != "$ACTUAL" ]; then
|
||||
echo " error: SHA-256 mismatch"
|
||||
echo " expected: $EXPECTED"
|
||||
echo " actual: $ACTUAL"
|
||||
rm -f '{dir}/{ybin}'
|
||||
exit 1
|
||||
else
|
||||
echo " ok: $ACTUAL"
|
||||
fi
|
||||
chmod +x '{dir}/{ybin}'
|
||||
|
||||
# ── deno ────────────────────────────────────────────────────────────────────
|
||||
echo '==> downloading deno'
|
||||
curl -fL --no-progress-meter --connect-timeout 30 --max-time 600 --retry 3 \
|
||||
-o '{dir}/deno.zip' '{durl}' &
|
||||
DLPID=$!
|
||||
while kill -0 $DLPID 2>/dev/null; do
|
||||
sleep 3
|
||||
SZ=$(wc -c < '{dir}/deno.zip' 2>/dev/null || echo 0)
|
||||
echo " deno.zip: $SZ bytes received..."
|
||||
done
|
||||
wait $DLPID
|
||||
echo " done: $(wc -c < '{dir}/deno.zip') bytes"
|
||||
echo " deno.zip SHA-256: $(sha256sum '{dir}/deno.zip' | awk '{{ print $1 }}')"
|
||||
|
||||
echo '==> extracting deno'
|
||||
unzip -o '{dir}/deno.zip' -d '{dir}'
|
||||
chmod +x '{dir}/deno'
|
||||
rm '{dir}/deno.zip'
|
||||
|
||||
echo '==> versions'
|
||||
'{dir}/{ybin}' --version
|
||||
'{dir}/deno' --version
|
||||
echo '==> done'"#,
|
||||
dir = dir_str, yurl = ytdlp_url, durl = deno_url, ybin = ytdlp_name,
|
||||
sums = sums_url, yasset = ytdlp_asset_name,
|
||||
);
|
||||
let mut cmd = Command::new("bash");
|
||||
cmd.arg("-c").arg(script);
|
||||
cmd
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue