Switch bundled yt-dlp to a Python venv with curl_cffi for impersonation

The previous standalone PyInstaller binary couldn't run `--impersonate`
because curl_cffi isn't packaged into it. Bundled mode lost the
bot-detection bypass as a result.

Install layout is now:
  ~/.local/share/yt-offline/
    bin/deno              ← unchanged: prepended to PATH for JS deciphering
    venv/bin/yt-dlp       ← new: pip-installed yt-dlp[default] + curl_cffi

The install script:
- Errors clearly if python3 or the `venv` module isn't available
  (the latter is split out on Debian as `python3-venv`).
- Creates the venv if missing, otherwise reuses it for upgrades.
- pip-installs `yt-dlp[default]` and `curl_cffi`, both with PyPI's
  built-in SHA-256 verification.
- Cleans up the legacy `bin/yt-dlp` PyInstaller binary if a prior
  install left one behind.
- Probes `--list-impersonate-targets` at the end and logs whether
  curl_cffi loaded successfully.

Downloader::apply_impersonation now passes --impersonate
unconditionally — both bundled (via the venv) and system yt-dlp (via
pip-installed curl_cffi) honor it.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/downloader.rs

@@ -184,14 +184,13 @@ impl Downloader {
         }
     }
 
-    /// Append `--impersonate Chrome-146:Macos-26` only when the active yt-dlp
+    /// Append `--impersonate Chrome-146:Macos-26` to bypass YouTube's bot
-    /// supports it. The standalone PyInstaller binary we ship as the bundled
+    /// detection. Both the bundled venv (which pip-installs `curl_cffi` via
-    /// option doesn't include `curl_cffi`, so passing `--impersonate` to it
+    /// [`ytdlp_bin::install_command`]) and a system yt-dlp with curl_cffi
-    /// errors out with `Impersonate target "chrome-146:macos-26" is not
+    /// support this. We leave it on unconditionally now; yt-dlp's
-    /// available`. System yt-dlp (typically pip-installed with curl_cffi)
+    /// `--list-impersonate-targets` warning surfaces in the install log if
-    /// works fine.
+    /// curl_cffi is unavailable, so the user has a clear path forward.
     fn apply_impersonation(&self, cmd: &mut Command) {
-        if self.use_bundled_ytdlp { return; }
         cmd.arg("--impersonate").arg("Chrome-146:Macos-26");
     }
 

src/web.rs

@@ -1255,13 +1255,11 @@ async fn get_preview(
     } else if !browser.is_empty() && browser != "none" {
         cmd.arg("--cookies-from-browser").arg(&browser);
     }
-    // Only attach --impersonate if the active yt-dlp supports it. Our
+    // The bundled venv install now ships `curl_cffi` (see
-    // bundled (PyInstaller) binary doesn't ship curl_cffi, so the flag
+    // `crate::ytdlp_bin::install_command`), so --impersonate works in both
-    // would error out. System yt-dlp installed via pip with curl_cffi
+    // bundled and system modes.
-    // does support it.
+    let _ = use_bundled;
-    if !use_bundled {
+    cmd.arg("--impersonate").arg("Chrome-146:Macos-26");
-        cmd.arg("--impersonate").arg("Chrome-146:Macos-26");
-    }
     let output = cmd
         .arg(&url)
         .stdin(Stdio::null())

src/ytdlp_bin.rs

@@ -1,43 +1,66 @@
-//! Management of bundled yt-dlp + deno binaries.
+//! Management of the bundled yt-dlp + deno binaries.
 //!
-//! When [`crate::config::BackupSection::use_bundled_ytdlp`] is enabled, all
+//! When [`crate::config::BackupSection::use_bundled_ytdlp`] is enabled the
-//! yt-dlp invocations use a binary under `~/.local/share/yt-offline/bin/`
+//! app invokes its own yt-dlp instead of whatever's on PATH. To get the
-//! instead of the system PATH. A bundled `deno` lives alongside so yt-dlp can
+//! full feature set — most importantly `curl_cffi`-backed `--impersonate`
-//! evaluate the JavaScript signature-deciphering code YouTube serves with the
+//! support — we install yt-dlp into a self-contained Python virtualenv
-//! player — without depending on a system-wide JS runtime.
+//! under `~/.local/share/yt-offline/venv/`. A bundled `deno` lives at
+//! `~/.local/share/yt-offline/bin/deno` so yt-dlp can evaluate the
+//! JavaScript signature-deciphering code YouTube serves with the player.
 //!
-//! Both binaries are downloaded on demand from the official GitHub releases by
+//! Layout:
-//! [`install_command`], which returns a shell pipeline that curls and unpacks
+//! ```text
-//! them. The pipeline is run as a regular yt-dlp [`crate::downloader::Job`] so
+//! ~/.local/share/yt-offline/
-//! the user sees progress in the same UI.
+//!   bin/                 ← prepended to PATH so yt-dlp finds deno
+//!     deno
+//!   venv/
+//!     bin/yt-dlp         ← the real entry point (or Scripts\yt-dlp.exe on Windows)
+//!     lib/python*/site-packages/{yt_dlp, curl_cffi, ...}
+//! ```
+//!
+//! [`install_command`] returns the shell pipeline that builds this tree.
+//! It's run as a regular [`crate::downloader::Job`] so the user sees
+//! progress in the same UI as their other downloads.
 
 use std::path::PathBuf;
 use std::process::Command;
 
-/// Directory holding the bundled binaries (`yt-dlp`, `deno`).
+/// Root directory holding everything bundled — venv + bin.
-pub fn bundled_dir() -> PathBuf {
+pub fn bundled_root() -> PathBuf {
     let home = std::env::var_os("HOME")
         .map(PathBuf::from)
         .unwrap_or_else(|| PathBuf::from("."));
-    home.join(".local").join("share").join("yt-offline").join("bin")
+    home.join(".local").join("share").join("yt-offline")
 }
 
-/// File name of the yt-dlp executable on this platform.
+/// Directory holding non-Python binaries (currently just `deno`). Also the
-fn ytdlp_filename() -> &'static str {
+/// path we prepend to `PATH` when spawning yt-dlp so it can locate deno.
-    if cfg!(windows) { "yt-dlp.exe" } else { "yt-dlp" }
+pub fn bundled_dir() -> PathBuf {
+    bundled_root().join("bin")
 }
 
-/// Full path the bundled yt-dlp binary should live at.
+/// Root of the Python virtualenv that hosts the bundled yt-dlp install.
+pub fn bundled_venv() -> PathBuf {
+    bundled_root().join("venv")
+}
+
+/// Full path to the bundled yt-dlp entry point. On Unix this lives at
+/// `venv/bin/yt-dlp`; on Windows it's `venv/Scripts/yt-dlp.exe`.
 pub fn bundled_ytdlp_path() -> PathBuf {
-    bundled_dir().join(ytdlp_filename())
+    let venv = bundled_venv();
+    if cfg!(windows) {
+        venv.join("Scripts").join("yt-dlp.exe")
+    } else {
+        venv.join("bin").join("yt-dlp")
+    }
 }
 
 /// Returns the yt-dlp invocation target for [`std::process::Command::new`].
 ///
-/// With `use_bundled = true` returns the absolute path to the bundled binary
+/// With `use_bundled = true` returns the absolute path to the venv-installed
-/// (even if it does not yet exist — yt-dlp simply fails to launch and the
+/// yt-dlp (even if it doesn't yet exist — the spawn error surfaces in the
-/// error surfaces in the job log, prompting the user to click Update).
+/// job log and prompts the user to click Install). Otherwise returns the
-/// Otherwise returns the bare `"yt-dlp"` string so the system PATH is used.
+/// bare `"yt-dlp"` string so the system PATH is used.
 pub fn ytdlp_invocation(use_bundled: bool) -> PathBuf {
     if use_bundled {
         bundled_ytdlp_path()
@@ -46,7 +69,7 @@ pub fn ytdlp_invocation(use_bundled: bool) -> PathBuf {
     }
 }
 
-/// True if the bundled yt-dlp binary currently exists on disk.
+/// True if the bundled yt-dlp has been installed.
 pub fn bundled_installed() -> bool {
     bundled_ytdlp_path().exists()
 }
@@ -62,6 +85,8 @@ pub fn ensure_bundled_executable() {
     #[cfg(unix)]
     {
         use std::os::unix::fs::PermissionsExt;
+        // The venv's `bin/` already gets +x via pip; we only need to
+        // re-apply on the extras we placed in `bundled_dir/`.
         let dir = bundled_dir();
         let Ok(entries) = std::fs::read_dir(&dir) else { return };
         for entry in entries.flatten() {
@@ -80,17 +105,6 @@ pub fn ensure_bundled_executable() {
     }
 }
 
-/// GitHub release asset name for yt-dlp on this platform.
-fn ytdlp_asset() -> &'static str {
-    if cfg!(target_os = "windows") {
-        "yt-dlp.exe"
-    } else if cfg!(target_os = "macos") {
-        "yt-dlp_macos"
-    } else {
-        "yt-dlp_linux"
-    }
-}
-
 /// GitHub release asset name for deno on this platform.
 fn deno_asset() -> &'static str {
     match (std::env::consts::OS, std::env::consts::ARCH) {
@@ -103,37 +117,46 @@ fn deno_asset() -> &'static str {
     }
 }
 
-/// Build a [`Command`] that installs or updates the bundled yt-dlp + deno
+/// Build a [`Command`] that installs (or upgrades) the bundled yt-dlp into
-/// binaries by running a curl/unzip pipeline through `bash -c` (or PowerShell
+/// a Python virtualenv with `curl_cffi` (for `--impersonate` support) and
-/// on Windows). On success, both binaries are present in [`bundled_dir`] with
+/// the rest of `yt-dlp[default]`, plus a sibling `deno` for JS deciphering.
-/// executable bit set.
+///
+/// Runs through `bash -c` on Unix and PowerShell on Windows.
 pub fn install_command() -> Command {
-    let dir = bundled_dir();
+    let root = bundled_root();
-    let dir_str = dir.display().to_string();
+    let root_str = root.display().to_string();
-    let ytdlp_url = format!(
+    let bin_str = bundled_dir().display().to_string();
-        "https://github.com/yt-dlp/yt-dlp/releases/latest/download/{}",
+    let venv_str = bundled_venv().display().to_string();
-        ytdlp_asset()
-    );
     let deno_url = format!(
         "https://github.com/denoland/deno/releases/latest/download/{}",
         deno_asset()
     );
-    let ytdlp_name = ytdlp_filename();
 
     #[cfg(windows)]
     {
         let script = format!(
             "$ErrorActionPreference='Stop'; \
-             New-Item -ItemType Directory -Force -Path '{dir}' | Out-Null; \
+             New-Item -ItemType Directory -Force -Path '{root}' | Out-Null; \
-             Write-Host '==> downloading yt-dlp'; \
+             New-Item -ItemType Directory -Force -Path '{bin}' | Out-Null; \
-             Invoke-WebRequest -Uri '{yurl}' -OutFile '{dir}\\{ybin}'; \
+             $py = (Get-Command py -ErrorAction SilentlyContinue) ?? (Get-Command python -ErrorAction SilentlyContinue); \
+             if (-not $py) {{ Write-Error 'python is not installed'; exit 1 }}; \
+             if (-not (Test-Path '{venv}\\Scripts\\python.exe')) {{ \
+               Write-Host '==> creating Python venv'; \
+               & $py.Source -m venv '{venv}'; \
+             }}; \
+             Write-Host '==> installing yt-dlp + curl_cffi'; \
+             & '{venv}\\Scripts\\python.exe' -m pip install --upgrade pip; \
+             & '{venv}\\Scripts\\python.exe' -m pip install --upgrade 'yt-dlp[default]' curl_cffi; \
              Write-Host '==> downloading deno'; \
-             Invoke-WebRequest -Uri '{durl}' -OutFile '{dir}\\deno.zip'; \
+             Invoke-WebRequest -Uri '{durl}' -OutFile '{bin}\\deno.zip'; \
              Write-Host '==> extracting deno'; \
-             Expand-Archive -Force '{dir}\\deno.zip' '{dir}'; \
+             Expand-Archive -Force '{bin}\\deno.zip' '{bin}'; \
-             Remove-Item '{dir}\\deno.zip'; \
+             Remove-Item '{bin}\\deno.zip'; \
-             Write-Host '==> versions'; & '{dir}\\{ybin}' --version; & '{dir}\\deno.exe' --version",
+             Write-Host '==> versions'; \
-            dir = dir_str, yurl = ytdlp_url, durl = deno_url, ybin = ytdlp_name,
+             & '{venv}\\Scripts\\yt-dlp.exe' --version; \
+             & '{bin}\\deno.exe' --version; \
+             Write-Host '==> done'",
+            root = root_str, bin = bin_str, venv = venv_str, durl = deno_url,
         );
         let mut cmd = Command::new("powershell");
         cmd.arg("-NoProfile").arg("-Command").arg(script);
@@ -141,83 +164,87 @@ pub fn install_command() -> Command {
     }
     #[cfg(not(windows))]
     {
-        // Download each file in the background and poll the growing file size
+        // Set up yt-dlp inside a venv so we get the full pip-installable
-        // every 3 s so the job log shows visible progress during the transfer
+        // distribution — including `curl_cffi`, which is what enables
-        // instead of going silent for minutes.
+        // `--impersonate` support. The standalone PyInstaller binary we
+        // previously shipped lacks curl_cffi and errors out on impersonate
+        // targets. The venv strategy is also what yt-dlp's own docs
+        // recommend for getting a "full" install.
         //
-        // For yt-dlp we additionally fetch its published `SHA2-256SUMS` file
+        // Pip-installed packages come with their own SHA-256 checks via
-        // and verify the binary against it — that file is signed-by-presence
+        // PyPI's metadata, so we skip the manual checksum step we used to
-        // in the same GitHub release, so any tampering would have to compromise
+        // do for the standalone binary.
-        // both URLs to slip a bad binary through. Deno doesn't publish a
+        //
-        // similarly convenient single-file checksum manifest, so we just print
+        // Deno is unchanged: it's a single static binary that we drop into
-        // its SHA-256 for visual inspection.
+        // `bin/` alongside the (unused) directory; yt-dlp finds it via PATH
-        let sums_url = "https://github.com/yt-dlp/yt-dlp/releases/latest/download/SHA2-256SUMS";
+        // when we prepend `bin/` in [`crate::downloader::Downloader::spawn_job`].
-        let ytdlp_asset_name = ytdlp_asset();
         let script = format!(
             r#"set -e
-command -v curl     >/dev/null || {{ echo 'error: curl not installed';     exit 1; }}
+command -v curl  >/dev/null || {{ echo 'error: curl not installed';  exit 1; }}
-command -v unzip    >/dev/null || {{ echo 'error: unzip not installed';    exit 1; }}
+command -v unzip >/dev/null || {{ echo 'error: unzip not installed'; exit 1; }}
-command -v sha256sum >/dev/null || {{ echo 'error: sha256sum not installed'; exit 1; }}
+if ! command -v python3 >/dev/null; then
-mkdir -p '{dir}'
+  echo 'error: python3 is not installed.'
-
+  echo '       Install it from your distro package manager and try again.'
-# ── yt-dlp ──────────────────────────────────────────────────────────────────
-echo '==> downloading yt-dlp'
-curl -fL --no-progress-meter --connect-timeout 30 --max-time 600 --retry 3 \
-  -o '{dir}/{ybin}' '{yurl}' &
-DLPID=$!
-while kill -0 $DLPID 2>/dev/null; do
-  sleep 3
-  SZ=$(wc -c < '{dir}/{ybin}' 2>/dev/null || echo 0)
-  echo "    yt-dlp: $SZ bytes received..."
-done
-wait $DLPID
-echo "    done: $(wc -c < '{dir}/{ybin}') bytes"
-
-echo '==> verifying yt-dlp SHA-256'
-curl -fL --no-progress-meter --connect-timeout 30 --max-time 60 \
-  -o '{dir}/yt-dlp.sums' '{sums}'
-EXPECTED=$(awk -v n="{yasset}" '$2==n {{ print $1 }}' '{dir}/yt-dlp.sums' | head -n1)
-ACTUAL=$(sha256sum '{dir}/{ybin}' | awk '{{ print $1 }}')
-rm '{dir}/yt-dlp.sums'
-if [ -z "$EXPECTED" ]; then
-  echo "    warn: no checksum entry for {yasset} in SHA2-256SUMS"
-  echo "    actual SHA-256: $ACTUAL"
-elif [ "$EXPECTED" != "$ACTUAL" ]; then
-  echo "    error: SHA-256 mismatch"
-  echo "    expected: $EXPECTED"
-  echo "    actual:   $ACTUAL"
-  rm -f '{dir}/{ybin}'
   exit 1
-else
-  echo "    ok: $ACTUAL"
 fi
-chmod +x '{dir}/{ybin}'
+# Detect whether the venv module is actually usable. On Debian/Ubuntu it
+# ships in the `python3-venv` package and fails noisily when missing.
+if ! python3 -c 'import venv' 2>/dev/null; then
+  echo 'error: the Python "venv" module is not installed.'
+  echo '       Install python3-venv (Debian/Ubuntu) or the equivalent.'
+  exit 1
+fi
+
+mkdir -p '{root}'
+mkdir -p '{bin}'
+
+# Remove any legacy PyInstaller binary from the prior install layout so
+# bundled_ytdlp_path()'s new venv-based path is the only one resolvable.
+rm -f '{bin}/yt-dlp'
+
+if [ ! -x '{venv}/bin/python' ]; then
+  echo '==> creating Python venv at {venv}'
+  python3 -m venv '{venv}'
+fi
+
+echo '==> upgrading pip in venv'
+'{venv}/bin/python' -m pip install --upgrade --quiet pip
+
+echo '==> installing yt-dlp[default] + curl_cffi (this fetches a few packages)'
+'{venv}/bin/python' -m pip install --upgrade --quiet --progress-bar off 'yt-dlp[default]' curl_cffi
 
 # ── deno ────────────────────────────────────────────────────────────────────
 echo '==> downloading deno'
 curl -fL --no-progress-meter --connect-timeout 30 --max-time 600 --retry 3 \
-  -o '{dir}/deno.zip' '{durl}' &
+  -o '{bin}/deno.zip' '{durl}' &
 DLPID=$!
 while kill -0 $DLPID 2>/dev/null; do
   sleep 3
-  SZ=$(wc -c < '{dir}/deno.zip' 2>/dev/null || echo 0)
+  SZ=$(wc -c < '{bin}/deno.zip' 2>/dev/null || echo 0)
   echo "    deno.zip: $SZ bytes received..."
 done
 wait $DLPID
-echo "    done: $(wc -c < '{dir}/deno.zip') bytes"
+echo "    done: $(wc -c < '{bin}/deno.zip') bytes"
-echo "    deno.zip SHA-256: $(sha256sum '{dir}/deno.zip' | awk '{{ print $1 }}')"
+if command -v sha256sum >/dev/null; then
+  echo "    deno.zip SHA-256: $(sha256sum '{bin}/deno.zip' | awk '{{ print $1 }}')"
+fi
 
 echo '==> extracting deno'
-unzip -o '{dir}/deno.zip' -d '{dir}'
+unzip -o '{bin}/deno.zip' -d '{bin}'
-chmod +x '{dir}/deno'
+chmod +x '{bin}/deno'
-rm '{dir}/deno.zip'
+rm '{bin}/deno.zip'
 
 echo '==> versions'
-'{dir}/{ybin}' --version
+'{venv}/bin/yt-dlp' --version
-'{dir}/deno' --version
+'{bin}/deno' --version
+echo '==> verifying curl_cffi (impersonation) is available'
+if '{venv}/bin/yt-dlp' --list-impersonate-targets 2>/dev/null | grep -qi 'chrome'; then
+  echo '    ok: impersonation targets available'
+else
+  echo '    warn: curl_cffi did not load; --impersonate will be skipped'
+fi
 echo '==> done'"#,
-            dir = dir_str, yurl = ytdlp_url, durl = deno_url, ybin = ytdlp_name,
+            root = root_str, bin = bin_str, venv = venv_str, durl = deno_url,
-            sums = sums_url, yasset = ytdlp_asset_name,
         );
         let mut cmd = Command::new("bash");
         cmd.arg("-c").arg(script);