Let ArchiveBox 0.9 actually crawl, and serve its snapshots

Every extractor binary 0.9 cannot find aborts the crawl runner, and abx-plugins
keeps adding new ones, so a fixed apt list always falls behind while the service
user cannot run apt. archivebox install now runs as root with the service user's
home, the way upstream's own package does, and resolves every system dependency
itself; mercury stays disabled because npm refuses its git dependency. Snapshot
iframes are built from BASE_URL, so the update sets it when missing or when it is
a stale IP, and migrated 0.7 collections get their snapshots re-indexed.

ct/archivebox.sh

@@ -31,34 +31,29 @@ function update_script() {
     exit
   fi
 
-  NODE_VERSION="22" setup_nodejs
   setup_uv
 
-  ensure_dependencies chromium ripgrep tesseract-ocr tesseract-ocr-eng imagemagick ffmpeg unzip wget
-
   msg_info "Stopping Service"
   systemctl stop archivebox
   msg_ok "Stopped Service"
 
-  # Earlier installs used python3.11, which caps ArchiveBox at 0.7.4.
+  local migrate=0 py port base runtime
   if [[ ! -x /opt/archivebox/venv/bin/archivebox ]]; then
+    migrate=1
     msg_info "Moving ArchiveBox to its own Python 3.13 environment"
-    # The distro interpreter first: a uv-managed one lands where the service user cannot exec it.
+    py="/usr/bin/python3.13"
-    local py="/usr/bin/python3.13"
+    [[ -x "$py" ]] || ensure_dependencies python3.13 || true
-    if [[ ! -x "$py" ]]; then
-      ensure_dependencies python3.13 || true
-    fi
     if [[ ! -x "$py" ]]; then
       $STD uv python install --install-dir /opt/archivebox/python 3.13
       py="$(UV_PYTHON_INSTALL_DIR=/opt/archivebox/python uv python find 3.13)"
     fi
     $STD uv venv --python "$py" /opt/archivebox/venv
-    # Keep whatever port this container already answers on, or a reverse proxy in front
+    msg_ok "Moved ArchiveBox to its own Python 3.13 environment"
-    # of it would silently stop resolving.
+  fi
-    local port
+
-    port="$(awk -F: '/^ExecStart=/ {print $NF}' /etc/systemd/system/archivebox.service 2>/dev/null)"
+  port="$(awk -F: '/^ExecStart=/ {print $NF}' /etc/systemd/system/archivebox.service 2>/dev/null)"
-    [[ "$port" =~ ^[0-9]+$ ]] || port=5797
+  [[ "$port" =~ ^[0-9]+$ ]] || port=5797
-    cat <<EOF >/etc/systemd/system/archivebox.service
+  cat <<EOF >/etc/systemd/system/archivebox.service
 [Unit]
 Description=ArchiveBox Server
 After=network.target
@@ -66,28 +61,45 @@ After=network.target
 [Service]
 User=archivebox
 WorkingDirectory=/opt/archivebox/data
+ExecStartPre=/opt/archivebox/venv/bin/archivebox init
 ExecStart=/opt/archivebox/venv/bin/archivebox server 0.0.0.0:${port}
 Restart=always
 
 [Install]
 WantedBy=multi-user.target
 EOF
-    systemctl daemon-reload
+  systemctl daemon-reload
-    msg_ok "Moved ArchiveBox to its own Python 3.13 environment (port ${port})"
-  fi
 
   msg_info "Updating ArchiveBox"
   $STD uv pip install --python /opt/archivebox/venv/bin/python --upgrade archivebox
-  chown -R archivebox:archivebox /opt/archivebox
-  # Without this a shell still reaches the old 0.7.4 entry point against a 0.9 collection.
   ln -sf /opt/archivebox/venv/bin/archivebox /usr/local/bin/archivebox
+  mkdir -p /home/archivebox/.config
+  chown -R archivebox:archivebox /opt/archivebox /home/archivebox
   cd /opt/archivebox/data
   $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox init
-  # npm refuses the git dependency behind @postlight/parser, so treat extractors as optional.
+  base="$(grep -oP '^BASE_URL\s*=\s*\K\S+' ArchiveBox.conf 2>/dev/null || true)"
-  $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox install ||
+  if [[ -z "$base" || "$base" =~ ^https?://[0-9.]+(:[0-9]+)?/?$ ]]; then
-    msg_warn "Some extractors stayed unavailable - ArchiveBox runs without them"
+    base="http://$(hostname -I | awk '{print $1}'):${port}"
+  fi
+  $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox config --set "BASE_URL=${base}" MERCURY_ENABLED=False
   msg_ok "Updated ArchiveBox"
 
+  msg_info "Installing Extractors (Patience)"
+  runtime="/tmp/archivebox-runtime-$(id -u archivebox)"
+  install -d -m 700 -o archivebox -g archivebox "$runtime"
+  if $STD env HOME=/home/archivebox USER=archivebox LOGNAME=archivebox XDG_CONFIG_HOME=/home/archivebox/.config XDG_RUNTIME_DIR="$runtime" \
+    /opt/archivebox/venv/bin/archivebox install; then
+    msg_ok "Installed Extractors"
+  else
+    msg_warn "Some extractors failed to install - crawls stop until the next update succeeds"
+  fi
+
+  if [[ "$migrate" == 1 ]]; then
+    msg_info "Migrating Existing Snapshots (Patience)"
+    $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox update --rescan --migrate-only --index-only
+    msg_ok "Migrated Existing Snapshots"
+  fi
+
   msg_info "Starting Service"
   systemctl start archivebox
   msg_ok "Started Service"

install/archivebox-install.sh

@@ -14,34 +14,23 @@ network_check
 update_os
 
 msg_info "Installing Dependencies"
-# ArchiveBox resolves these through its env provider, so it never has to reach for apt itself.
 $STD apt-get install -y \
   git \
   procps \
   dnsutils \
-  chromium \
-  ripgrep \
-  tesseract-ocr \
-  tesseract-ocr-eng \
-  imagemagick \
-  ffmpeg \
-  unzip \
-  wget \
   python3.13
 msg_ok "Installed Dependencies"
 
-# No NODE_MODULE: 0.9 pulls single-file and readability itself, and postlight-parser is gone.
-NODE_VERSION="22" setup_nodejs
 setup_uv
 
 msg_info "Installing ArchiveBox"
 mkdir -p /opt/archivebox/data
 $STD adduser --system --shell /bin/bash --gecos 'Archive Box User' --group --disabled-password --home /home/archivebox archivebox
-# The distro interpreter: a uv-managed one lands where the service user cannot execute it.
 $STD uv venv --python /usr/bin/python3.13 /opt/archivebox/venv
 $STD uv pip install --python /opt/archivebox/venv/bin/python archivebox
 ln -sf /opt/archivebox/venv/bin/archivebox /usr/local/bin/archivebox
-chown -R archivebox:archivebox /opt/archivebox
+mkdir -p /home/archivebox/.config
+chown -R archivebox:archivebox /opt/archivebox /home/archivebox
 msg_ok "Installed ArchiveBox"
 
 msg_info "Initializing ArchiveBox"
@@ -52,18 +41,15 @@ $STD sudo -u archivebox env \
   DJANGO_SUPERUSER_EMAIL=archivebox@archivebox.local \
   DJANGO_SUPERUSER_PASSWORD=community-scripts.org \
   /opt/archivebox/venv/bin/archivebox manage createsuperuser --noinput
-# Without a pinned canonical URL the admin greets every visitor with a red banner.
+$STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox config --set "BASE_URL=http://$(get_ip):5797" MERCURY_ENABLED=False
-$STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox config --set "BASE_URL=http://$(get_ip):5797"
 msg_ok "Initialized ArchiveBox"
 
-msg_info "Installing Extractors"
+msg_info "Installing Extractors (Patience)"
-# One unreachable extractor must not sink the install: @postlight/parser pulls a git
+AB_RUNTIME="/tmp/archivebox-runtime-$(id -u archivebox)"
-# dependency that npm refuses, and readability covers the same job.
+install -d -m 700 -o archivebox -g archivebox "$AB_RUNTIME"
-if $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox install; then
+$STD env HOME=/home/archivebox USER=archivebox LOGNAME=archivebox XDG_CONFIG_HOME=/home/archivebox/.config XDG_RUNTIME_DIR="$AB_RUNTIME" \
-  msg_ok "Installed Extractors"
+  /opt/archivebox/venv/bin/archivebox install
-else
+msg_ok "Installed Extractors"
-  msg_warn "Some extractors stayed unavailable - ArchiveBox runs without them"
-fi
 
 msg_info "Creating Service"
 cat <<EOF >/etc/systemd/system/archivebox.service
@@ -74,6 +60,7 @@ After=network.target
 [Service]
 User=archivebox
 WorkingDirectory=/opt/archivebox/data
+ExecStartPre=/opt/archivebox/venv/bin/archivebox init
 ExecStart=/opt/archivebox/venv/bin/archivebox server 0.0.0.0:5797
 Restart=always