Let ArchiveBox 0.9 actually crawl, and serve its snapshots

Every extractor binary 0.9 cannot find aborts the crawl runner, and abx-plugins
keeps adding new ones, so a fixed apt list always falls behind while the service
user cannot run apt. archivebox install now runs as root with the service user's
home, the way upstream's own package does, and resolves every system dependency
itself; mercury stays disabled because npm refuses its git dependency. Snapshot
iframes are built from BASE_URL, so the update sets it when missing or when it is
a stale IP, and migrated 0.7 collections get their snapshots re-indexed.

ct/archivebox.sh

@@ -31,34 +31,29 @@ function update_script() {
     exit
   fi
 
-  NODE_VERSION="22" setup_nodejs
   setup_uv
 
-  ensure_dependencies chromium ripgrep tesseract-ocr tesseract-ocr-eng imagemagick ffmpeg unzip wget
-
   msg_info "Stopping Service"
   systemctl stop archivebox
   msg_ok "Stopped Service"
 
-  # Earlier installs used python3.11, which caps ArchiveBox at 0.7.4.
+  local migrate=0 py port base runtime
   if [[ ! -x /opt/archivebox/venv/bin/archivebox ]]; then
+    migrate=1
     msg_info "Moving ArchiveBox to its own Python 3.13 environment"
-    # The distro interpreter first: a uv-managed one lands where the service user cannot exec it.
-    local py="/usr/bin/python3.13"
-    if [[ ! -x "$py" ]]; then
-      ensure_dependencies python3.13 || true
-    fi
+    py="/usr/bin/python3.13"
+    [[ -x "$py" ]] || ensure_dependencies python3.13 || true
     if [[ ! -x "$py" ]]; then
       $STD uv python install --install-dir /opt/archivebox/python 3.13
       py="$(UV_PYTHON_INSTALL_DIR=/opt/archivebox/python uv python find 3.13)"
     fi
     $STD uv venv --python "$py" /opt/archivebox/venv
-    # Keep whatever port this container already answers on, or a reverse proxy in front
-    # of it would silently stop resolving.
-    local port
-    port="$(awk -F: '/^ExecStart=/ {print $NF}' /etc/systemd/system/archivebox.service 2>/dev/null)"
-    [[ "$port" =~ ^[0-9]+$ ]] || port=5797
-    cat <<EOF >/etc/systemd/system/archivebox.service
+    msg_ok "Moved ArchiveBox to its own Python 3.13 environment"
+  fi
+
+  port="$(awk -F: '/^ExecStart=/ {print $NF}' /etc/systemd/system/archivebox.service 2>/dev/null)"
+  [[ "$port" =~ ^[0-9]+$ ]] || port=5797
+  cat <<EOF >/etc/systemd/system/archivebox.service
 [Unit]
 Description=ArchiveBox Server
 After=network.target
@@ -66,28 +61,45 @@ After=network.target
 [Service]
 User=archivebox
 WorkingDirectory=/opt/archivebox/data
+ExecStartPre=/opt/archivebox/venv/bin/archivebox init
 ExecStart=/opt/archivebox/venv/bin/archivebox server 0.0.0.0:${port}
 Restart=always
 
 [Install]
 WantedBy=multi-user.target
 EOF
-    systemctl daemon-reload
-    msg_ok "Moved ArchiveBox to its own Python 3.13 environment (port ${port})"
-  fi
+  systemctl daemon-reload
 
   msg_info "Updating ArchiveBox"
   $STD uv pip install --python /opt/archivebox/venv/bin/python --upgrade archivebox
-  chown -R archivebox:archivebox /opt/archivebox
-  # Without this a shell still reaches the old 0.7.4 entry point against a 0.9 collection.
   ln -sf /opt/archivebox/venv/bin/archivebox /usr/local/bin/archivebox
+  mkdir -p /home/archivebox/.config
+  chown -R archivebox:archivebox /opt/archivebox /home/archivebox
   cd /opt/archivebox/data
   $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox init
-  # npm refuses the git dependency behind @postlight/parser, so treat extractors as optional.
-  $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox install ||
-    msg_warn "Some extractors stayed unavailable - ArchiveBox runs without them"
+  base="$(grep -oP '^BASE_URL\s*=\s*\K\S+' ArchiveBox.conf 2>/dev/null || true)"
+  if [[ -z "$base" || "$base" =~ ^https?://[0-9.]+(:[0-9]+)?/?$ ]]; then
+    base="http://$(hostname -I | awk '{print $1}'):${port}"
+  fi
+  $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox config --set "BASE_URL=${base}" MERCURY_ENABLED=False
   msg_ok "Updated ArchiveBox"
 
+  msg_info "Installing Extractors (Patience)"
+  runtime="/tmp/archivebox-runtime-$(id -u archivebox)"
+  install -d -m 700 -o archivebox -g archivebox "$runtime"
+  if $STD env HOME=/home/archivebox USER=archivebox LOGNAME=archivebox XDG_CONFIG_HOME=/home/archivebox/.config XDG_RUNTIME_DIR="$runtime" \
+    /opt/archivebox/venv/bin/archivebox install; then
+    msg_ok "Installed Extractors"
+  else
+    msg_warn "Some extractors failed to install - crawls stop until the next update succeeds"
+  fi
+
+  if [[ "$migrate" == 1 ]]; then
+    msg_info "Migrating Existing Snapshots (Patience)"
+    $STD sudo -u archivebox /opt/archivebox/venv/bin/archivebox update --rescan --migrate-only --index-only
+    msg_ok "Migrated Existing Snapshots"
+  fi
+
   msg_info "Starting Service"
   systemctl start archivebox
   msg_ok "Started Service"