Compare commits

..
Author SHA1 Message Date
MickLesk 8c9a017c63 ArchiveBox: keep the 1 GB default
Raising it to 2 GB made every existing container count as under-provisioned on update, and an unattended update stops there.
2026-11-16 14:00:54 +01:00
MickLesk 575e7921d2 Merge main into fix/archivebox-0.9
ArchiveBox runs as root and drops to its own account, so the service user, sudo and the BASE_URL guesswork go. The 0.9 update is pinned through check_for_gh_release, migrated containers drop the old system package, the Playwright cache and the NodeSource Node.js whose npm refused mercury's git dependency.
2026-11-16 13:34:25 +01:00
MickLesk dd38ed34e0 Let ArchiveBox 0.9 actually crawl, and serve its snapshots
Every extractor binary 0.9 cannot find aborts the crawl runner, and abx-plugins
keeps adding new ones, so a fixed apt list always falls behind while the service
user cannot run apt. archivebox install now runs as root with the service user's
home, the way upstream's own package does, and resolves every system dependency
itself; mercury stays disabled because npm refuses its git dependency. Snapshot
iframes are built from BASE_URL, so the update sets it when missing or when it is
a stale IP, and migrated 0.7 collections get their snapshots re-indexed.
2026-09-28 09:25:34 +02:00
MickLesk 049f2dd151 Name the admin account the way the docs do
createsuperuser was given admin, but the documented default user is archivebox -
which is what 0.7.4 produced, because its expect block sent an empty username and
Django then fell back to the OS user the command ran as.
2026-09-25 18:12:51 +02:00
MickLesk 57c299d068 Treat extractors as optional
@postlight/parser depends on a git URL and npm refuses to fetch those, so one
unreachable extractor aborted the whole install; readability and trafilatura cover
the same job and provision fine.
2026-09-25 16:14:52 +02:00
MickLesk 959a86983c Build the venv on the distro python and drop the obsolete extractors
A uv-managed interpreter lives where the service user cannot execute it, so every
archivebox call as that user died with 'Permission denied' while 0.9 drops privileges
on import; Debian 13 ships python3.13 in apt, so the venv now points at /usr/bin.
0.9 also provisions single-file and readability itself through abx-dl - the two npm
modules no longer exist - and it needs BASE_URL pinned plus tesseract, imagemagick,
ffmpeg, unzip and wget present, or archivebox install reaches for apt as a user that
has no root.
2026-09-25 14:30:30 +02:00
MickLesk eb1005e9b5 minor fixes 2026-09-25 13:43:44 +02:00
MickLesk 6a33fc45ad Let ArchiveBox reach 0.9 instead of staying on 0.7.4
uv pip install --system targets the container's Python 3.11, and every ArchiveBox
release from 0.9 on requires 3.13, so the resolver silently fell back to 0.7.4.
The package now lives in its own 3.13 venv, with the interpreter outside root's
home because ArchiveBox drops privileges while importing. The expect block is
gone: init and createsuperuser both run unattended now, and the server moved to
its new default port.
2026-09-25 11:30:16 +02:00
4 changed files with 182 additions and 186 deletions

No files matched your search

+38 -22
View File
@@ -12,7 +12,7 @@ var_cpu="${var_cpu:-2}"
var_ram="${var_ram:-1024}"
var_disk="${var_disk:-8}"
var_os="${var_os:-debian}"
var_version="${var_version:-12}"
var_version="${var_version:-13}"
var_arm64="${var_arm64:-yes}"
var_unprivileged="${var_unprivileged:-1}"
@@ -30,30 +30,46 @@ function update_script() {
exit
fi
NODE_VERSION="22" NODE_MODULE="@postlight/parser@latest,single-file-cli@latest" setup_nodejs
PYTHON_VERSION="3.13" setup_uv
if check_for_gh_release "archivebox" "ArchiveBox/ArchiveBox"; then
msg_info "Stopping ArchiveBox"
systemctl stop archivebox
msg_ok "Stopped ArchiveBox"
ensure_dependencies chromium
export UV_PYTHON_INSTALL_DIR="/opt/archivebox/python"
PYTHON_VERSION="3.13" setup_uv
ensure_dependencies chromium
msg_info "Stopping Service"
systemctl stop archivebox
msg_ok "Stopped Service"
if [[ ! -x /opt/archivebox/.venv/bin/archivebox ]]; then
msg_info "Migrating ArchiveBox to 0.9"
$STD uv pip uninstall --system --break-system-packages archivebox playwright
rm -rf /home/archivebox/.cache/ms-playwright
remove_old_tool_version "nodejs" "nodesource"
sed -i -e '/^User=archivebox$/d' -e 's|^ExecStart=/usr/local/bin/archivebox |ExecStart=/opt/archivebox/.venv/bin/archivebox |' /etc/systemd/system/archivebox.service
systemctl daemon-reload
$STD uv venv --python 3.13 /opt/archivebox/.venv
msg_ok "Migrated ArchiveBox to 0.9"
fi
msg_info "Upgrading Playwright"
$STD uv pip install playwright --system --break-system-packages
$STD playwright install-deps chromium
msg_ok "Upgraded Playwright"
msg_info "Updating ArchiveBox"
$STD uv pip install --python /opt/archivebox/.venv/bin/python "archivebox==${CHECK_UPDATE_RELEASE#v}"
ln -sf /opt/archivebox/.venv/bin/archivebox /usr/local/bin/archivebox
cd /opt/archivebox/data
$STD archivebox init
if ! grep -q "^BASE_URL" ArchiveBox.conf; then
$STD archivebox config --set "BASE_URL=http://${LOCAL_IP}:$(grep -oP 'server 0\.0\.0\.0:\K[0-9]+' /etc/systemd/system/archivebox.service)"
fi
$STD archivebox install
$STD archivebox update --migrate-only
cat <<EOF >~/.archivebox
${CHECK_UPDATE_RELEASE#v}
EOF
msg_ok "Updated ArchiveBox"
msg_info "Updating ArchiveBox"
cd /opt/archivebox/data
$STD uv pip install --system --break-system-packages --upgrade --no-reinstall archivebox
sudo -u archivebox archivebox init
msg_ok "Updated ArchiveBox"
msg_info "Starting Service"
systemctl start archivebox
msg_ok "Started Service"
msg_ok "Updated successfully!"
msg_info "Starting ArchiveBox"
systemctl start archivebox
msg_ok "Started ArchiveBox"
msg_ok "Updated successfully!"
fi
exit
}
@@ -64,4 +80,4 @@ description
msg_ok "Completed successfully!\n"
echo -e "${CREATING}${GN}${APP} setup has been successfully initialized!${CL}"
echo -e "${INFO}${YW}Access it using the following URL:${CL}"
echo -e "${GATEWAY}${BGN}http://${IP}:8000/admin/login${CL}"
echo -e "${GATEWAY}${BGN}http://${IP}:5797/admin/${CL}"
+38 -65
View File
@@ -31,7 +31,7 @@ function update_script() {
exit
fi
ensure_dependencies libjpeg-dev poppler-utils file locales xvfb xauth python3
ensure_dependencies libjpeg-dev poppler-utils file locales
msg_info "Generating Locales"
for l in cs_CZ de_DE en_GB en_US es_ES fr_FR id_ID it_IT ja_JP ko_KR \
@@ -44,83 +44,56 @@ function update_script() {
fi
msg_ok "Generated Locales"
NODE_VERSION="24" setup_nodejs
VENV_PATH="/opt/changedetection/.venv"
CHANGEDETECTION_BIN="${VENV_PATH}/bin/changedetection.io"
SERVICE_FILE="/etc/systemd/system/changedetection.service"
PYTHON_VERSION="3.13" setup_uv
msg_info "Updating ${APP}"
[[ -x "$CHANGEDETECTION_BIN" ]] || $STD uv venv --clear "$VENV_PATH"
$STD uv pip install --python "$VENV_PATH/bin/python" --upgrade changedetection.io playwright
if ! grep -q "$CHANGEDETECTION_BIN" "$SERVICE_FILE"; then
sed -i "s|^ExecStart=.*|ExecStart=$CHANGEDETECTION_BIN -d /opt/changedetection -p 5000|" "$SERVICE_FILE"
fi
msg_ok "Updated ${APP}"
if [[ ! -f /etc/apt/sources.list.d/google-chrome.sources ]]; then
msg_info "Installing Chrome"
setup_deb822_repo \
"google-chrome" \
"https://dl.google.com/linux/linux_signing_key.pub" \
"https://dl.google.com/linux/chrome/deb/" \
"stable"
$STD apt install -y google-chrome-stable
rm -f /etc/apt/sources.list.d/google-chrome.list
msg_ok "Installed Chrome"
if [[ ! -d "$VENV_PATH" || ! -x "$CHANGEDETECTION_BIN" ]]; then
msg_info "Migrating to uv/venv"
rm -rf "$VENV_PATH"
$STD uv venv --clear "$VENV_PATH"
$STD "$VENV_PATH/bin/python" -m ensurepip --upgrade
$STD "$VENV_PATH/bin/python" -m pip install --upgrade pip
$STD "$VENV_PATH/bin/python" -m pip install changedetection.io playwright
msg_ok "Migrated to uv/venv"
else
msg_info "Updating Chrome"
apt_update_safe
$STD apt install -y --only-upgrade google-chrome-stable
msg_ok "Updated Chrome"
msg_info "Updating ${APP}"
$STD "$VENV_PATH/bin/python" -m pip install --upgrade changedetection.io playwright
msg_ok "Updated ${APP}"
fi
if check_for_gh_release "sockpuppetbrowser" "dgtlmoon/sockpuppetbrowser"; then
[[ -f /etc/systemd/system/sockpuppetbrowser.service ]] && systemctl stop sockpuppetbrowser
CLEAN_INSTALL=1 fetch_and_deploy_gh_release "sockpuppetbrowser" "dgtlmoon/sockpuppetbrowser" "tarball"
msg_info "Setting up SockPuppetBrowser"
id -u chrome &>/dev/null || useradd -r -m -s /usr/sbin/nologin chrome
$STD uv venv --clear --python /usr/bin/python3 /opt/sockpuppetbrowser/.venv
$STD uv pip install --python /opt/sockpuppetbrowser/.venv/bin/python -r /opt/sockpuppetbrowser/requirements.txt
msg_ok "Set up SockPuppetBrowser"
SERVICE_FILE="/etc/systemd/system/changedetection.service"
if ! grep -q "${VENV_PATH}/bin/changedetection.io" "$SERVICE_FILE"; then
msg_info "Updating systemd service"
sed -i "s|^ExecStart=.*|ExecStart=${VENV_PATH}/bin/changedetection.io -d /opt/changedetection -p 5000|" "$SERVICE_FILE"
$STD systemctl daemon-reload
msg_ok "Updated systemd service"
fi
if [[ -f /etc/systemd/system/browserless.service ]]; then
msg_info "Replacing Browserless with SockPuppetBrowser"
systemctl disable -q --now browserless
rm -f /etc/systemd/system/browserless.service
rm -rf /opt/browserless /root/.cache/ms-playwright
$STD apt purge -y microsoft-edge-stable
rm -f /etc/apt/sources.list.d/microsoft-edge-stable.list
cat <<EOF >/etc/systemd/system/sockpuppetbrowser.service
[Unit]
Description=SockPuppetBrowser
After=network.target
[Service]
User=chrome
Group=chrome
WorkingDirectory=/opt/sockpuppetbrowser/backend
Environment=CHROME_BIN=/usr/bin/google-chrome-stable
Environment=LOG_LEVEL=INFO
ExecStart=/opt/sockpuppetbrowser/.venv/bin/python server.py --host 127.0.0.1 --port 3000 --sport 8080
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
sed -i 's/browserless\.service/sockpuppetbrowser.service/g' "$SERVICE_FILE"
sed -i -e '/^WEBDRIVER_URL=/d' \
-e 's|^PLAYWRIGHT_DRIVER_URL=ws://localhost:3000/chrome.*|PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000/?headful=true\&--window-size=1920,1080|' \
/opt/changedetection/.env
systemctl daemon-reload
systemctl enable -q sockpuppetbrowser
msg_ok "Replaced Browserless with SockPuppetBrowser"
msg_info "Updating Browserless (Patience)"
$STD git -C /opt/browserless/ fetch --all
$STD git -C /opt/browserless/ reset --hard origin/main
$STD npm ci --include=optional --include=dev --prefix /opt/browserless
$STD npm install --save-exact playwright-core@1.62.1 --prefix /opt/browserless
$STD /opt/browserless/node_modules/playwright-core/cli.js install --with-deps
# Update Chrome separately, as it has to be done with the force option. Otherwise the installation of other browsers will not be done if Chrome is already installed.
$STD /opt/browserless/node_modules/playwright-core/cli.js install --force chrome
$STD /opt/browserless/node_modules/playwright-core/cli.js install --force msedge
$STD /opt/browserless/node_modules/playwright-core/cli.js install chromium firefox webkit
$STD npm install --prefix /opt/browserless esbuild typescript ts-node @types/node --save-dev
$STD npm run build --prefix /opt/browserless
$STD npm run build:function --prefix /opt/browserless
$STD npm prune production --prefix /opt/browserless
systemctl restart browserless
msg_ok "Updated Browserless"
else
msg_error "No Browserless Installation Found!"
fi
systemctl daemon-reload
systemctl restart sockpuppetbrowser
systemctl restart changedetection
msg_ok "Updated successfully!"
exit
+27 -57
View File
@@ -14,69 +14,40 @@ network_check
update_os
msg_info "Installing Dependencies"
$STD apt install -y \
git \
expect \
libssl-dev \
libldap2-dev \
libsasl2-dev \
procps \
dnsutils \
ripgrep \
chromium
$STD apt install -y chromium
msg_ok "Installed Dependencies"
msg_info "Installing Python Dependencies"
$STD apt install -y \
python3-ldap \
python3-msgpack \
python3-regex
msg_ok "Installed Python Dependencies"
NODE_VERSION="22" NODE_MODULE="@postlight/parser@latest,single-file-cli@latest" setup_nodejs
export UV_PYTHON_INSTALL_DIR="/opt/archivebox/python"
PYTHON_VERSION="3.13" setup_uv
msg_info "Installing Playwright"
$STD uv pip install playwright --system --break-system-packages
$STD playwright install-deps chromium
msg_ok "Installed Playwright"
msg_info "Installing ArchiveBox"
mkdir -p /opt/archivebox/{data,.npm,.cache,.local}
$STD adduser --system --shell /bin/bash --gecos 'Archive Box User' --group --disabled-password --home /home/archivebox archivebox
chown -R archivebox:archivebox /opt/archivebox/{data,.npm,.cache,.local}
chmod -R 755 /opt/archivebox/data
$STD uv pip install archivebox --system --break-system-packages
cd /opt/archivebox/data
ADMIN_PASS=$(random_password 20)
expect <<EOF
set timeout -1
log_user 0
spawn sudo -u archivebox playwright install chromium
spawn sudo -u archivebox archivebox setup
expect "Username"
send "\r"
expect "Email address"
send "\r"
expect "Password"
send "${ADMIN_PASS}\r"
expect "Password (again)"
send "${ADMIN_PASS}\r"
expect eof
ARCHIVEBOX_VERSION="${var_appversion:-$(get_latest_github_release "ArchiveBox/ArchiveBox")}"
mkdir -p /opt/archivebox/data
$STD uv venv --python 3.13 /opt/archivebox/.venv
$STD uv pip install --python /opt/archivebox/.venv/bin/python "archivebox==${ARCHIVEBOX_VERSION#v}"
ln -sf /opt/archivebox/.venv/bin/archivebox /usr/local/bin/archivebox
cat <<EOF >~/.archivebox
${ARCHIVEBOX_VERSION#v}
EOF
{
echo "ArchiveBox Credentials"
echo "Username: archivebox"
echo "Password: ${ADMIN_PASS}"
} >~/archivebox.creds
msg_ok "Installed ArchiveBox"
msg_info "Initializing ArchiveBox"
cd /opt/archivebox/data
$STD archivebox init
ADMIN_PASS=$(random_password 20)
DJANGO_SUPERUSER_USERNAME=archivebox DJANGO_SUPERUSER_EMAIL=archivebox@archivebox.local DJANGO_SUPERUSER_PASSWORD="$ADMIN_PASS" $STD archivebox manage createsuperuser --noinput
$STD archivebox config --set "BASE_URL=http://${LOCAL_IP}:5797"
cat <<EOF >~/archivebox.creds
ArchiveBox Credentials
Username: archivebox
Password: ${ADMIN_PASS}
EOF
msg_ok "Initialized ArchiveBox"
msg_info "Installing ArchiveBox Extractors"
$STD archivebox install
msg_ok "Installed ArchiveBox Extractors"
msg_info "Creating Service"
cat <<EOF >/etc/systemd/system/archivebox.service
[Unit]
@@ -84,9 +55,8 @@ Description=ArchiveBox Server
After=network.target
[Service]
User=archivebox
WorkingDirectory=/opt/archivebox/data
ExecStart=/usr/local/bin/archivebox server 0.0.0.0:8000
ExecStart=/opt/archivebox/.venv/bin/archivebox server 0.0.0.0:5797
Restart=always
[Install]
+79 -42
View File
@@ -13,13 +13,32 @@ setting_up_container
network_check
update_os
msg_info "Installing Dependencies"
msg_info "Installing Dependencies (Patience)"
$STD apt install -y \
git \
build-essential \
dumb-init \
libjpeg-dev \
libatk-bridge2.0-0 \
libasound2 \
libatk1.0-0 \
libcairo2 \
libcups2 \
libdbus-1-3 \
libexpat1 \
libgbm-dev \
libgbm1 \
libgdk-pixbuf-2.0-0 \
libglib2.0-0 \
libgtk-3-0 \
libnspr4 \
libnss3 \
libpango-1.0-0 \
libpangocairo-1.0-0 \
qpdf \
xdg-utils \
xvfb \
xauth \
python3 \
ca-certificates \
locales \
poppler-utils \
file
@@ -35,25 +54,35 @@ msg_ok "Generated Locales"
PYTHON_VERSION="3.13" setup_uv
NODE_VERSION="24" setup_nodejs
msg_info "Installing Change Detection"
mkdir -p /opt/changedetection
$STD uv venv --clear /opt/changedetection/.venv
$STD uv pip install --python /opt/changedetection/.venv/bin/python changedetection.io playwright
$STD /opt/changedetection/.venv/bin/python -m ensurepip --upgrade
$STD /opt/changedetection/.venv/bin/python -m pip install --upgrade pip
$STD /opt/changedetection/.venv/bin/python -m pip install changedetection.io
cat <<EOF >/opt/changedetection/.env
PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000/?headful=true&--window-size=1920,1080
WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub
PLAYWRIGHT_DRIVER_URL=ws://localhost:3000/chrome?launch=eyJkZWZhdWx0Vmlld3BvcnQiOnsiaGVpZ2h0Ijo3MjAsIndpZHRoIjoxMjgwfSwiaGVhZGxlc3MiOmZhbHNlLCJzdGVhbHRoIjp0cnVlfQ==&blockAds=true
LC_ALL=en_US.UTF-8
EOF
msg_ok "Installed Change Detection"
msg_info "Installing Chrome"
setup_deb822_repo \
"google-chrome" \
"https://dl.google.com/linux/linux_signing_key.pub" \
"https://dl.google.com/linux/chrome/deb/" \
"stable"
$STD apt install -y google-chrome-stable
rm -f /etc/apt/sources.list.d/google-chrome.list
msg_ok "Installed Chrome"
msg_info "Installing Browserless & Playwright"
mkdir /opt/browserless
$STD /opt/changedetection/.venv/bin/python -m pip install playwright
$STD git clone https://github.com/browserless/chrome /opt/browserless
$STD npm ci --include=optional --include=dev --prefix /opt/browserless
$STD npm install --save-exact playwright-core@1.62.1 --prefix /opt/browserless
$STD /opt/browserless/node_modules/playwright-core/cli.js install --with-deps &>/dev/null
$STD /opt/browserless/node_modules/playwright-core/cli.js install --force chrome &>/dev/null
$STD /opt/browserless/node_modules/playwright-core/cli.js install chromium firefox webkit &>/dev/null
$STD /opt/browserless/node_modules/playwright-core/cli.js install --force msedge
$STD npm run build --prefix /opt/browserless
$STD npm run build:function --prefix /opt/browserless
$STD npm prune production --prefix /opt/browserless
msg_ok "Installed Browserless & Playwright"
msg_info "Installing Font Packages"
$STD apt install -y \
@@ -70,38 +99,30 @@ $STD apt install -y \
fonts-wqy-zenhei
msg_ok "Installed Font Packages"
fetch_and_deploy_gh_release "sockpuppetbrowser" "dgtlmoon/sockpuppetbrowser" "tarball"
msg_info "Setting up SockPuppetBrowser"
useradd -r -m -s /usr/sbin/nologin chrome
$STD uv venv --clear --python /usr/bin/python3 /opt/sockpuppetbrowser/.venv
$STD uv pip install --python /opt/sockpuppetbrowser/.venv/bin/python -r /opt/sockpuppetbrowser/requirements.txt
msg_ok "Set up SockPuppetBrowser"
msg_info "Installing X11 Packages"
$STD apt install -y \
libx11-6 \
libx11-xcb1 \
libxcb1 \
libxcomposite1 \
libxcursor1 \
libxdamage1 \
libxext6 \
libxfixes3 \
libxi6 \
libxrandr2 \
libxrender1 \
libxss1 \
libxtst6
msg_ok "Installed X11 Packages"
msg_info "Creating Services"
cat <<EOF >/etc/systemd/system/sockpuppetbrowser.service
[Unit]
Description=SockPuppetBrowser
After=network.target
[Service]
User=chrome
Group=chrome
WorkingDirectory=/opt/sockpuppetbrowser/backend
Environment=CHROME_BIN=/usr/bin/google-chrome-stable
Environment=LOG_LEVEL=INFO
ExecStart=/opt/sockpuppetbrowser/.venv/bin/python server.py --host 127.0.0.1 --port 3000 --sport 8080
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
cat <<EOF >/etc/systemd/system/changedetection.service
[Unit]
Description=Change Detection
After=network-online.target sockpuppetbrowser.service
Wants=sockpuppetbrowser.service
After=network-online.target
After=network.target browserless.service
Wants=browserless.service
[Service]
Type=simple
@@ -112,7 +133,23 @@ ExecStart=/opt/changedetection/.venv/bin/changedetection.io -d /opt/changedetect
[Install]
WantedBy=multi-user.target
EOF
systemctl enable -q --now sockpuppetbrowser changedetection
cat <<EOF >/etc/systemd/system/browserless.service
[Unit]
Description=browserless service
After=network.target
[Service]
Environment=CONNECTION_TIMEOUT=60000
WorkingDirectory=/opt/browserless
ExecStart=/opt/browserless/scripts/start.sh
SyslogIdentifier=browserless
[Install]
WantedBy=default.target
EOF
systemctl enable -q --now browserless
systemctl enable -q --now changedetection
msg_ok "Created Services"
motd_ssh