From ddcb5237bc04733a3909d7d70c6402c504ba99b3 Mon Sep 17 00:00:00 2001 From: skkzsh Date: Wed, 11 Nov 2026 18:13:43 +0900 Subject: [PATCH] changedetection: replace Browserless with SockpuppetBrowser --- ct/changedetection.sh | 175 ++++++++++++++++++++++------- install/changedetection-install.sh | 99 ++++++++++------ 2 files changed, 202 insertions(+), 72 deletions(-) diff --git a/ct/changedetection.sh b/ct/changedetection.sh index 1a18cc0dd..5a769eaf3 100644 --- a/ct/changedetection.sh +++ b/ct/changedetection.sh @@ -21,80 +21,175 @@ variables color catch_errors +wait_http() { + local -r url="$1" timeout="${2:-30}" + local -r deadline=$((SECONDS + timeout)) + local code + while ((SECONDS < deadline)); do + code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null)" || code="000" + [[ "$code" =~ ^([23][0-9][0-9]|401|403)$ ]] && return 0 + sleep 2 + done + return 1 +} + function update_script() { header_info check_container_storage check_container_resources - if [[ ! -f /etc/systemd/system/changedetection.service ]]; then + local -r CDIO_DIR="/opt/changedetection" + local -r CDIO_ENV="${CDIO_DIR}/.env" + local -r CDIO_VENV="${CDIO_DIR}/.venv" + local -r CDIO_BIN="${CDIO_VENV}/bin/changedetection.io" + local -r CDIO_UNIT="/etc/systemd/system/changedetection.service" + local -r CDIO_URL="http://127.0.0.1:5000/" + + local -r SOCKPUPPET_DIR="/opt/sockpuppet" + local -r SOCKPUPPET_UNIT="/etc/systemd/system/sockpuppet.service" + local -r SOCKPUPPET_STATS="http://127.0.0.1:8080/stats" + + local -r BROWSERLESS_UNIT="/etc/systemd/system/browserless.service" + + if [[ ! -f "$CDIO_UNIT" ]]; then msg_error "No ${APP} Installation Found!" exit fi - ensure_dependencies libjpeg-dev poppler-utils file locales + ensure_dependencies libjpeg-dev poppler-utils file locales xvfb xauth - msg_info "Generating Locales" + msg_info "Regenerating Locales" for l in cs_CZ de_DE en_GB en_US es_ES fr_FR id_ID it_IT ja_JP ko_KR \ pl_PL pt_BR ru_RU tr_TR uk_UA zh_CN zh_TW; do sed -i "s/^# *${l}.UTF-8 UTF-8/${l}.UTF-8 UTF-8/" /etc/locale.gen done $STD locale-gen - if ! grep -q "^LC_ALL=" /opt/changedetection/.env 2>/dev/null; then - echo "LC_ALL=en_US.UTF-8" >>/opt/changedetection/.env + if ! grep -q "^LC_ALL=" "$CDIO_ENV" 2>/dev/null; then + echo "LC_ALL=en_US.UTF-8" >>"$CDIO_ENV" fi - msg_ok "Generated Locales" - - NODE_VERSION="24" setup_nodejs - - VENV_PATH="/opt/changedetection/.venv" - CHANGEDETECTION_BIN="${VENV_PATH}/bin/changedetection.io" + msg_ok "Regenerated Locales" + # sockpuppet runs as the non-root chrome user, so uv's Python must live outside /root (mode 700). + export UV_PYTHON_INSTALL_DIR=/opt/uv-python PYTHON_VERSION="3.13" setup_uv - if [[ ! -d "$VENV_PATH" || ! -x "$CHANGEDETECTION_BIN" ]]; then + if [[ ! -d "$CDIO_VENV" || ! -x "$CDIO_BIN" ]]; then msg_info "Migrating to uv/venv" - rm -rf "$VENV_PATH" - $STD uv venv --clear "$VENV_PATH" - $STD "$VENV_PATH/bin/python" -m ensurepip --upgrade - $STD "$VENV_PATH/bin/python" -m pip install --upgrade pip - $STD "$VENV_PATH/bin/python" -m pip install changedetection.io playwright + rm -rf "$CDIO_VENV" + $STD uv venv --clear "$CDIO_VENV" + $STD "$CDIO_VENV/bin/python" -m ensurepip --upgrade + $STD "$CDIO_VENV/bin/python" -m pip install --upgrade pip + $STD "$CDIO_VENV/bin/python" -m pip install changedetection.io playwright msg_ok "Migrated to uv/venv" else msg_info "Updating ${APP}" - $STD "$VENV_PATH/bin/python" -m pip install --upgrade changedetection.io playwright + $STD "$CDIO_VENV/bin/python" -m pip install --upgrade changedetection.io playwright msg_ok "Updated ${APP}" fi - SERVICE_FILE="/etc/systemd/system/changedetection.service" - if ! grep -q "${VENV_PATH}/bin/changedetection.io" "$SERVICE_FILE"; then + if ! grep -qF "$CDIO_BIN" "$CDIO_UNIT"; then msg_info "Updating systemd service" - sed -i "s|^ExecStart=.*|ExecStart=${VENV_PATH}/bin/changedetection.io -d /opt/changedetection -p 5000|" "$SERVICE_FILE" + sed -i "s|^ExecStart=.*|ExecStart=${CDIO_BIN} -d ${CDIO_DIR} -p 5000|" "$CDIO_UNIT" $STD systemctl daemon-reload msg_ok "Updated systemd service" fi - if [[ -f /etc/systemd/system/browserless.service ]]; then - msg_info "Updating Browserless (Patience)" - $STD git -C /opt/browserless/ fetch --all - $STD git -C /opt/browserless/ reset --hard origin/main - $STD npm ci --include=optional --include=dev --prefix /opt/browserless - $STD npm install --save-exact playwright-core@1.62.1 --prefix /opt/browserless - $STD /opt/browserless/node_modules/playwright-core/cli.js install --with-deps - # Update Chrome separately, as it has to be done with the force option. Otherwise the installation of other browsers will not be done if Chrome is already installed. - $STD /opt/browserless/node_modules/playwright-core/cli.js install --force chrome - $STD /opt/browserless/node_modules/playwright-core/cli.js install --force msedge - $STD /opt/browserless/node_modules/playwright-core/cli.js install chromium firefox webkit - $STD npm install --prefix /opt/browserless esbuild typescript ts-node @types/node --save-dev - $STD npm run build --prefix /opt/browserless - $STD npm run build:function --prefix /opt/browserless - $STD npm prune production --prefix /opt/browserless - systemctl restart browserless - msg_ok "Updated Browserless" + msg_info "Updating Chrome" + if [[ ! -f /etc/apt/sources.list.d/google-chrome.sources ]]; then + setup_deb822_repo "google-chrome" \ + "https://dl.google.com/linux/linux_signing_key.pub" \ + "https://dl.google.com/linux/chrome/deb/" "stable" else - msg_error "No Browserless Installation Found!" + apt_update_safe + fi + $STD apt install -y google-chrome-stable + msg_ok "Updated Chrome" + + groupadd -f chrome 2>/dev/null || true + id -u chrome >/dev/null 2>&1 || + useradd --system --create-home --shell /usr/sbin/nologin -g chrome chrome + + if check_for_gh_branch "sockpuppet" "dgtlmoon/sockpuppetbrowser" "master"; then + systemctl stop sockpuppet 2>/dev/null || true + fetch_and_deploy_gh_branch "sockpuppet" "dgtlmoon/sockpuppetbrowser" "master" + $STD uv venv --clear --python 3.12 "${SOCKPUPPET_DIR}/.venv" + $STD uv pip install --python "${SOCKPUPPET_DIR}/.venv/bin/python" -r "${SOCKPUPPET_DIR}/requirements.txt" + fi + + if [[ ! -f "$SOCKPUPPET_UNIT" ]]; then + cat <"$SOCKPUPPET_UNIT" +[Unit] +Description=sockpuppetbrowser service +After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Service] +User=chrome +Group=chrome +WorkingDirectory=${SOCKPUPPET_DIR}/backend +Environment=CHROME_BIN=/usr/bin/google-chrome-stable +Environment=CHROME_HEADFUL=true +Environment=PYTHONDONTWRITEBYTECODE=1 +ExecStart=${SOCKPUPPET_DIR}/.venv/bin/python server.py --host 127.0.0.1 --port 3000 --sport 8080 +Restart=on-failure +RestartSec=5 +SyslogIdentifier=sockpuppet + +[Install] +WantedBy=default.target +EOF + fi + systemctl enable -q sockpuppet + + if [[ -f "$BROWSERLESS_UNIT" ]]; then + systemctl disable --now browserless 2>/dev/null || true + fi + + if ! safe_service_restart sockpuppet || ! wait_http "$SOCKPUPPET_STATS" 30 || + ! curl -sS --max-time 5 "$SOCKPUPPET_STATS" | grep -qE '"chrome_version":[[:space:]]*"Google Chrome'; then + msg_error "Sockpuppet failed to start" + systemctl status sockpuppet --no-pager -l | head -20 || true + systemctl disable --now sockpuppet 2>/dev/null || true + if [[ -f "$BROWSERLESS_UNIT" ]]; then + systemctl enable --now browserless 2>/dev/null || true + msg_warn "Restarted Browserless; changedetection was not switched" + fi + exit 1 + fi + msg_ok "Started Sockpuppet" + + local -r STAMP="$(date +%Y%m%d-%H%M%S)" + + if grep -q "browserless\.service" "$CDIO_UNIT"; then + sed -i."$STAMP" 's/browserless\.service/sockpuppet.service/g' "$CDIO_UNIT" + $STD systemctl daemon-reload + fi + + if grep -q '^WEBDRIVER_URL=' "$CDIO_ENV" 2>/dev/null || + grep -q '^PLAYWRIGHT_DRIVER_URL=ws://localhost:3000/chrome' "$CDIO_ENV"; then + sed -i."$STAMP" -e '/^WEBDRIVER_URL=/d' -e '/^PLAYWRIGHT_DRIVER_URL=/d' "$CDIO_ENV" + echo "PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000/?headful=true&--window-size=1920,1080" >>"$CDIO_ENV" + fi + + msg_info "Starting Change Detection" + if ! safe_service_restart changedetection || ! wait_http "$CDIO_URL" 30; then + msg_error "changedetection failed to start" + systemctl status changedetection --no-pager -l | head -20 || true + exit 1 + fi + msg_ok "Started Change Detection" + + if [[ -f "$BROWSERLESS_UNIT" ]]; then + msg_info "Removing Browserless" + systemctl disable --now browserless 2>/dev/null || true + rm -f "$BROWSERLESS_UNIT" + rm -rf /opt/browserless + rm -rf /root/.cache/ms-playwright + $STD systemctl daemon-reload + msg_ok "Removed Browserless" fi - systemctl restart changedetection msg_ok "Updated successfully!" exit } diff --git a/install/changedetection-install.sh b/install/changedetection-install.sh index 71b321a34..1ce44be9b 100644 --- a/install/changedetection-install.sh +++ b/install/changedetection-install.sh @@ -13,11 +13,21 @@ setting_up_container network_check update_os -msg_info "Installing Dependencies (Patience)" +wait_http() { + local -r url="$1" timeout="${2:-30}" + local -r deadline=$((SECONDS + timeout)) + local code + while ((SECONDS < deadline)); do + code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null)" || code="000" + [[ "$code" =~ ^([23][0-9][0-9]|401|403)$ ]] && return 0 + sleep 2 + done + return 1 +} + +msg_info "Installing Dependencies" $STD apt install -y \ - git \ build-essential \ - dumb-init \ libjpeg-dev \ libatk-bridge2.0-0 \ libasound2 \ @@ -38,7 +48,7 @@ $STD apt install -y \ qpdf \ xdg-utils \ xvfb \ - ca-certificates \ + xauth \ locales \ poppler-utils \ file @@ -52,37 +62,37 @@ done $STD locale-gen msg_ok "Generated Locales" +# sockpuppet runs as the non-root chrome user, so uv's Python must live outside /root (mode 700). +export UV_PYTHON_INSTALL_DIR=/opt/uv-python PYTHON_VERSION="3.13" setup_uv -NODE_VERSION="24" setup_nodejs - msg_info "Installing Change Detection" mkdir -p /opt/changedetection $STD uv venv --clear /opt/changedetection/.venv $STD /opt/changedetection/.venv/bin/python -m ensurepip --upgrade $STD /opt/changedetection/.venv/bin/python -m pip install --upgrade pip -$STD /opt/changedetection/.venv/bin/python -m pip install changedetection.io +$STD /opt/changedetection/.venv/bin/python -m pip install changedetection.io playwright cat </opt/changedetection/.env -WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub -PLAYWRIGHT_DRIVER_URL=ws://localhost:3000/chrome?launch=eyJkZWZhdWx0Vmlld3BvcnQiOnsiaGVpZ2h0Ijo3MjAsIndpZHRoIjoxMjgwfSwiaGVhZGxlc3MiOmZhbHNlLCJzdGVhbHRoIjp0cnVlfQ==&blockAds=true +PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000/?headful=true&--window-size=1920,1080 LC_ALL=en_US.UTF-8 EOF msg_ok "Installed Change Detection" -msg_info "Installing Browserless & Playwright" -mkdir /opt/browserless -$STD /opt/changedetection/.venv/bin/python -m pip install playwright -$STD git clone https://github.com/browserless/chrome /opt/browserless -$STD npm ci --include=optional --include=dev --prefix /opt/browserless -$STD npm install --save-exact playwright-core@1.62.1 --prefix /opt/browserless -$STD /opt/browserless/node_modules/playwright-core/cli.js install --with-deps &>/dev/null -$STD /opt/browserless/node_modules/playwright-core/cli.js install --force chrome &>/dev/null -$STD /opt/browserless/node_modules/playwright-core/cli.js install chromium firefox webkit &>/dev/null -$STD /opt/browserless/node_modules/playwright-core/cli.js install --force msedge -$STD npm run build --prefix /opt/browserless -$STD npm run build:function --prefix /opt/browserless -$STD npm prune production --prefix /opt/browserless -msg_ok "Installed Browserless & Playwright" +msg_info "Installing Chrome" +setup_deb822_repo "google-chrome" \ + "https://dl.google.com/linux/linux_signing_key.pub" \ + "https://dl.google.com/linux/chrome/deb/" "stable" +$STD apt install -y google-chrome-stable +msg_ok "Installed Chrome" + +msg_info "Creating Chrome User" +groupadd -f chrome 2>/dev/null || true +id -u chrome >/dev/null 2>&1 || useradd --system --create-home --shell /usr/sbin/nologin -g chrome chrome +msg_ok "Created Chrome User" + +fetch_and_deploy_gh_branch "sockpuppet" "dgtlmoon/sockpuppetbrowser" "master" +$STD uv venv --python 3.12 /opt/sockpuppet/.venv +$STD uv pip install --python /opt/sockpuppet/.venv/bin/python -r /opt/sockpuppet/requirements.txt msg_info "Installing Font Packages" $STD apt install -y \ @@ -121,8 +131,8 @@ cat </etc/systemd/system/changedetection.service [Unit] Description=Change Detection After=network-online.target -After=network.target browserless.service -Wants=browserless.service +After=network.target sockpuppet.service +Wants=sockpuppet.service [Service] Type=simple @@ -134,24 +144,49 @@ ExecStart=/opt/changedetection/.venv/bin/changedetection.io -d /opt/changedetect WantedBy=multi-user.target EOF -cat </etc/systemd/system/browserless.service +cat </etc/systemd/system/sockpuppet.service [Unit] -Description=browserless service +Description=sockpuppetbrowser service After=network.target +StartLimitIntervalSec=60 +StartLimitBurst=3 [Service] -Environment=CONNECTION_TIMEOUT=60000 -WorkingDirectory=/opt/browserless -ExecStart=/opt/browserless/scripts/start.sh -SyslogIdentifier=browserless +User=chrome +Group=chrome +WorkingDirectory=/opt/sockpuppet/backend +Environment=CHROME_BIN=/usr/bin/google-chrome-stable +Environment=CHROME_HEADFUL=true +Environment=PYTHONDONTWRITEBYTECODE=1 +ExecStart=/opt/sockpuppet/.venv/bin/python server.py --host 127.0.0.1 --port 3000 --sport 8080 +Restart=on-failure +RestartSec=5 +SyslogIdentifier=sockpuppet [Install] WantedBy=default.target EOF -systemctl enable -q --now browserless +systemctl enable -q --now sockpuppet systemctl enable -q --now changedetection msg_ok "Created Services" +msg_info "Verifying Sockpuppet" +if ! wait_http "http://127.0.0.1:8080/stats" 30 || + ! curl -sS --max-time 5 "http://127.0.0.1:8080/stats" | grep -qE '"chrome_version":[[:space:]]*"Google Chrome'; then + msg_error "Sockpuppet failed to start" + systemctl status sockpuppet --no-pager -l | head -20 || true + exit 1 +fi +msg_ok "Verified Sockpuppet" + +msg_info "Verifying Change Detection" +if ! wait_http "http://127.0.0.1:5000/" 30; then + msg_error "Change Detection is not responding" + systemctl status changedetection --no-pager -l | head -20 || true + exit 1 +fi +msg_ok "Verified Change Detection" + motd_ssh customize cleanup_lxc