changedetection: replace Browserless with SockpuppetBrowser

ct/changedetection.sh

@@ -21,80 +21,175 @@ variables
 color
 catch_errors
 
+wait_http() {
+  local -r url="$1" timeout="${2:-30}"
+  local -r deadline=$((SECONDS + timeout))
+  local code
+  while ((SECONDS < deadline)); do
+    code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null)" || code="000"
+    [[ "$code" =~ ^([23][0-9][0-9]|401|403)$ ]] && return 0
+    sleep 2
+  done
+  return 1
+}
+
 function update_script() {
   header_info
   check_container_storage
   check_container_resources
 
-  if [[ ! -f /etc/systemd/system/changedetection.service ]]; then
+  local -r CDIO_DIR="/opt/changedetection"
+  local -r CDIO_ENV="${CDIO_DIR}/.env"
+  local -r CDIO_VENV="${CDIO_DIR}/.venv"
+  local -r CDIO_BIN="${CDIO_VENV}/bin/changedetection.io"
+  local -r CDIO_UNIT="/etc/systemd/system/changedetection.service"
+  local -r CDIO_URL="http://127.0.0.1:5000/"
+
+  local -r SOCKPUPPET_DIR="/opt/sockpuppet"
+  local -r SOCKPUPPET_UNIT="/etc/systemd/system/sockpuppet.service"
+  local -r SOCKPUPPET_STATS="http://127.0.0.1:8080/stats"
+
+  local -r BROWSERLESS_UNIT="/etc/systemd/system/browserless.service"
+
+  if [[ ! -f "$CDIO_UNIT" ]]; then
     msg_error "No ${APP} Installation Found!"
     exit
   fi
 
-  ensure_dependencies libjpeg-dev poppler-utils file locales
+  ensure_dependencies libjpeg-dev poppler-utils file locales xvfb xauth
 
-  msg_info "Generating Locales"
+  msg_info "Regenerating Locales"
   for l in cs_CZ de_DE en_GB en_US es_ES fr_FR id_ID it_IT ja_JP ko_KR \
     pl_PL pt_BR ru_RU tr_TR uk_UA zh_CN zh_TW; do
     sed -i "s/^# *${l}.UTF-8 UTF-8/${l}.UTF-8 UTF-8/" /etc/locale.gen
   done
   $STD locale-gen
-  if ! grep -q "^LC_ALL=" /opt/changedetection/.env 2>/dev/null; then
-    echo "LC_ALL=en_US.UTF-8" >>/opt/changedetection/.env
+  if ! grep -q "^LC_ALL=" "$CDIO_ENV" 2>/dev/null; then
+    echo "LC_ALL=en_US.UTF-8" >>"$CDIO_ENV"
   fi
-  msg_ok "Generated Locales"
-
-  NODE_VERSION="24" setup_nodejs
-
-  VENV_PATH="/opt/changedetection/.venv"
-  CHANGEDETECTION_BIN="${VENV_PATH}/bin/changedetection.io"
+  msg_ok "Regenerated Locales"
 
+  # sockpuppet runs as the non-root chrome user, so uv's Python must live outside /root (mode 700).
+  export UV_PYTHON_INSTALL_DIR=/opt/uv-python
   PYTHON_VERSION="3.13" setup_uv
 
-  if [[ ! -d "$VENV_PATH" || ! -x "$CHANGEDETECTION_BIN" ]]; then
+  if [[ ! -d "$CDIO_VENV" || ! -x "$CDIO_BIN" ]]; then
     msg_info "Migrating to uv/venv"
-    rm -rf "$VENV_PATH"
-    $STD uv venv --clear "$VENV_PATH"
-    $STD "$VENV_PATH/bin/python" -m ensurepip --upgrade
-    $STD "$VENV_PATH/bin/python" -m pip install --upgrade pip
-    $STD "$VENV_PATH/bin/python" -m pip install changedetection.io playwright
+    rm -rf "$CDIO_VENV"
+    $STD uv venv --clear "$CDIO_VENV"
+    $STD "$CDIO_VENV/bin/python" -m ensurepip --upgrade
+    $STD "$CDIO_VENV/bin/python" -m pip install --upgrade pip
+    $STD "$CDIO_VENV/bin/python" -m pip install changedetection.io playwright
     msg_ok "Migrated to uv/venv"
   else
     msg_info "Updating ${APP}"
-    $STD "$VENV_PATH/bin/python" -m pip install --upgrade changedetection.io playwright
+    $STD "$CDIO_VENV/bin/python" -m pip install --upgrade changedetection.io playwright
     msg_ok "Updated ${APP}"
   fi
 
-  SERVICE_FILE="/etc/systemd/system/changedetection.service"
-  if ! grep -q "${VENV_PATH}/bin/changedetection.io" "$SERVICE_FILE"; then
+  if ! grep -qF "$CDIO_BIN" "$CDIO_UNIT"; then
     msg_info "Updating systemd service"
-    sed -i "s|^ExecStart=.*|ExecStart=${VENV_PATH}/bin/changedetection.io -d /opt/changedetection -p 5000|" "$SERVICE_FILE"
+    sed -i "s|^ExecStart=.*|ExecStart=${CDIO_BIN} -d ${CDIO_DIR} -p 5000|" "$CDIO_UNIT"
     $STD systemctl daemon-reload
     msg_ok "Updated systemd service"
   fi
 
-  if [[ -f /etc/systemd/system/browserless.service ]]; then
-    msg_info "Updating Browserless (Patience)"
-    $STD git -C /opt/browserless/ fetch --all
-    $STD git -C /opt/browserless/ reset --hard origin/main
-    $STD npm ci --include=optional --include=dev --prefix /opt/browserless
-    $STD npm install --save-exact playwright-core@1.62.1 --prefix /opt/browserless
-    $STD /opt/browserless/node_modules/playwright-core/cli.js install --with-deps
-    # Update Chrome separately, as it has to be done with the force option. Otherwise the installation of other browsers will not be done if Chrome is already installed.
-    $STD /opt/browserless/node_modules/playwright-core/cli.js install --force chrome
-    $STD /opt/browserless/node_modules/playwright-core/cli.js install --force msedge
-    $STD /opt/browserless/node_modules/playwright-core/cli.js install chromium firefox webkit
-    $STD npm install --prefix /opt/browserless esbuild typescript ts-node @types/node --save-dev
-    $STD npm run build --prefix /opt/browserless
-    $STD npm run build:function --prefix /opt/browserless
-    $STD npm prune production --prefix /opt/browserless
-    systemctl restart browserless
-    msg_ok "Updated Browserless"
+  msg_info "Updating Chrome"
+  if [[ ! -f /etc/apt/sources.list.d/google-chrome.sources ]]; then
+    setup_deb822_repo "google-chrome" \
+      "https://dl.google.com/linux/linux_signing_key.pub" \
+      "https://dl.google.com/linux/chrome/deb/" "stable"
   else
-    msg_error "No Browserless Installation Found!"
+    apt_update_safe
+  fi
+  $STD apt install -y google-chrome-stable
+  msg_ok "Updated Chrome"
+
+  groupadd -f chrome 2>/dev/null || true
+  id -u chrome >/dev/null 2>&1 ||
+    useradd --system --create-home --shell /usr/sbin/nologin -g chrome chrome
+
+  if check_for_gh_branch "sockpuppet" "dgtlmoon/sockpuppetbrowser" "master"; then
+    systemctl stop sockpuppet 2>/dev/null || true
+    fetch_and_deploy_gh_branch "sockpuppet" "dgtlmoon/sockpuppetbrowser" "master"
+    $STD uv venv --clear --python 3.12 "${SOCKPUPPET_DIR}/.venv"
+    $STD uv pip install --python "${SOCKPUPPET_DIR}/.venv/bin/python" -r "${SOCKPUPPET_DIR}/requirements.txt"
+  fi
+
+  if [[ ! -f "$SOCKPUPPET_UNIT" ]]; then
+    cat <<EOF >"$SOCKPUPPET_UNIT"
+[Unit]
+Description=sockpuppetbrowser service
+After=network.target
+StartLimitIntervalSec=60
+StartLimitBurst=3
+
+[Service]
+User=chrome
+Group=chrome
+WorkingDirectory=${SOCKPUPPET_DIR}/backend
+Environment=CHROME_BIN=/usr/bin/google-chrome-stable
+Environment=CHROME_HEADFUL=true
+Environment=PYTHONDONTWRITEBYTECODE=1
+ExecStart=${SOCKPUPPET_DIR}/.venv/bin/python server.py --host 127.0.0.1 --port 3000 --sport 8080
+Restart=on-failure
+RestartSec=5
+SyslogIdentifier=sockpuppet
+
+[Install]
+WantedBy=default.target
+EOF
+  fi
+  systemctl enable -q sockpuppet
+
+  if [[ -f "$BROWSERLESS_UNIT" ]]; then
+    systemctl disable --now browserless 2>/dev/null || true
+  fi
+
+  if ! safe_service_restart sockpuppet || ! wait_http "$SOCKPUPPET_STATS" 30 ||
+    ! curl -sS --max-time 5 "$SOCKPUPPET_STATS" | grep -qE '"chrome_version":[[:space:]]*"Google Chrome'; then
+    msg_error "Sockpuppet failed to start"
+    systemctl status sockpuppet --no-pager -l | head -20 || true
+    systemctl disable --now sockpuppet 2>/dev/null || true
+    if [[ -f "$BROWSERLESS_UNIT" ]]; then
+      systemctl enable --now browserless 2>/dev/null || true
+      msg_warn "Restarted Browserless; changedetection was not switched"
+    fi
+    exit 1
+  fi
+  msg_ok "Started Sockpuppet"
+
+  local -r STAMP="$(date +%Y%m%d-%H%M%S)"
+
+  if grep -q "browserless\.service" "$CDIO_UNIT"; then
+    sed -i."$STAMP" 's/browserless\.service/sockpuppet.service/g' "$CDIO_UNIT"
+    $STD systemctl daemon-reload
+  fi
+
+  if grep -q '^WEBDRIVER_URL=' "$CDIO_ENV" 2>/dev/null ||
+    grep -q '^PLAYWRIGHT_DRIVER_URL=ws://localhost:3000/chrome' "$CDIO_ENV"; then
+    sed -i."$STAMP" -e '/^WEBDRIVER_URL=/d' -e '/^PLAYWRIGHT_DRIVER_URL=/d' "$CDIO_ENV"
+    echo "PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000/?headful=true&--window-size=1920,1080" >>"$CDIO_ENV"
+  fi
+
+  msg_info "Starting Change Detection"
+  if ! safe_service_restart changedetection || ! wait_http "$CDIO_URL" 30; then
+    msg_error "changedetection failed to start"
+    systemctl status changedetection --no-pager -l | head -20 || true
+    exit 1
+  fi
+  msg_ok "Started Change Detection"
+
+  if [[ -f "$BROWSERLESS_UNIT" ]]; then
+    msg_info "Removing Browserless"
+    systemctl disable --now browserless 2>/dev/null || true
+    rm -f "$BROWSERLESS_UNIT"
+    rm -rf /opt/browserless
+    rm -rf /root/.cache/ms-playwright
+    $STD systemctl daemon-reload
+    msg_ok "Removed Browserless"
   fi
 
-  systemctl restart changedetection
   msg_ok "Updated successfully!"
   exit
 }

install/changedetection-install.sh

@@ -13,11 +13,21 @@ setting_up_container
 network_check
 update_os
 
-msg_info "Installing Dependencies (Patience)"
+wait_http() {
+  local -r url="$1" timeout="${2:-30}"
+  local -r deadline=$((SECONDS + timeout))
+  local code
+  while ((SECONDS < deadline)); do
+    code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null)" || code="000"
+    [[ "$code" =~ ^([23][0-9][0-9]|401|403)$ ]] && return 0
+    sleep 2
+  done
+  return 1
+}
+
+msg_info "Installing Dependencies"
 $STD apt install -y \
-  git \
   build-essential \
-  dumb-init \
   libjpeg-dev \
   libatk-bridge2.0-0 \
   libasound2 \
@@ -38,7 +48,7 @@ $STD apt install -y \
   qpdf \
   xdg-utils \
   xvfb \
-  ca-certificates \
+  xauth \
   locales \
   poppler-utils \
   file
@@ -52,37 +62,37 @@ done
 $STD locale-gen
 msg_ok "Generated Locales"
 
+# sockpuppet runs as the non-root chrome user, so uv's Python must live outside /root (mode 700).
+export UV_PYTHON_INSTALL_DIR=/opt/uv-python
 PYTHON_VERSION="3.13" setup_uv
 
-NODE_VERSION="24" setup_nodejs
-
 msg_info "Installing Change Detection"
 mkdir -p /opt/changedetection
 $STD uv venv --clear /opt/changedetection/.venv
 $STD /opt/changedetection/.venv/bin/python -m ensurepip --upgrade
 $STD /opt/changedetection/.venv/bin/python -m pip install --upgrade pip
-$STD /opt/changedetection/.venv/bin/python -m pip install changedetection.io
+$STD /opt/changedetection/.venv/bin/python -m pip install changedetection.io playwright
 cat <<EOF >/opt/changedetection/.env
-WEBDRIVER_URL=http://127.0.0.1:4444/wd/hub
-PLAYWRIGHT_DRIVER_URL=ws://localhost:3000/chrome?launch=eyJkZWZhdWx0Vmlld3BvcnQiOnsiaGVpZ2h0Ijo3MjAsIndpZHRoIjoxMjgwfSwiaGVhZGxlc3MiOmZhbHNlLCJzdGVhbHRoIjp0cnVlfQ==&blockAds=true
+PLAYWRIGHT_DRIVER_URL=ws://127.0.0.1:3000/?headful=true&--window-size=1920,1080
 LC_ALL=en_US.UTF-8
 EOF
 msg_ok "Installed Change Detection"
 
-msg_info "Installing Browserless & Playwright"
-mkdir /opt/browserless
-$STD /opt/changedetection/.venv/bin/python -m pip install playwright
-$STD git clone https://github.com/browserless/chrome /opt/browserless
-$STD npm ci --include=optional --include=dev --prefix /opt/browserless
-$STD npm install --save-exact playwright-core@1.62.1 --prefix /opt/browserless
-$STD /opt/browserless/node_modules/playwright-core/cli.js install --with-deps &>/dev/null
-$STD /opt/browserless/node_modules/playwright-core/cli.js install --force chrome &>/dev/null
-$STD /opt/browserless/node_modules/playwright-core/cli.js install chromium firefox webkit &>/dev/null
-$STD /opt/browserless/node_modules/playwright-core/cli.js install --force msedge
-$STD npm run build --prefix /opt/browserless
-$STD npm run build:function --prefix /opt/browserless
-$STD npm prune production --prefix /opt/browserless
-msg_ok "Installed Browserless & Playwright"
+msg_info "Installing Chrome"
+setup_deb822_repo "google-chrome" \
+  "https://dl.google.com/linux/linux_signing_key.pub" \
+  "https://dl.google.com/linux/chrome/deb/" "stable"
+$STD apt install -y google-chrome-stable
+msg_ok "Installed Chrome"
+
+msg_info "Creating Chrome User"
+groupadd -f chrome 2>/dev/null || true
+id -u chrome >/dev/null 2>&1 || useradd --system --create-home --shell /usr/sbin/nologin -g chrome chrome
+msg_ok "Created Chrome User"
+
+fetch_and_deploy_gh_branch "sockpuppet" "dgtlmoon/sockpuppetbrowser" "master"
+$STD uv venv --python 3.12 /opt/sockpuppet/.venv
+$STD uv pip install --python /opt/sockpuppet/.venv/bin/python -r /opt/sockpuppet/requirements.txt
 
 msg_info "Installing Font Packages"
 $STD apt install -y \
@@ -121,8 +131,8 @@ cat <<EOF >/etc/systemd/system/changedetection.service
 [Unit]
 Description=Change Detection
 After=network-online.target
-After=network.target browserless.service
-Wants=browserless.service
+After=network.target sockpuppet.service
+Wants=sockpuppet.service
 
 [Service]
 Type=simple
@@ -134,24 +144,49 @@ ExecStart=/opt/changedetection/.venv/bin/changedetection.io -d /opt/changedetect
 WantedBy=multi-user.target
 EOF
 
-cat <<EOF >/etc/systemd/system/browserless.service
+cat <<EOF >/etc/systemd/system/sockpuppet.service
 [Unit]
-Description=browserless service
+Description=sockpuppetbrowser service
 After=network.target
+StartLimitIntervalSec=60
+StartLimitBurst=3
 
 [Service]
-Environment=CONNECTION_TIMEOUT=60000
-WorkingDirectory=/opt/browserless
-ExecStart=/opt/browserless/scripts/start.sh
-SyslogIdentifier=browserless
+User=chrome
+Group=chrome
+WorkingDirectory=/opt/sockpuppet/backend
+Environment=CHROME_BIN=/usr/bin/google-chrome-stable
+Environment=CHROME_HEADFUL=true
+Environment=PYTHONDONTWRITEBYTECODE=1
+ExecStart=/opt/sockpuppet/.venv/bin/python server.py --host 127.0.0.1 --port 3000 --sport 8080
+Restart=on-failure
+RestartSec=5
+SyslogIdentifier=sockpuppet
 
 [Install]
 WantedBy=default.target
 EOF
-systemctl enable -q --now browserless
+systemctl enable -q --now sockpuppet
 systemctl enable -q --now changedetection
 msg_ok "Created Services"
 
+msg_info "Verifying Sockpuppet"
+if ! wait_http "http://127.0.0.1:8080/stats" 30 ||
+  ! curl -sS --max-time 5 "http://127.0.0.1:8080/stats" | grep -qE '"chrome_version":[[:space:]]*"Google Chrome'; then
+  msg_error "Sockpuppet failed to start"
+  systemctl status sockpuppet --no-pager -l | head -20 || true
+  exit 1
+fi
+msg_ok "Verified Sockpuppet"
+
+msg_info "Verifying Change Detection"
+if ! wait_http "http://127.0.0.1:5000/" 30; then
+  msg_error "Change Detection is not responding"
+  systemctl status changedetection --no-pager -l | head -20 || true
+  exit 1
+fi
+msg_ok "Verified Change Detection"
+
 motd_ssh
 customize
 cleanup_lxc