From 3aca471c7bfbccf54e6ee8af088a8915bb8255bc Mon Sep 17 00:00:00 2001 From: "CanbiZ (MickLesk)" <47820557+MickLesk@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:30:11 +0100 Subject: [PATCH 1/4] Immich: bump to 3.4.0 (#18600) 3.4.0 adds FUTO Backups, which run restic from PATH and use restic-proxy when it is there. Both are built from the refs the server Dockerfile pins, and an update only rebuilds them when those change. pigz is installed, which database backups now use for compression when present. mise skips its own restic, and the extism-js fallback follows the js-pdk version in mise.toml (v1.7.0) instead of a fixed v1.6.0. --- ct/immich.sh | 30 ++++++++++++++++++++++++++---- install/immich-install.sh | 27 ++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/ct/immich.sh b/ct/immich.sh index dd422937a..2849ba083 100644 --- a/ct/immich.sh +++ b/ct/immich.sh @@ -75,7 +75,7 @@ EOF BASE_DIR=${STAGING_DIR}/base-images SOURCE_DIR=${STAGING_DIR}/image-source cd /tmp - RELEASE="v3.3.6" + RELEASE="v3.4.0" if [[ -f ~/.intel_version ]]; then INTEL_SRC="https://raw.githubusercontent.com/immich-app/immich/${RELEASE}/machine-learning" curl -fsSL "${INTEL_SRC}/scripts/install-intel-runtime.sh" -o ./intel-runtime 2>/dev/null || @@ -109,7 +109,7 @@ EOF libraries=("libjxl" "jpegli" "libheif" "libraw" "imagemagick" "libvips") cd "$BASE_DIR" msg_warn "Checking for updates to custom image-processing libraries (recompile time: 2-15min per library)" - ensure_dependencies liblcms2-dev libjpeg62-turbo-dev libspng-dev libexif-dev libopenjp2-7-dev + ensure_dependencies liblcms2-dev libjpeg62-turbo-dev libspng-dev libexif-dev libopenjp2-7-dev pigz $STD git pull for library in "${libraries[@]}"; do compile_"$library" @@ -234,7 +234,7 @@ EOF # plugins cd "$SRC_DIR" export MISE_TRUSTED_CONFIG_PATHS="$SRC_DIR"/mise.toml - export MISE_DISABLE_TOOLS=github:jellyfin/jellyfin-ffmpeg,java,terragrunt,opentofu,npm:@openapitools/openapi-generator-cli + export MISE_DISABLE_TOOLS=github:jellyfin/jellyfin-ffmpeg,java,terragrunt,opentofu,restic,npm:@openapitools/openapi-generator-cli export MISE_LOCKFILE_MODE=merge MISE_LOCKFILE_PLATFORMS="linux-$(arch_resolve x64 arm64)" mise_ok=0 for i in 1 2 3; do @@ -251,7 +251,8 @@ EOF # extism-js ships as a bare gzip-compressed single binary (.gz) that # fetch_and_deploy_gh_release cannot deploy; fetch + gunzip it directly. EXTISM_ARCH="$(arch_resolve x86_64 aarch64)" - curl_download /tmp/extism-js.gz "https://github.com/extism/js-pdk/releases/download/v1.6.0/extism-js-${EXTISM_ARCH}-linux-v1.6.0.gz" + EXTISM_VERSION="$(grep -oP '"github:extism/js-pdk"\s*=\s*"\K[^"]+' "$SRC_DIR"/mise.toml)" + curl_download /tmp/extism-js.gz "https://github.com/extism/js-pdk/releases/download/${EXTISM_VERSION}/extism-js-${EXTISM_ARCH}-linux-${EXTISM_VERSION}.gz" gunzip -f /tmp/extism-js.gz install -m 0755 /tmp/extism-js /usr/local/bin/extism-js rm -f /tmp/extism-js @@ -276,6 +277,27 @@ EOF cp ./packages/plugin-core/manifest.json "$PLUGIN_DIR" msg_ok "Updated Immich server, web, cli and plugins" + RESTIC_REF="$(grep -oP 'futo-org/restic:sha-\K[0-9a-f]+' "$SRC_DIR"/server/Dockerfile)" + RESTIC_PROXY_REF="$(grep -oP 'futo-org/backups/restic-proxy:\K[^@]+' "$SRC_DIR"/server/Dockerfile)" + if [[ "$(cat ~/.immich_restic 2>/dev/null)" != "${RESTIC_REF} ${RESTIC_PROXY_REF}" || ! -x /usr/local/bin/restic ]]; then + FUTO_DIR=/opt/staging/futo + rm -rf "$FUTO_DIR" + $STD git clone --filter=blob:none --no-checkout https://github.com/futo-org/restic "$FUTO_DIR"/restic + $STD git -C "$FUTO_DIR"/restic checkout "$RESTIC_REF" + $STD git clone --depth 1 --filter=blob:none --sparse -b "$RESTIC_PROXY_REF" https://github.com/futo-org/backups "$FUTO_DIR"/backups + $STD git -C "$FUTO_DIR"/backups sparse-checkout set packages/restic-proxy + GO_VERSION="$(awk '/^go /{print $2}' "$FUTO_DIR"/backups/packages/restic-proxy/go.mod)" setup_go + msg_info "Building restic for FUTO Backups" + cd "$FUTO_DIR"/restic + $STD env CGO_ENABLED=0 go build -o /usr/local/bin/restic ./cmd/restic + cd "$FUTO_DIR"/backups/packages/restic-proxy + $STD env CGO_ENABLED=0 go build -o /usr/local/bin/restic-proxy . + cd "$SRC_DIR" + rm -rf "$FUTO_DIR" + echo "${RESTIC_REF} ${RESTIC_PROXY_REF}" >~/.immich_restic + msg_ok "Built restic for FUTO Backups" + fi + cd "$SRC_DIR"/machine-learning mkdir -p "$ML_DIR" # chown excluding upload dir contents (may be a mount with restricted permissions) diff --git a/install/immich-install.sh b/install/immich-install.sh index 23b4d7401..783eb0551 100644 --- a/install/immich-install.sh +++ b/install/immich-install.sh @@ -23,7 +23,7 @@ if [[ "$(arch_resolve)" == "amd64" ]] && ! grep -qm1 sse4_2 /proc/cpuinfo; then exit 236 fi -RELEASE="v3.3.6" +RELEASE="v3.4.0" if lscpu | grep -q 'GenuineIntel'; then echo "" echo "" @@ -68,6 +68,7 @@ msg_info "Installing dependencies" $STD apt install --no-install-recommends -y \ git \ redis \ + pigz \ autoconf \ build-essential \ python3-dev \ @@ -415,7 +416,7 @@ cp -a web/build "$APP_DIR"/www cp LICENSE "$APP_DIR" cd "$SRC_DIR" export MISE_TRUSTED_CONFIG_PATHS="$SRC_DIR"/mise.toml -export MISE_DISABLE_TOOLS=github:jellyfin/jellyfin-ffmpeg,java,terragrunt,opentofu,npm:@openapitools/openapi-generator-cli +export MISE_DISABLE_TOOLS=github:jellyfin/jellyfin-ffmpeg,java,terragrunt,opentofu,restic,npm:@openapitools/openapi-generator-cli export MISE_LOCKFILE_MODE=merge MISE_LOCKFILE_PLATFORMS="linux-$(arch_resolve x64 arm64)" mise_ok=0 for i in 1 2 3; do @@ -433,7 +434,8 @@ if ! command -v extism-js >/dev/null 2>&1; then # fetch_and_deploy_gh_release cannot deploy (singlefile leaves it compressed, # prebuild only handles zip/tar). Fetch + gunzip it directly. EXTISM_ARCH="$(arch_resolve x86_64 aarch64)" - curl_download /tmp/extism-js.gz "https://github.com/extism/js-pdk/releases/download/v1.6.0/extism-js-${EXTISM_ARCH}-linux-v1.6.0.gz" + EXTISM_VERSION="$(grep -oP '"github:extism/js-pdk"\s*=\s*"\K[^"]+' "$SRC_DIR"/mise.toml)" + curl_download /tmp/extism-js.gz "https://github.com/extism/js-pdk/releases/download/${EXTISM_VERSION}/extism-js-${EXTISM_ARCH}-linux-${EXTISM_VERSION}.gz" gunzip -f /tmp/extism-js.gz install -m 0755 /tmp/extism-js /usr/local/bin/extism-js rm -f /tmp/extism-js @@ -445,6 +447,25 @@ cp -r ./packages/plugin-core/dist "$PLUGIN_DIR"/dist cp ./packages/plugin-core/manifest.json "$PLUGIN_DIR" msg_ok "Installed Immich Server, Web and Plugin Components" +RESTIC_REF="$(grep -oP 'futo-org/restic:sha-\K[0-9a-f]+' "$SRC_DIR"/server/Dockerfile)" +RESTIC_PROXY_REF="$(grep -oP 'futo-org/backups/restic-proxy:\K[^@]+' "$SRC_DIR"/server/Dockerfile)" +FUTO_DIR=/opt/staging/futo +rm -rf "$FUTO_DIR" +$STD git clone --filter=blob:none --no-checkout https://github.com/futo-org/restic "$FUTO_DIR"/restic +$STD git -C "$FUTO_DIR"/restic checkout "$RESTIC_REF" +$STD git clone --depth 1 --filter=blob:none --sparse -b "$RESTIC_PROXY_REF" https://github.com/futo-org/backups "$FUTO_DIR"/backups +$STD git -C "$FUTO_DIR"/backups sparse-checkout set packages/restic-proxy +GO_VERSION="$(awk '/^go /{print $2}' "$FUTO_DIR"/backups/packages/restic-proxy/go.mod)" setup_go +msg_info "Building restic for FUTO Backups" +cd "$FUTO_DIR"/restic +$STD env CGO_ENABLED=0 go build -o /usr/local/bin/restic ./cmd/restic +cd "$FUTO_DIR"/backups/packages/restic-proxy +$STD env CGO_ENABLED=0 go build -o /usr/local/bin/restic-proxy . +cd "$SRC_DIR" +rm -rf "$FUTO_DIR" +echo "${RESTIC_REF} ${RESTIC_PROXY_REF}" >~/.immich_restic +msg_ok "Built restic for FUTO Backups" + cd "$SRC_DIR"/machine-learning $STD useradd -U -s /usr/sbin/nologin -r -M -d "$INSTALL_DIR" immich mkdir -p "$ML_DIR" From b60ff8540a7f5819eaad8e32db25e2d8fd5d720b Mon Sep 17 00:00:00 2001 From: "community-scripts-pr-app[bot]" <189241966+community-scripts-pr-app[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:30:42 +0000 Subject: [PATCH 2/4] Update CHANGELOG.md (#18606) Co-authored-by: github-actions[bot] --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9ad0f341..cd9201ec1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -583,6 +583,12 @@ Exercise vigilance regarding copycat or coat-tailing sites that seek to exploit ## 2026-11-11 +### ๐Ÿš€ Updated Scripts + + - #### โœจ New Features + + - Immich: bump to 3.4.0 [@MickLesk](https://github.com/MickLesk) ([#18600](https://github.com/community-scripts/ProxmoxVE/pull/18600)) + ### ๐Ÿ’พ Core - feat: let setup_deb822_repo use a keyring already on the system [@MickLesk](https://github.com/MickLesk) ([core#227](https://github.com/community-scripts/core/pull/227)) From 01f37f501fcbcae8a330ce024c16da1d7638333d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B6gl-Brunner=20Michel?= Date: Wed, 11 Nov 2026 11:48:57 +0100 Subject: [PATCH 3/4] pr-script-test: build vm/ scripts, and every ct/ script on Incus too - vm/ scripts run unattended (VM_UNATTENDED=1) on the Proxmox hosts, IDs 9100-9199, one VM per host at a time. Checks: success line, the VM keeps running, a DHCP lease, and a ${VM_IP} URL answering 200. Console screenshot from QEMU when there is no web UI. ARM64-only scripts are listed as not built. - Every ct/ script now builds on pve and incus (new incus-test runners, incus-ci-host-setup.sh); the comment gets a Platform column. - Spinner frames are dropped from logs and live output. - The sweeper ignores and removes locks of killed jobs (older than 3h). --- .github/workflows/pr-script-test.yml | 110 ++++++---- .../workflows/scripts/incus-ci-host-setup.sh | 193 ++++++++++++++++++ .../workflows/scripts/pve-ci-host-setup.sh | 23 ++- .github/workflows/scripts/pve-run-script.sh | 157 ++++++++++---- .github/workflows/scripts/pve-run-vm.sh | 140 +++++++++++++ 5 files changed, 538 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/scripts/incus-ci-host-setup.sh create mode 100644 .github/workflows/scripts/pve-run-vm.sh diff --git a/.github/workflows/pr-script-test.yml b/.github/workflows/pr-script-test.yml index 10590d60c..ae8f51b5d 100644 --- a/.github/workflows/pr-script-test.yml +++ b/.github/workflows/pr-script-test.yml @@ -1,6 +1,6 @@ name: PR script test -# Builds the ct/ scripts a pull request touches on a real Proxmox host and +# Builds the ct/ and vm/ scripts a pull request touches on real Proxmox and Incus hosts and # reports the outcome on the PR, as a check run and as one comment kept up to # date. # @@ -16,7 +16,8 @@ name: PR script test # even one that lands in the seconds before this job runs. # # gate ubuntu-latest who asked, which scripts, which commit -# test proxmox-test build + checks, one matrix job per script/OS +# test proxmox-test / build + checks, one matrix job per script, platform +# incus-test (pve or incus) and OS # report ubuntu-latest check run + PR comment # # The test job gets a read-only token and never sees a secret: the host it runs @@ -46,7 +47,7 @@ on: required: false type: string scripts: - description: "Space-separated ct/ paths (empty = everything the PR touches)" + description: "Space-separated ct/ or vm/ paths (empty = everything the PR touches)" required: false type: string @@ -171,8 +172,9 @@ jobs: core.notice(why); }; if (isIssue) { - // App names (bark-server) or ct/ paths; the default branch is trusted. - entries = tokens.map(t => /^ct\/[A-Za-z0-9._-]+\.sh$/.test(t) ? t + // App names (bark-server) or ct/ and vm/ paths; the default branch + // is trusted. A name without a ct/ script falls back to vm/ below. + entries = tokens.map(t => /^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(t) ? t : /^[A-Za-z0-9._-]+$/.test(t) ? `ct/${t.replace(/\.sh$/, '')}.sh` : t); if (entries.length === 0) { await refuse('In an issue, `/test` needs to know which app.', 'Comment `/test `, for example `/test bark-server` (the name of its `ct/` script).'); @@ -202,10 +204,10 @@ jobs: return; } } - if (entries.some(e => !/^ct\/[A-Za-z0-9._-]+\.sh$/.test(e))) { + if (entries.some(e => !/^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(e))) { await refuse( - isIssue ? 'Only app names (as in `ct/.sh`) can be tested.' : 'Only `ct/.sh` paths can be requested.', - isIssue ? 'For example: `/test bark-server`' : 'For example: `/test ct/bark-server.sh`, or just `/test`.'); + isIssue ? 'Only app names (as in `ct/.sh` or `vm/.sh`) can be tested.' : 'Only `ct/.sh` and `vm/.sh` paths can be requested.', + isIssue ? 'For example: `/test bark-server` or `/test debian-vm`' : 'For example: `/test ct/bark-server.sh`, or just `/test`.'); return; } // Everything the PR changes that this test cannot build is named in @@ -219,29 +221,48 @@ jobs: if (f.status === 'removed') continue; let m; if ((m = f.filename.match(/^install\/([A-Za-z0-9._-]+)-install\.sh$/))) set.add(`ct/${m[1]}.sh`); - else if (/^ct\/[A-Za-z0-9._-]+\.sh$/.test(f.filename)) set.add(f.filename); - else if (/^vm\/.*\.sh$/.test(f.filename)) notBuilt.push(`${code(f.filename)}: \`vm/\` scripts are not covered`); - else if (/^(tools|turnkey)\/.*\.sh$/.test(f.filename)) notBuilt.push(`${code(f.filename)}: only \`ct/\` scripts are built`); + else if (/^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(f.filename)) set.add(f.filename); + else if (/^(tools|turnkey)\/.*\.sh$/.test(f.filename)) notBuilt.push(`${code(f.filename)}: only \`ct/\` and \`vm/\` scripts are built`); } entries = [...set].sort(); } const include = []; - for (const script of entries.slice(0, MAX)) { - let text; + const fetchScript = async p => { try { - const res = await github.rest.repos.getContent({ owner: head.owner.login, repo: head.name, path: script, ref: headSha }); - text = Buffer.from(res.data.content || '', 'base64').toString('utf8'); - } catch (e) { + const res = await github.rest.repos.getContent({ owner: head.owner.login, repo: head.name, path: p, ref: headSha }); + return Buffer.from(res.data.content || '', 'base64').toString('utf8'); + } catch (e) { return null; } + }; + for (let script of entries.slice(0, MAX)) { + let text = await fetchScript(script); + // /test debian-vm in an issue: there is no ct/debian-vm.sh, but a vm/ one. + if (text === null && isIssue && script.startsWith('ct/')) { + const vm = `vm/${script.slice(3)}`; + if ((text = await fetchScript(vm)) !== null) script = vm; + } + if (text === null) { notBuilt.push(isIssue - ? `\`${script}\` does not exist on the default branch` + ? (script.startsWith('ct/') ? `${code(script.slice(3, -3))}: neither \`ct/\` nor \`vm/\` has it on the default branch` : `${code(script)} does not exist on the default branch`) : `\`${script}\` does not exist in this PR (an \`install/\` script is built through the \`ct/\` script of the same name)`); continue; } + const isVm = script.startsWith('vm/'); + // The test hosts are amd64; an ARM-only VM script refuses to run there. + if (isVm && /^\s*vm_require_arch\s+arm64\b/m.test(text)) { + notBuilt.push(`${code(script)}: ARM64 only, the test hosts are amd64`); + continue; + } // Scripts that ask for the OS get built once per OS. - const oses = /msg_menu\s+"Choose the container OS"/.test(text) ? ['debian', 'alpine'] : ['']; - const name = script.slice(3, -3); - for (const os of oses) include.push({ script, os, name: os ? `${name}-${os}` : name }); + const oses = !isVm && /msg_menu\s+"Choose the container OS"/.test(text) ? ['debian', 'alpine'] : ['']; + const name = isVm ? `vm-${script.slice(3, -3)}` : script.slice(3, -3); + // Every ct/ script on every platform: the same files run on Proxmox + // and, through community-scripts/core's Incus backend, on Incus. + // VMs are Proxmox only. + const platforms = isVm ? ['pve'] : ['pve', 'incus']; + for (const platform of platforms) { + for (const os of oses) include.push({ script, os, platform, name: `${os ? `${name}-${os}` : name}-${platform}` }); + } } for (const script of entries.slice(MAX)) notBuilt.push(`\`${script}\`: at most ${MAX} scripts per run, name it with \`/test ${script}\``); if (include.length === 0) { @@ -249,10 +270,10 @@ jobs: MARK, '### Proxmox script test', '', isIssue ? 'โ„น๏ธ Nothing to build:' - : 'โ„น๏ธ Nothing to build: this test builds `ct/` scripts (and changed `install/` scripts through their `ct/` script).', '', - ...(notBuilt.length ? notBuilt.map(n => `- ${n}`) : ['- this PR changes no `ct/` or `install/` script']), + : 'โ„น๏ธ Nothing to build: this test builds `ct/` and `vm/` scripts (and changed `install/` scripts through their `ct/` script).', '', + ...(notBuilt.length ? notBuilt.map(n => `- ${n}`) : ['- this PR changes no `ct/`, `vm/` or `install/` script']), ].join('\n')); - core.notice('No ct/ script to build'); + core.notice('No ct/ or vm/ script to build'); return; } @@ -263,7 +284,7 @@ jobs: const { data: check } = await github.rest.checks.create({ owner, repo, name: 'Proxmox script test', head_sha: headSha, status: 'in_progress', details_url: runUrl, - output: { title: `Building ${include.length} on Proxmox`, summary: include.map(i => `- \`${i.script}\` ${i.os}`).join('\n') }, + output: { title: `Building ${include.length}`, summary: include.map(i => `- \`${i.script}\` ${[i.platform, i.os].filter(Boolean).join(' ยท ')}`).join('\n') }, }); checkId = String(check.id); } @@ -280,7 +301,7 @@ jobs: await sticky([ MARK, '### Proxmox script test', '', `โณ Building ${include.length} at ${headSha.slice(0, 7)}${isIssue ? ' (default branch)' : ''} โ€” [watch it live](${runUrl}) (open the \`test\` job)`, '', - ...include.map(i => `- \`${i.script}\`${i.os ? ` (${i.os})` : ''}`), + ...include.map(i => `- \`${i.script}\` (${[i.platform, i.os].filter(Boolean).join(' ยท ')})`), ...(notBuilt.length ? ['', '**Not built:**', ...notBuilt.map(n => `- ${n}`)] : []), ].join('\n')); @@ -289,7 +310,7 @@ jobs: if: needs.gate.outputs.matrix != '[]' && needs.gate.outputs.matrix != '' # Only `proxmox-test`, never `self-hosted`: the runner is registered without # default labels so that no other workflow can be scheduled onto it. - runs-on: proxmox-test + runs-on: ${{ matrix.platform == 'incus' && 'incus-test' || 'proxmox-test' }} permissions: contents: read timeout-minutes: 90 # install (45) + update (30) + checks @@ -311,6 +332,7 @@ jobs: PR_SHA: ${{ needs.gate.outputs.sha }} SCRIPT: ${{ matrix.script }} OS: ${{ matrix.os }} + PLATFORM: ${{ matrix.platform }} UPSTREAM: ${{ github.repository }} CTID_BASE: ${{ github.repository == 'community-scripts/ProxmoxVE' && '9050' || '9000' }} # Only so release lookups are not rate limited: unauthenticated, all @@ -390,7 +412,10 @@ jobs: r.update_log = fs.existsSync(ulog) ? fs.readFileSync(ulog, 'utf8') : ''; const ushot = path.join(dir, 'screenshot-update.png'); r.shot_update_file = fs.existsSync(ushot) ? ushot : ''; - found.set(`${r.script}|${r.os}`, r); + // Only the two platforms this workflow knows; an unknown one would + // never match a matrix row anyway. + if (!['pve', 'incus'].includes(r.platform)) r.platform = 'pve'; + found.set(`${r.script}|${r.platform}|${r.os}`, r); } catch (err) { core.warning(`unreadable ${p}: ${err.message}`); } } } @@ -398,12 +423,15 @@ jobs: walk('results'); // A job that died before writing result.json still gets a row. - const rows = matrix.map(m => ({ ...(found.get(`${m.script}|${m.os || 'default'}`) || { - script: m.script, os: m.os || 'default', + const rows = matrix.map(m => ({ ...(found.get(`${m.script}|${m.platform || 'pve'}|${m.os || 'default'}`) || { + script: m.script, os: m.os || 'default', platform: m.platform || 'pve', status: process.env.TEST_RESULT === 'cancelled' ? 'cancelled' : 'error', step: 'no result (runner or job failure)', duration: 0, log_tail: '', install_log: '', shot_file: '', update_status: '', update_log: '', shot_update_file: '', }), name: m.name })); + // Where a row ran: "pve", "incus ยท alpine", or "VM" (Proxmox only). + const isVm = r => r.script.startsWith('vm/'); + const where = r => isVm(r) ? 'VM' : r.os && r.os !== 'default' ? `${r.platform} ยท ${r.os}` : r.platform; // Screenshots go to an orphan branch, because a comment can only show // an image that has a URL. One file per PR and script, overwritten on @@ -434,7 +462,7 @@ jobs: catch (e) { if (e.status !== 404) throw e; } const { data } = await github.rest.repos.createOrUpdateFileContents({ owner, repo, branch: BRANCH, path: file, sha, content: buf.toString('base64'), - message: `Screenshot: ${r.script} (${r.os})${suffix} for #${prNum} at ${process.env.SHA.slice(0, 7)}`, + message: `Screenshot: ${r.script} (${where(r)})${suffix} for #${prNum} at ${process.env.SHA.slice(0, 7)}`, }); return `https://raw.githubusercontent.com/${owner}/${repo}/${data.commit.sha}/${file}`; } catch (e) { @@ -482,13 +510,15 @@ jobs: const updateIcon = { updated: 'โœ…', 'no-update': 'โž–', fail: 'โŒ', error: 'โš ๏ธ' }; const detail = r => { if (r.status !== 'pass') return clean(r.step); + const ip = /^[0-9.]{7,15}$/.test(r.ip || '') ? r.ip : ''; + if (isVm(r) && !r.url) return ip ? `booted, got ${ip}` : 'booted, no DHCP lease (an installer ISO, or no network)'; const reach = r.url ? `${clean(r.url)} โ†’ ${r.http_code}` : 'no URL to check'; return r.update_status === 'no-update' ? `${reach}; update: nothing to update${r.update_step ? ` (${clean(r.update_step)})` : ''}` : reach; }; const table = [ - '| Script | OS | Install | Update | Duration | Detail |', + '| Script | Platform | Install | Update | Duration | Detail |', '|---|---|---|---|---|---|', - ...rows.map(r => `| \`${r.script}\` | ${r.os} | ${r.update_status ? 'โœ…' : icon[r.status] || 'โ”'} | ${updateIcon[r.update_status] || 'โ€“'} | ${dur(r.duration || 0)} | ${detail(r)} |`), + ...rows.map(r => `| \`${r.script}\` | ${where(r)} | ${r.update_status ? 'โœ…' : icon[r.status] || 'โ”'} | ${updateIcon[r.update_status] || 'โ€“'} | ${dur(r.duration || 0)} | ${detail(r)} |`), ].join('\n'); // A comment holds 65536 characters. Each script gets an equal share @@ -510,25 +540,27 @@ jobs: const logs = rows.map(r => { const parts = []; const failed = r.status !== 'pass'; - const buildShare = failed && r.log_tail ? Math.min(r.log_tail.length, Math.floor(budget / 3)) : 0; - if (buildShare) parts.push(details(`${icon[r.status] || 'โ”'} ${r.script} (${r.os}) โ€” build log`, r.log_tail, buildShare)); + // A VM script's own output is all there is: no install log inside. + const buildShare = (failed || isVm(r)) && r.log_tail ? Math.min(r.log_tail.length, Math.floor(budget / 3)) : 0; + if (buildShare) parts.push(details(`${icon[r.status] || 'โ”'} ${r.script} (${where(r)}) โ€” build log`, r.log_tail, buildShare)); + if (isVm(r)) return parts.join('\n\n'); // The update log gets up to a third of what is left, the install log the rest. const updateShare = r.update_log ? Math.min(r.update_log.length, Math.floor((budget - buildShare) / 3)) : 0; if (r.install_log) { const lines = r.install_log.trimEnd().split('\n').length; - parts.push(details(`๐Ÿ“„ ${r.script} (${r.os}) โ€” install log (${lines} lines)`, r.install_log, budget - buildShare - updateShare - 400)); + parts.push(details(`๐Ÿ“„ ${r.script} (${where(r)}) โ€” install log (${lines} lines)`, r.install_log, budget - buildShare - updateShare - 400)); } else if (!buildShare) { - parts.push(`${r.script} (${r.os}): no install log was captured`); + parts.push(`${r.script} (${where(r)}): no install log was captured`); } if (updateShare) { const lines = r.update_log.trimEnd().split('\n').length; - parts.push(details(`๐Ÿ”„ ${r.script} (${r.os}) โ€” update log (${lines} lines)`, r.update_log, updateShare)); + parts.push(details(`๐Ÿ”„ ${r.script} (${where(r)}) โ€” update log (${lines} lines)`, r.update_log, updateShare)); } return parts.join('\n\n'); }); const shots = rows.filter(r => r.shot_url || r.shot_update_url).map(r => [ - `
๐Ÿ“ธ ${r.script} (${r.os}) โ€” ${clean(r.url)}${r.shot_url && r.shot_update_url ? ': after install / after update' : r.shot_update_url ? ': after update' : ''}`, '', + `
๐Ÿ“ธ ${r.script} (${where(r)}) โ€” ${r.url ? clean(r.url) : 'console'}${r.shot_url && r.shot_update_url ? ': after install / after update' : r.shot_update_url ? ': after update' : ''}`, '', [r.shot_url, r.shot_update_url].filter(Boolean) .map((u, i, all) => `Screenshot of ${r.script}`).join(' '), '', '
', @@ -638,7 +670,7 @@ jobs: owner, repo, pull_number: pr, commit_id: process.env.SHA, event: 'REQUEST_CHANGES', body: [ REVIEW_MARK, `**Proxmox script test failed** at \`${sha7}\`:`, '', - ...failedRows.map(r => `- \`${r.script}\` (${r.os}): ${clean(r.step)}`), '', + ...failedRows.map(r => `- \`${r.script}\` (${where(r)}): ${clean(r.step)}`), '', `Logs and details: [test result](${posted.html_url}). Push a fix, then ask for \`/test\` again.`, ].join('\n'), }); diff --git a/.github/workflows/scripts/incus-ci-host-setup.sh b/.github/workflows/scripts/incus-ci-host-setup.sh new file mode 100644 index 000000000..516e0a309 --- /dev/null +++ b/.github/workflows/scripts/incus-ci-host-setup.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# One-time setup of an Incus test host behind pr-script-test.yml: the same +# ct/ scripts as on Proxmox, built through community-scripts/core's Incus +# backend. Safe to re-run. Run as root on a disposable Debian 13 machine (it +# builds pull request code as root). +# +# incus-ci-host-setup.sh host only +# incus-ci-host-setup.sh host + a runner for that repo +# +# The token is a runner *registration* token (valid one hour): +# gh api -X POST repos//actions/runners/registration-token --jq .token + +set -euo pipefail + +RUNNER_VERSION="${RUNNER_VERSION:-2.338.2}" +SERVE_ROOT=/srv/ci +SERVE_PORT=18080 +NET_ADDR=10.98.0.1 +NET_CIDR=10.98.0.0/24 + +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq incus jq curl python3 iptables chromium fonts-dejavu fonts-noto-color-emoji >/dev/null + +# Containers get a NATed bridge of their own (Incus runs DHCP and DNS on it) +# and a plain directory pool on the root disk. +if ! incus storage show default &>/dev/null; then + incus admin init --preseed </dev/null || useradd --system --create-home --shell /usr/sbin/nologin ci-shot + +# Same rules as on the Proxmox hosts: the screenshot browser may reach the +# test containers, DNS and the public internet, nothing private, and not this +# host's own address on the container network (the PR tree server). +cat >/usr/local/sbin/ci-shot-firewall </dev/null || \$fw -w -F CI_SHOT + \$fw -w -C OUTPUT -m owner --uid-owner "\$uid" -j CI_SHOT 2>/dev/null || + \$fw -w -I OUTPUT -m owner --uid-owner "\$uid" -j CI_SHOT +done +iptables -w -A CI_SHOT -d ${NET_ADDR} -j REJECT +iptables -w -A CI_SHOT -d ${NET_CIDR} -j ACCEPT +# systemd-resolved's stub; it forwards to the lab resolver itself. +iptables -w -A CI_SHOT -d 127.0.0.53 -p udp --dport 53 -j ACCEPT +iptables -w -A CI_SHOT -d 127.0.0.53 -p tcp --dport 53 -j ACCEPT +for ns in \$(awk '/^nameserver/ && \$2 !~ /:/ {print \$2}' /etc/resolv.conf /run/systemd/resolve/resolv.conf 2>/dev/null); do + iptables -w -A CI_SHOT -d "\$ns" -p udp --dport 53 -j ACCEPT + iptables -w -A CI_SHOT -d "\$ns" -p tcp --dport 53 -j ACCEPT +done +for net in 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.168.0.0/16 224.0.0.0/3; do + iptables -w -A CI_SHOT -d "\$net" -j REJECT +done +for net in ::1/128 fc00::/7 fe80::/10 ff00::/8; do + ip6tables -w -A CI_SHOT -d "\$net" -j REJECT +done +EOF +chmod +x /usr/local/sbin/ci-shot-firewall +cat >/etc/systemd/system/ci-shot-firewall.service <<'EOF' +[Unit] +Description=Keep the screenshot browser away from this host and the lab network +After=network-online.target incus.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/ci-shot-firewall + +[Install] +WantedBy=multi-user.target +EOF + +[[ -s /etc/timezone ]] || { + timedatectl set-timezone Europe/Vienna + echo Europe/Vienna >/etc/timezone +} + +# The rewritten PR trees, served to the engine and to the containers. Waits +# for incusbr0, which only exists once Incus is up. +mkdir -p "$SERVE_ROOT" +cat >/etc/systemd/system/ci-serve.service </dev/null 2>&1; sleep 1; done; exit 1' +ExecStart=/usr/bin/python3 -m http.server ${SERVE_PORT} --bind ${NET_ADDR} --directory ${SERVE_ROOT} +DynamicUser=yes +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +EOF + +# Leftovers of a job that was killed before its cleanup trap ran. +cat >/usr/local/sbin/ci-sweep <<'EOF' +#!/usr/bin/env bash +# Delete CI containers (ci-9000 .. ci-9099) that no running job holds, and stale trees. +# A job holds its guest with /run/ci-jobs/. A job killed outright (runner +# restart, timeout) never removes it, so a lock older than any job can run +# (90 minutes) holds nothing. +held() { [[ -e "/run/ci-jobs/$1" && -z "$(find "/run/ci-jobs/$1" -mmin +180)" ]]; } +for name in $(incus list -c n -f csv 2>/dev/null | grep -E '^ci-9[0-9]{3}$'); do + held "$name" && continue + incus delete "$name" --force &>/dev/null && echo "swept $name" +done +find /run/ci-jobs -maxdepth 1 -type f ! -name ".*" -mmin +180 -delete 2>/dev/null +find /srv/ci -mindepth 1 -maxdepth 1 -type d -mmin +180 -exec rm -rf {} + +find /usr/local/community-scripts/logs /var/log/community-scripts -type f -mtime +2 -delete 2>/dev/null +find /opt/actions-runner/*/_diag -type f -mtime +14 -delete 2>/dev/null +true +EOF +chmod +x /usr/local/sbin/ci-sweep +cat >/etc/systemd/system/ci-sweep.service <<'EOF' +[Unit] +Description=Remove leftover CI containers + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ci-sweep +EOF +cat >/etc/systemd/system/ci-sweep.timer <<'EOF' +[Unit] +Description=Remove leftover CI containers every 3 hours + +[Timer] +OnCalendar=0/3:30 +Persistent=true + +[Install] +WantedBy=timers.target +EOF + +systemctl daemon-reload +systemctl enable --now ci-serve.service ci-sweep.timer ci-shot-firewall.service >/dev/null +systemctl restart ci-serve.service ci-shot-firewall.service + +# --- Runner (one per repository) ---------------------------------------------- +if [[ $# -eq 2 ]]; then + repo="$1" + token="$2" + [[ "$repo" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad repo" >&2; exit 2; } + dir="/opt/actions-runner/${repo#*/}" + mkdir -p "$dir" + cd "$dir" + if [[ ! -x ./config.sh ]]; then + curl -fsSL "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" | tar -xz + ./bin/installdependencies.sh >/dev/null + fi + if [[ -f .runner ]]; then + ./svc.sh stop >/dev/null 2>&1 || true + ./svc.sh uninstall >/dev/null 2>&1 || true + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token "$token" >/dev/null 2>&1 || true + fi + # Only `incus-test`: see pve-ci-host-setup.sh for why there are no default labels. + RUNNER_ALLOW_RUNASROOT=1 ./config.sh --unattended --replace --no-default-labels \ + --url "https://github.com/${repo}" --token "$token" \ + --name "$(hostname -s)-${repo#*/}" --labels incus-test --work _work + ./svc.sh install root >/dev/null + ./svc.sh start >/dev/null + echo "runner for ${repo} registered" +fi + +echo "incus test host ready: ${SERVE_ROOT} on ${NET_ADDR}:${SERVE_PORT}, sweeper every 3h" diff --git a/.github/workflows/scripts/pve-ci-host-setup.sh b/.github/workflows/scripts/pve-ci-host-setup.sh index 02193f784..43e7f4d2e 100644 --- a/.github/workflows/scripts/pve-ci-host-setup.sh +++ b/.github/workflows/scripts/pve-ci-host-setup.sh @@ -102,8 +102,9 @@ systemctl restart dnsmasq echo Europe/Vienna >/etc/timezone } -# Containers live on the root filesystem; there is no LVM in a cloud-image PVE. -pvesm set local --content iso,vztmpl,backup,rootdir,images +# Containers and VM disks live on the root filesystem; there is no LVM in a +# cloud-image PVE. Snippets hold the Cloud-Init configs some vm/ scripts write. +pvesm set local --content iso,vztmpl,backup,rootdir,images,snippets # Templates are fetched once here rather than on every first build. pveam update >/dev/null @@ -130,12 +131,26 @@ EOF # Leftovers of a job that was killed before its cleanup trap ran. cat >/usr/local/sbin/ci-sweep <<'EOF' #!/usr/bin/env bash -# Destroy CI containers (9000-9099) that no running job holds, and stale trees. +# Destroy CI containers (9000-9099) and VMs (9100-9199) that no running job +# holds, and stale trees. +# A job holds its guest with /run/ci-jobs/. A job killed outright (runner +# restart, timeout) never removes it, so a lock older than any job can run +# (90 minutes) holds nothing. +held() { [[ -e "/run/ci-jobs/$1" && -z "$(find "/run/ci-jobs/$1" -mmin +180)" ]]; } for id in $(pct list | awk 'NR>1 && $1>=9000 && $1<=9099 {print $1}'); do - [[ -e "/run/ci-jobs/$id" ]] && continue + held "$id" && continue pct stop "$id" --skiplock 1 &>/dev/null || true pct destroy "$id" --purge 1 --force 1 &>/dev/null && echo "swept CT $id" done +for id in $(qm list | awk 'NR>1 && $1>=9100 && $1<=9199 {print $1}'); do + held "$id" && continue + qm stop "$id" --skiplock 1 --timeout 30 &>/dev/null || true + qm destroy "$id" --purge 1 --destroy-unreferenced-disks 1 --skiplock 1 &>/dev/null && echo "swept VM $id" +done +# VM images and ISOs are cached between builds; drop the ones no build used +# for a week. Container templates (*.tar.*) stay. +find /var/lib/vz/template/cache /var/lib/vz/template/iso -maxdepth 1 -type f ! -name '*.tar.*' -atime +7 -delete 2>/dev/null +find /run/ci-jobs -maxdepth 1 -type f ! -name ".*" -mmin +180 -delete 2>/dev/null find /srv/ci -mindepth 1 -maxdepth 1 -type d -mmin +180 -exec rm -rf {} + # Engine logs that a killed job never got to remove. find /tmp -maxdepth 1 -type f \( -name 'create-lxc-*.log' -o -name 'pct_create_*.log' \ diff --git a/.github/workflows/scripts/pve-run-script.sh b/.github/workflows/scripts/pve-run-script.sh index e34cc426c..062f57b88 100644 --- a/.github/workflows/scripts/pve-run-script.sh +++ b/.github/workflows/scripts/pve-run-script.sh @@ -1,6 +1,12 @@ #!/usr/bin/env bash -# Build one ct/ script from a pull request on this Proxmox host, check that the -# container really works, destroy it, and leave a result.json behind. +# Build one ct/ or vm/ script from a pull request on this test host, check +# that the container or VM really works, destroy it, and leave a result.json +# behind. The vm/ part lives in pve-run-vm.sh, sourced from here. +# +# PLATFORM says what this host is: pve (Proxmox VE, pct) or incus. The ct/ +# scripts are the same for both; community-scripts/core picks its Incus +# backend by itself when it finds incus instead of pveversion. Only the way +# this harness reaches into the container differs (guest_* below). # # Runs on the self-hosted runner, which *is* the disposable test host. The # harness comes from the default branch; only the script under test comes from @@ -17,10 +23,17 @@ set -uo pipefail : "${PR_REPO:?}" "${PR_SHA:?}" "${SCRIPT:?}" "${UPSTREAM:?}" "${CTID_BASE:?}" "${OUT_DIR:?}" OS="${OS:-}" +PLATFORM="${PLATFORM:-pve}" CTID_COUNT="${CTID_COUNT:-50}" SERVE_ROOT="${SERVE_ROOT:-/srv/ci}" -SERVE_URL="${SERVE_URL:-http://10.99.0.1:18080}" -BRIDGE="${BRIDGE:-vmbr1}" +# The containers' own network: vmbr1 + dnsmasq on Proxmox, incusbr0 on Incus. +if [[ "$PLATFORM" == "incus" ]]; then + SERVE_URL="${SERVE_URL:-http://10.98.0.1:18080}" + BRIDGE="${BRIDGE:-incusbr0}" +else + SERVE_URL="${SERVE_URL:-http://10.99.0.1:18080}" + BRIDGE="${BRIDGE:-vmbr1}" +fi BUILD_TIMEOUT="${BUILD_TIMEOUT:-45m}" URL_WAIT="${URL_WAIT:-120}" @@ -28,8 +41,10 @@ URL_WAIT="${URL_WAIT:-120}" [[ "$PR_REPO" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad PR_REPO" >&2; exit 2; } [[ "$UPSTREAM" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad UPSTREAM" >&2; exit 2; } [[ "$PR_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "bad PR_SHA" >&2; exit 2; } -[[ "$SCRIPT" =~ ^ct/[A-Za-z0-9._-]+\.sh$ ]] || { echo "bad SCRIPT" >&2; exit 2; } +[[ "$SCRIPT" =~ ^(ct|vm)/[A-Za-z0-9._-]+\.sh$ ]] || { echo "bad SCRIPT" >&2; exit 2; } [[ -z "$OS" || "$OS" =~ ^(debian|ubuntu|alpine)$ ]] || { echo "bad OS" >&2; exit 2; } +[[ "$PLATFORM" =~ ^(pve|incus)$ ]] || { echo "bad PLATFORM" >&2; exit 2; } +[[ "$PLATFORM" == "pve" || "$SCRIPT" == ct/* ]] || { echo "vm/ scripts run on pve only" >&2; exit 2; } [[ "$CTID_BASE" =~ ^[0-9]+$ ]] || { echo "bad CTID_BASE" >&2; exit 2; } mkdir -p "$OUT_DIR" @@ -41,8 +56,9 @@ UPDATE_RAW="$OUT_DIR/update.raw.log" UPDATE_LOG="$OUT_DIR/update.log" UPDATE_TIMEOUT="${UPDATE_TIMEOUT:-30m}" +kind="${SCRIPT%%/*}" # ct or vm app=$(basename "$SCRIPT" .sh) -job="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${app}-${OS:-default}" +job="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${PLATFORM}-${kind}-${app}-${OS:-default}" tree="$SERVE_ROOT/$job" base="$SERVE_URL/$job" started=$(date +%s) @@ -50,7 +66,30 @@ started=$(date +%s) status="fail" step="" exit_code="" -ctid="" +ctid="" # the guest's ID (pve: 9000, incus: ci-9000), container or VM + +# The container, whatever the platform. ctid is set once a slot is reserved. +guest_exec() { + if [[ "$PLATFORM" == "incus" ]]; then incus exec "$ctid" -- "$@"; else pct exec "$ctid" -- "$@"; fi +} +guest_exists() { + if [[ "$PLATFORM" == "incus" ]]; then incus info "$ctid" &>/dev/null; else pct status "$ctid" &>/dev/null; fi +} +guest_running() { + if [[ "$PLATFORM" == "incus" ]]; then + [[ "$(incus list "^${ctid}\$" -c s -f csv 2>/dev/null)" == "RUNNING" ]] + else + pct status "$ctid" 2>/dev/null | grep -q running + fi +} +guest_destroy() { + if [[ "$PLATFORM" == "incus" ]]; then + incus delete "$ctid" --force &>/dev/null || true + else + pct stop "$ctid" --skiplock 1 &>/dev/null || true + pct destroy "$ctid" --purge 1 --force 1 &>/dev/null || true + fi +} ct_ip="" url="" http_code="" @@ -60,8 +99,13 @@ failed_units="" update_status="" update_step="" +# Spinner frames: the engine redraws "โ ‹ Starting VM" a few times a second +# without a tty too. An alternation, not a bracket: these are multibyte and +# the runner's locale may be C. +SPINNER_RE='^\s*(โ ‹|โ ™|โ น|โ ธ|โ ผ|โ ด|โ ฆ|โ ง|โ ‡|โ )' + strip_ansi() { - sed -E 's/\x1B\[[0-9;?]*[A-Za-z]//g; s/\x1B[()][A-Z0-9]//g; s/\r/\n/g' | grep -vE '^\s*$' + sed -E 's/\x1B\[[0-9;?]*[A-Za-z]//g; s/\x1B[()][A-Z0-9]//g; s/\r/\n/g' | grep -vE '^\s*$' | grep -vE "$SPINNER_RE" } # The install script's own output: everything $STD hides from the build log. @@ -71,11 +115,12 @@ strip_ansi() { collect_install_log() { [[ -n "$ctid" ]] || return 0 # Exactly one build ran in this container, so the newest install log is ours. - if pct status "$ctid" &>/dev/null && - pct exec "$ctid" -- sh -c 'cat "$(ls -t /root/.install-*.log 2>/dev/null | head -n1)"' 2>/dev/null | + if guest_exists && + guest_exec sh -c 'cat "$(ls -t /root/.install-*.log 2>/dev/null | head -n1)"' 2>/dev/null | strip_ansi >"$INSTALL_LOG_OUT" && [[ -s "$INSTALL_LOG_OUT" ]]; then return 0 fi + [[ "$PLATFORM" == "pve" ]] || { rm -f "$INSTALL_LOG_OUT"; return 0; } # The engine names its host-side files after the session: pct_create___.log local sid f sid=$(find /tmp -maxdepth 1 -name "pct_create_${ctid}_*.log" -newer "$RAW_LOG.start" -printf '%f\n' 2>/dev/null | @@ -120,15 +165,15 @@ finish() { [[ -f "$UPDATE_RAW" ]] && strip_ansi <"$UPDATE_RAW" >"$UPDATE_LOG" rm -f "$UPDATE_RAW" jq -n \ - --arg script "$SCRIPT" --arg os "${OS:-default}" --arg status "$status" \ + --arg script "$SCRIPT" --arg kind "$kind" --arg platform "$PLATFORM" --arg os "${OS:-default}" --arg status "$status" \ --arg step "$step" --arg exit_code "$exit_code" --arg sha "$PR_SHA" \ --arg ctid "$ctid" --arg url "$url" --arg http_code "$http_code" \ - --arg failed_units "$failed_units" \ + --arg failed_units "$failed_units" --arg ip "$ct_ip" \ --arg update_status "$update_status" --arg update_step "$update_step" \ --argjson duration "$(($(date +%s) - started))" \ --arg tail "$([[ -f "$LOG" ]] && tail -n 60 "$LOG")" \ - '{script: $script, os: $os, status: $status, step: $step, exit_code: $exit_code, - sha: $sha, ctid: $ctid, url: $url, http_code: $http_code, + '{script: $script, kind: $kind, platform: $platform, os: $os, status: $status, step: $step, exit_code: $exit_code, + sha: $sha, ctid: $ctid, ip: $ip, url: $url, http_code: $http_code, failed_units: $failed_units, update_status: $update_status, update_step: $update_step, duration: $duration, log_tail: $tail}' >"$RESULT" echo "::group::result" @@ -150,7 +195,7 @@ install_pid="" # Colours stay, cursor movement and spinner redraws go. live_filter() { - sed -u -E 's/\r/\n/g; s/\x1B\[[0-9;?]*[ABCDEFGHJKSTfhilnsu]//g' | sed -u -E '/^\s*$/d' + sed -u -E 's/\r/\n/g; s/\x1B\[[0-9;?]*[ABCDEFGHJKSTfhilnsu]//g' | sed -u -E "/^\s*\$/d; /${SPINNER_RE}/d" } start_streams() { @@ -158,17 +203,21 @@ start_streams() { echo "::stop-commands::${stream_token}" tail -n +1 -F --pid="$build_pid" "$RAW_LOG" 2>/dev/null > >(live_filter) & log_pid=$! + # A VM has no install log to follow: its script runs on this host only. + [[ "$kind" == "ct" ]] || return 0 # setsid: its own process group, so stop_streams can take down the - # pct exec / tail pair inside it in one go. + # pct/incus exec + tail pair inside it in one go. setsid bash -c ' + gx() { if [ "$2" = incus ]; then incus exec "$1" -- "${@:3}"; else pct exec "$1" -- "${@:3}"; fi; } for _ in $(seq 1 900); do - f=$(pct exec "$1" -- sh -c "ls -t /root/.install-*.log 2>/dev/null | head -n1" 2>/dev/null) + f=$(gx "$1" "$2" sh -c "ls -t /root/.install-*.log 2>/dev/null | head -n1" 2>/dev/null) if [ -n "$f" ]; then echo "โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ install log of CT $1 (live) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" + if [ "$2" = incus ]; then exec incus exec "$1" -- tail -n +1 -F "$f"; fi exec pct exec "$1" -- tail -n +1 -F "$f" fi sleep 2 - done' _ "$ctid" 2>/dev/null > >(live_filter | sed -u 's/^/ โ”‚ /') & + done' _ "$ctid" "$PLATFORM" 2>/dev/null > >(live_filter | sed -u 's/^/ โ”‚ /') & install_pid=$! } @@ -195,15 +244,14 @@ cleanup() { kill -TERM "$pid" &>/dev/null || true sleep 2 done - if [[ -n "$ctid" ]]; then - if pct status "$ctid" &>/dev/null; then - pct stop "$ctid" --skiplock 1 &>/dev/null || true - pct destroy "$ctid" --purge 1 --force 1 &>/dev/null || true - fi + if [[ -n "$ctid" && "$kind" == "vm" ]]; then + vm_cleanup + elif [[ -n "$ctid" ]]; then + guest_exists && guest_destroy rm -f "/run/ci-jobs/$ctid" # What the engine leaves on the host for this container and its session. local f sid - for f in /tmp/pct_create_"${ctid}"_*.log; do + [[ "$PLATFORM" == "pve" ]] && for f in /tmp/pct_create_"${ctid}"_*.log; do [[ -f "$f" && "$f" -nt "$RAW_LOG.start" ]] || continue sid=$(sed -E 's/.*_([0-9a-f]+)\.log$/\1/' <<<"$f") rm -f -- "$f" /tmp/create-lxc-"${sid}".log /tmp/*-"${ctid}"-"${sid}".log @@ -245,19 +293,33 @@ setup_error() { # --- Reserve a container ID --------------------------------------------------- # A lock file per ID, not just "first free one", because the engine only creates # the container minutes later and two jobs would otherwise pick the same ID. +# VMs take the range 100 above the containers' (DevScripts 9100-9149, +# ProxmoxVE 9150-9199). mkdir -p /run/ci-jobs +if [[ "$kind" == "vm" ]]; then + source "$(dirname "$0")/pve-run-vm.sh" + CTID_BASE=$((CTID_BASE + 100)) + vm_wait_turn +fi exec 9>/run/ci-jobs/.lock flock 9 +# On Incus the ID is an instance name: var_ctid=ci-9000 names the container. for ((id = CTID_BASE; id < CTID_BASE + CTID_COUNT; id++)); do - [[ -e "/run/ci-jobs/$id" ]] && continue - pvesh get /cluster/nextid --vmid "$id" &>/dev/null || continue - echo "$job" >"/run/ci-jobs/$id" - ctid=$id + name=$id + [[ "$PLATFORM" == "incus" ]] && name="ci-$id" + [[ -e "/run/ci-jobs/$name" ]] && continue + if [[ "$PLATFORM" == "incus" ]]; then + incus info "$name" &>/dev/null && continue + else + pvesh get /cluster/nextid --vmid "$id" &>/dev/null || continue + fi + echo "$job" >"/run/ci-jobs/$name" + ctid=$name break done flock -u 9 exec 9>&- -[[ -n "$ctid" ]] || setup_error "no free container ID in ${CTID_BASE}-$((CTID_BASE + CTID_COUNT - 1))" +[[ -n "$ctid" ]] || setup_error "no free ${kind} ID in ${CTID_BASE}-$((CTID_BASE + CTID_COUNT - 1))" # --- Fetch the approved commit and point it at ourselves ------------------------ mkdir -p "$tree" @@ -269,17 +331,26 @@ upstream_re="https://raw\.githubusercontent\.com/${UPSTREAM//./\\.}/(refs/heads/ grep -rlE "$upstream_re" "$tree" --include='*.sh' --include='*.func' | xargs -r sed -i -E "s#${upstream_re}#${base}#g" +if [[ "$kind" == "vm" ]]; then + vm_build_and_check + exit 0 +fi + # --- Build ---------------------------------------------------------------------- -echo "Building ${SCRIPT} (${OS:-default OS}) as CT ${ctid} from ${PR_REPO}@${PR_SHA:0:7}" +echo "Building ${SCRIPT} (${OS:-default OS}) on ${PLATFORM} as CT ${ctid} from ${PR_REPO}@${PR_SHA:0:7}" hostname_safe="ci-$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}" | cut -c1-50)" build_env=( PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root TERM=dumb LANG=C.UTF-8 mode=default MODE=default PHS_SILENT=1 DIAGNOSTICS=no - var_ctid="$ctid" var_hostname="$hostname_safe" - var_container_storage=local var_template_storage=local var_brg="$BRIDGE" + var_ctid="$ctid" var_hostname="$hostname_safe" var_brg="$BRIDGE" "dev_mode=timing,keep" COMMUNITY_SCRIPTS_URL="$base" ) +if [[ "$PLATFORM" == "incus" ]]; then + build_env+=(var_container_storage=default) +else + build_env+=(var_container_storage=local var_template_storage=local) +fi touch "$RAW_LOG.start" [[ -n "$OS" ]] && build_env+=(var_os="$OS") # Both engines turn var_github_token into GITHUB_TOKEN inside the container. @@ -414,9 +485,9 @@ fi grep -qi 'completed successfully' "$RAW_LOG" || fail "build exited 0 but never reported success" # --- Checks --------------------------------------------------------------------- -pct status "$ctid" 2>/dev/null | grep -q running || fail "container ${ctid} is not running" +guest_running || fail "container ${ctid} is not running" -ct_ip=$(pct exec "$ctid" -- sh -c "ip -4 -o addr show dev eth0 | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null | head -n1) +ct_ip=$(guest_exec sh -c "ip -4 -o addr show dev eth0 | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null | head -n1) # What the app's page looks like, for the PR comment. Chromium runs as its own # unprivileged user with its sandbox on: the page is served by code under test. @@ -450,11 +521,11 @@ health_problem="" check_health() { health_problem="" failed_units="" - if pct exec "$ctid" -- sh -c 'command -v systemctl' &>/dev/null; then + if guest_exec sh -c 'command -v systemctl' &>/dev/null; then sleep 10 - failed_units=$(pct exec "$ctid" -- systemctl --failed --no-legend --plain 2>/dev/null | awk '{print $1}' | paste -sd, -) - elif pct exec "$ctid" -- sh -c 'command -v rc-status' &>/dev/null; then - failed_units=$(pct exec "$ctid" -- rc-status --crashed 2>/dev/null | awk '{print $1}' | paste -sd, -) + failed_units=$(guest_exec systemctl --failed --no-legend --plain 2>/dev/null | awk '{print $1}' | paste -sd, -) + elif guest_exec sh -c 'command -v rc-status' &>/dev/null; then + failed_units=$(guest_exec rc-status --crashed 2>/dev/null | awk '{print $1}' | paste -sd, -) fi [[ -z "$failed_units" ]] || { health_problem="failed services: ${failed_units}"; return 1; } [[ -n "$url" ]] || return 0 @@ -486,14 +557,14 @@ update_fail() { fail "update: $1" } -version_files=$(pct exec "$ctid" -- sh -c ' +version_files=$(guest_exec sh -c ' for f in /root/.*; do [ -f "$f" ] || continue case "$f" in */.install-* | *.pending | *.bak | */.bash* | */.profile | */.hushlogin | */.viminfo | */.lesshst | */.wget-hsts | */.selected_editor) continue ;; esac [ "$(wc -l <"$f")" -le 1 ] && grep -qE "^v?[0-9]+(\.[0-9A-Za-z_+-]+)+$" "$f" && echo "$f" done' 2>/dev/null | grep -E '^/root/\.[A-Za-z0-9._-]+$') for f in $version_files; do - pct exec "$ctid" -- sh -c "echo '$UPDATE_FAKE' > '$f'; rm -f '$f.pending'" || + guest_exec sh -c "echo '$UPDATE_FAKE' > '$f'; rm -f '$f.pending'" || update_fail "could not reset $f" done [[ -n "$version_files" ]] && echo "Update test: set back $(tr '\n' ' ' <<<"$version_files")to ${UPDATE_FAKE}" @@ -502,7 +573,9 @@ done update_token=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n') echo "โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ update test (live) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" echo "::stop-commands::${update_token}" -timeout --kill-after=1m "$UPDATE_TIMEOUT" pct exec "$ctid" -- env PHS_SILENT=1 mode=default MODE=default \ +# timeout runs a program, not a shell function, so spell out the exec here. +if [[ "$PLATFORM" == "incus" ]]; then guest_cmd=(incus exec "$ctid" --); else guest_cmd=(pct exec "$ctid" --); fi +timeout --kill-after=1m "$UPDATE_TIMEOUT" "${guest_cmd[@]}" env PHS_SILENT=1 mode=default MODE=default \ TERM=dumb DIAGNOSTICS=no var_ignore_breaking_changes=yes ${GH_API_TOKEN:+GITHUB_TOKEN="$GH_API_TOKEN"} update >(tee "$UPDATE_RAW" | live_filter) 2>&1 & update_pid=$! @@ -538,7 +611,7 @@ fi if strip_ansi <"$UPDATE_RAW" | grep -qi 'updated successfully'; then update_status="updated" for f in $version_files; do - [[ "$(pct exec "$ctid" -- cat "$f" 2>/dev/null)" != "$UPDATE_FAKE" ]] || + [[ "$(guest_exec cat "$f" 2>/dev/null)" != "$UPDATE_FAKE" ]] || update_fail "reported success, but $f is still ${UPDATE_FAKE}" done else diff --git a/.github/workflows/scripts/pve-run-vm.sh b/.github/workflows/scripts/pve-run-vm.sh new file mode 100644 index 000000000..5492566de --- /dev/null +++ b/.github/workflows/scripts/pve-run-vm.sh @@ -0,0 +1,140 @@ +# shellcheck shell=bash +# The vm/ half of pve-run-script.sh, sourced from it: same inputs, same +# result.json, same live output. Only building and checking differ. +# +# A VM script runs entirely on this host (there is no install script inside +# the guest to follow), unattended through the core engine's VM_UNATTENDED=1: +# the script's defaults, with ID, name, bridge and storage set from here. The +# guest then boots on vmbr1 like the containers do, so its address is whatever +# this host's dnsmasq leased to its MAC; no guest agent needed. +# +# What passes: the script exits 0 and says "Completed successfully", the VM is +# running and keeps running through the boot wait, and where the script prints +# a URL built from the VM's address (http://${VM_IP}:9000) that URL answers 200. +# An installer ISO or an image without DHCP gets no address; that is reported, +# not failed. There is no update test: a VM has no `update`. + +VM_BOOT_WAIT="${VM_BOOT_WAIT:-300}" +VM_URL_WAIT="${VM_URL_WAIT:-600}" +LEASES="${LEASES:-/var/lib/misc/dnsmasq.leases}" + +# One VM build per host at a time: a VM takes 2-8 GB of RAM and a disk image, +# where a container takes a fraction. Held until the job exits; the build is +# started with this descriptor closed, so a QEMU process can never inherit it. +vm_wait_turn() { + exec 7>/run/ci-jobs/.vm-turn + flock -n 7 && return 0 + echo "Another VM build is running on this host; waiting for it (one VM at a time)" + flock -w 3600 7 || setup_error "waited an hour for this host's VM slot" +} + +vm_cleanup() { + if qm status "$ctid" &>/dev/null; then + qm stop "$ctid" --skiplock 1 --timeout 30 &>/dev/null || true + qm destroy "$ctid" --purge 1 --destroy-unreferenced-disks 1 --skiplock 1 &>/dev/null || true + fi + rm -f "/run/ci-jobs/$ctid" "/run/ci-jobs/console-${ctid}.png" + # Cloud-Init network snippets the script wrote for this VM. + [[ -f "$RAW_LOG.start" ]] && + find /var/lib/vz/snippets -maxdepth 1 -type f -name "*${ctid}*" -newer "$RAW_LOG.start" -delete 2>/dev/null + true +} + +# The VM's screen, straight from QEMU. Pixels only, but they come from the +# guest, so the same checks as for a browser screenshot apply. +vm_console_shot() { + local f="/run/ci-jobs/console-${ctid}.png" + rm -f "$f" + printf 'screendump %s -f png\n' "$f" | timeout 30 qm monitor "$ctid" &>/dev/null + if [[ -s "$f" && $(stat -c %s "$f") -lt 3000000 ]] && + [[ "$(head -c 8 "$f" | od -An -tx1 | tr -d ' \n')" == "89504e470d0a1a0a" ]]; then + cp "$f" "$OUT_DIR/screenshot.png" + fi + rm -f "$f" +} + +vm_running() { + qm status "$ctid" 2>/dev/null | grep -q 'status: running' +} + +vm_build_and_check() { + local hn mac pattern waited last + hn="ci-$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}" | cut -c1-40)-${ctid}" + echo "Building ${SCRIPT} as VM ${ctid} from ${PR_REPO}@${PR_SHA:0:7}" + local vm_env=( + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + HOME=/root TERM=dumb LANG=C.UTF-8 DIAGNOSTICS=no + VM_UNATTENDED=1 VM_VMID="$ctid" VM_HOSTNAME="$hn" VM_BRIDGE="$BRIDGE" + VM_STORAGE=local VM_ISO_STORAGE=local VM_START=yes + COMMUNITY_SCRIPTS_URL="$base" + ) + # Release lookups (vm_latest_from_github and the like). + [[ -n "${GH_API_TOKEN:-}" ]] && vm_env+=(GITHUB_TOKEN="$GH_API_TOKEN") + touch "$RAW_LOG.start" + + env -i "${vm_env[@]}" timeout --kill-after=1m "$BUILD_TIMEOUT" \ + bash -c 'bash -c "$(curl -fsSL "$1")"' _ "$base/$SCRIPT" "$RAW_LOG" 2>&1 7>&- & + build_pid=$! + start_streams + wait "$build_pid" + exit_code=$? + build_pid="" + stop_streams + + ((exit_code == 124 || exit_code == 137)) && fail "build timed out after ${BUILD_TIMEOUT}" + if ((exit_code != 0)); then + last=$(failure_line) + step="script exited ${exit_code}${last:+: $last}" + status="fail" + classify_cpu_failure + fail "$step" "$status" + fi + # vm_finish ends on a msg_ok of the script's choosing ("VM created. ..."), + # older scripts on "Completed successfully". exit_script and friends leave + # with 0 too, but on a cross. + last=$(strip_ansi <"$RAW_LOG" | tail -n 1 | cut -c1-200) + if ! grep -qi 'completed successfully' "$RAW_LOG" && ! [[ "$last" =~ ^[[:space:]]*โœ” ]]; then + fail "script exited 0 but never reported success${last:+ (last line: ${last})}" + fi + vm_running || fail "VM ${ctid} is not running after the script finished" + + # The address dnsmasq handed to this VM's MAC. The VM must stay up while it + # boots: one that powers itself off or crashes is a failure. + mac=$(qm config "$ctid" 2>/dev/null | sed -nE 's/^net0:[^=]*=([0-9A-Fa-f:]{17}).*/\1/p' | tr 'A-F' 'a-f') + echo "Waiting up to ${VM_BOOT_WAIT}s for VM ${ctid} (${mac:-no NIC}) to boot and get an address" + for ((waited = 0; waited < VM_BOOT_WAIT; waited += 5)); do + vm_running || fail "VM ${ctid} stopped by itself while booting" + [[ -n "$mac" ]] && ct_ip=$(awk -v m="$mac" '$2 == m {print $3}' "$LEASES" 2>/dev/null | tail -n 1) + [[ -n "$ct_ip" ]] && break + sleep 5 + done + [[ -n "$ct_ip" ]] && echo "VM ${ctid} got ${ct_ip}" || echo "VM ${ctid} got no DHCP lease within ${VM_BOOT_WAIT}s" + + # Only a URL the script builds from the address it found; in a + # summary is a hint for later (after an installer ran), not a promise. + # ${VM_IP}, $VM_IP and ${VM_IP:-} all count. + local ip_re='\$\{?VM_IP(:-[^}]*)?\}?' + pattern=$(grep -oE "https?://${ip_re}(:[0-9]+)?(/[A-Za-z0-9._~/?=&%-]*)?" "$tree/$SCRIPT" | head -n1) + if [[ -n "$pattern" ]]; then + [[ -n "$ct_ip" ]] || fail "no IP address within ${VM_BOOT_WAIT}s, so ${pattern} could not be checked" + url=$(sed -E "s#${ip_re}#${ct_ip}#" <<<"$pattern") + for ((waited = 0; waited < VM_URL_WAIT; waited += 10)); do + vm_running || fail "VM ${ctid} stopped by itself while its web UI was awaited" + http_code=$(curl -sk -L --max-redirs 5 -o /dev/null -w '%{http_code}' --max-time 10 "$url" || true) + [[ "$http_code" =~ ^2[0-9][0-9]$ ]] && break + sleep 10 + done + [[ "$http_code" =~ ^[1-5][0-9][0-9]$ ]] && take_screenshot screenshot.png + else + # Let the console settle on a login prompt rather than a boot message. + sleep 20 + fi + [[ -s "$OUT_DIR/screenshot.png" ]] || vm_console_shot + if [[ -n "$url" && ! "$http_code" =~ ^2[0-9][0-9]$ ]]; then + fail "${url} answered ${http_code/#000/nothing} instead of 200 within ${VM_URL_WAIT}s" + fi + vm_running || fail "VM ${ctid} stopped by itself" + + status="pass" + finish +} From 009c7b0c0eda9b7e83d54b88e53045f6d7e73c34 Mon Sep 17 00:00:00 2001 From: "community-scripts-pr-app[bot]" <189241966+community-scripts-pr-app[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:30:22 +0000 Subject: [PATCH 4/4] Update CHANGELOG.md (#18608) Co-authored-by: github-actions[bot] --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd9201ec1..c2c8f0e72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -591,7 +591,8 @@ Exercise vigilance regarding copycat or coat-tailing sites that seek to exploit ### ๐Ÿ’พ Core - - feat: let setup_deb822_repo use a keyring already on the system [@MickLesk](https://github.com/MickLesk) ([core#227](https://github.com/community-scripts/core/pull/227)) + - core: drop the SUDO_ variables an update through sudo carries [@MickLesk](https://github.com/MickLesk) ([core#229](https://github.com/community-scripts/core/pull/229)) +- feat: let setup_deb822_repo use a keyring already on the system [@MickLesk](https://github.com/MickLesk) ([core#227](https://github.com/community-scripts/core/pull/227)) ## 2026-11-10