From e6554d695616daa79b39fc5b1d5968b82ec77bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B6gl-Brunner=20Michel?= Date: Mon, 16 Nov 2026 11:16:21 +0100 Subject: [PATCH] PR script test: call the shared workflow in community-scripts/ci pr-script-test.yml and arm-script-test.yml keep only their triggers and the queue and call the reusable workflows in community-scripts/ci, which hold the logic and the harness scripts (identical in DevScripts and ProxmoxVE so far). Screenshots move from the ci-screenshots branch here to the screenshots branch of community-scripts/ci, squashed weekly; the report job no longer needs contents: write on this repository. --- .github/workflows/arm-script-test.yml | 108 +-- .github/workflows/pr-script-test.yml | 852 +---------------- .github/workflows/scripts/arm-runner-setup.sh | 192 ---- .github/workflows/scripts/pve-run-script.sh | 863 ------------------ .github/workflows/scripts/pve-run-vm.sh | 140 --- 5 files changed, 33 insertions(+), 2122 deletions(-) delete mode 100644 .github/workflows/scripts/arm-runner-setup.sh delete mode 100644 .github/workflows/scripts/pve-run-script.sh delete mode 100644 .github/workflows/scripts/pve-run-vm.sh diff --git a/.github/workflows/arm-script-test.yml b/.github/workflows/arm-script-test.yml index 9b551cdb4..1d6a9dd67 100644 --- a/.github/workflows/arm-script-test.yml +++ b/.github/workflows/arm-script-test.yml @@ -1,21 +1,9 @@ name: ARM script test (manual) -# Builds one ct/ script on GitHub's free arm64 runner (ubuntu-24.04-arm), -# inside Incus, with the same harness as pr-script-test.yml. Started by hand -# from the Actions tab; it posts nothing anywhere: the result is in the job -# summary and the artifact (with the screenshots). -# -# pr-script-test.yml runs the same ARM build for every /test, as information -# only. This one is for trying a script or branch on ARM without a PR. -# -# Public repositories get the standard hosted runners, arm64 included, at no -# cost. Each job is a fresh VM that is thrown away afterwards, so the script -# under test never touches the lab. -# -# The engine refuses to guess on arm64: a script that leaves var_arm64 unset -# stops with "refusing to guess". Here var_arm64=no means "not built", yes is -# built as is, and unset is built with var_arm64=yes to see whether it would -# work. +# Builds one ct/ script on GitHub's free arm64 runner, inside Incus. Started +# by hand from the Actions tab; the result is in the job summary and the +# artifact. The workflow lives in community-scripts/ci, shared with +# DevScripts/ProxmoxVE; this file is identical in both repositories. on: workflow_dispatch: @@ -30,85 +18,13 @@ on: default: main type: string -permissions: - contents: read +permissions: {} jobs: - arm: - runs-on: ubuntu-24.04-arm - timeout-minutes: 120 - steps: - # The harness from the default branch, as in pr-script-test.yml. - - uses: actions/checkout@v7 - with: - sparse-checkout: .github/workflows/scripts - persist-credentials: false - - - name: Decide var_arm64 - id: plan - env: - SCRIPT: ${{ inputs.script }} - REF: ${{ inputs.ref }} - GH_TOKEN: ${{ github.token }} - run: | - [[ "$SCRIPT" =~ ^ct/[A-Za-z0-9._-]+\.sh$ ]] || { echo "::error::script must be ct/.sh"; exit 1; } - [[ "$REF" =~ ^[A-Za-z0-9._/-]+$ ]] || { echo "::error::bad ref"; exit 1; } - sha=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${REF}" --jq .sha) - text=$(gh api "repos/${GITHUB_REPOSITORY}/contents/${SCRIPT}?ref=${sha}" --jq .content | base64 -d) - # A commented-out line ("#var_arm64=...") counts as unset. - arm=$(grep -m1 -oE '^[[:space:]]*var_arm64="[$][{]var_arm64:-(yes|no)[}]"' <<<"$text" | grep -oE ':-(yes|no)' | cut -c3- || true) - echo "sha=$sha" >>"$GITHUB_OUTPUT" - echo "arm64=${arm:-unset}" >>"$GITHUB_OUTPUT" - case "$arm" in - no) echo "::notice::${SCRIPT} sets var_arm64=no: not built on ARM" ;; - yes) echo "${SCRIPT} sets var_arm64=yes" ;; - *) echo "::notice::${SCRIPT} leaves var_arm64 unset: building with var_arm64=yes to see whether it works" ;; - esac - echo "Building ${SCRIPT} at ${sha:0:7} on $(uname -m), $(nproc) cores, $(free -g | awk '/^Mem/ {print $2}') GB RAM, $(df -h / | awk 'NR==2 {print $4}') free" - - - name: Set up Incus - if: steps.plan.outputs.arm64 != 'no' - timeout-minutes: 20 # about 2 when healthy; a hang must not hold the report - run: sudo bash .github/workflows/scripts/arm-runner-setup.sh - - - name: Build and check ${{ inputs.script }} - if: steps.plan.outputs.arm64 != 'no' - env: - PLATFORM: incus - VAR_ARM64: "yes" - PR_REPO: ${{ github.repository }} - PR_SHA: ${{ steps.plan.outputs.sha }} - SCRIPT: ${{ inputs.script }} - UPSTREAM: ${{ github.repository }} - CTID_BASE: "9000" - GH_API_TOKEN: ${{ github.token }} - OUT_DIR: ${{ runner.temp }}/result - BUILD_TIMEOUT: 90m - run: sudo --preserve-env=PLATFORM,VAR_ARM64,PR_REPO,PR_SHA,SCRIPT,UPSTREAM,CTID_BASE,GH_API_TOKEN,OUT_DIR,BUILD_TIMEOUT,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT bash .github/workflows/scripts/pve-run-script.sh - - - name: Summary - if: always() && steps.plan.outputs.arm64 != 'no' - env: - OUT_DIR: ${{ runner.temp }}/result - ARM64: ${{ steps.plan.outputs.arm64 }} - run: | - sudo chmod -R a+rX "$OUT_DIR" 2>/dev/null || true - r="$OUT_DIR/result.json" - [[ -f "$r" ]] || { echo "No result.json" >>"$GITHUB_STEP_SUMMARY"; exit 0; } - { - echo "### ARM (arm64, Incus): $(jq -r .script "$r")" - echo - echo "| Status | Install | Update | Duration | var_arm64 | Detail |" - echo "|---|---|---|---|---|---|" - jq -r --arg arm "$ARM64" '"| \(.status) | \(if .update_status != "" then "pass" else .status end) | \(if .update_status == "" then "-" else .update_status end) | \(.duration)s | \($arm) | \(if .status == "pass" then (.url // "" | if . == "" then "no URL" else . + " → " + $ARGS.named.c end) else .step end) |"' --arg c "$(jq -r .http_code "$r")" "$r" - echo - ls "$OUT_DIR"/screenshot*.png &>/dev/null && echo "Screenshots: in the \`result-arm\` artifact." || echo "No screenshot." - } >>"$GITHUB_STEP_SUMMARY" - - - if: always() - uses: actions/upload-artifact@v7 - with: - name: result-arm - path: ${{ runner.temp }}/result - if-no-files-found: ignore - retention-days: 14 + arm-script-test: + permissions: + contents: read + uses: community-scripts/ci/.github/workflows/arm-script-test.yml@main + with: + script: ${{ inputs.script }} + ref: ${{ inputs.ref }} diff --git a/.github/workflows/pr-script-test.yml b/.github/workflows/pr-script-test.yml index d1100769b..c14102df5 100644 --- a/.github/workflows/pr-script-test.yml +++ b/.github/workflows/pr-script-test.yml @@ -1,35 +1,14 @@ name: PR script test -# Builds the ct/ and vm/ scripts a pull request touches on real Proxmox and Incus hosts and -# reports the outcome on the PR, as a check run and as one comment kept up to -# date. +# Builds the ct/ and vm/ scripts a pull request touches on real Proxmox and +# Incus hosts and reports the outcome on the PR, as a check run and as one +# comment kept up to date. Comment `/test` (optionally a full SHA, app names +# and platforms) to build again. # -# Every push to a non-draft PR is built automatically, forks included: the -# maintainers decided (2026-11-12) that the lab hosts run contributors' code -# unattended. They are isolated, hold no secret, and are treated as -# compromised anyway; the sweeper and fresh guests per build keep them usable. -# Someone with write access can also comment `/test` (optionally followed by -# a full SHA to pin, the scripts to build, and the platforms to build on) to -# build again or pick what to build. Only a commit that existed when the -# comment was written gets built: a push after it is never covered, not even -# one that lands in the seconds before this job runs. -# -# Approval stays narrow: only PRs the push-app-to-main app opens on a branch -# of this repository (a script moving over from DevScripts, reviewed and -# tested there) are approved when every script passes. Every other PR gets -# the report, the check, and "changes requested" on a real failure. -# -# gate ubuntu-latest who asked, which scripts, which commit -# test proxmox-test / build + checks, one matrix job per script, platform -# incus-test / (pve, incus, arm) and OS; arm runs on GitHub's -# ubuntu-24.04-arm arm64 runner and is information only -# report ubuntu-latest check run + PR comment -# -# The test job gets a read-only token and never sees a secret: the host it runs -# on executes untrusted code and has to be treated as compromised. -# -# This file is identical in DevScripts and ProxmoxVE; github.repository decides -# which URLs get rewritten and which container ID range is used. +# The workflow itself lives in community-scripts/ci and is shared with +# DevScripts/ProxmoxVE; this file only holds the triggers and the queue, and +# is identical in both repositories. It runs in this repository's context: +# its token, its runners, its PRs. on: issue_comment: @@ -38,8 +17,8 @@ on: # default branch, as `/test ` in the issue would. issues: types: [labeled] - # Every PR, see above. pull_request_target runs this file from main, never - # the PR's copy of it, and does not wait for "approve and run" on forks. + # pull_request_target runs this file from main, never the PR's copy of it, + # and does not wait for "approve and run" on forks. pull_request_target: types: [opened, reopened, synchronize, ready_for_review] paths: @@ -78,808 +57,19 @@ concurrency: cancel-in-progress: false jobs: - gate: - if: >- - github.event_name == 'workflow_dispatch' || - (github.event_name == 'issue_comment' && startsWith(github.event.comment.body, '/test')) || - (github.event_name == 'issues' && github.event.label.name == 'Ready For Testing' && !github.event.issue.pull_request) || - (github.event_name == 'pull_request_target' && !github.event.pull_request.draft) - runs-on: ubuntu-latest - permissions: - actions: read # when the run was dispatched, for a /test without SHA - checks: write - contents: read - issues: write - pull-requests: write - outputs: - matrix: ${{ steps.plan.outputs.matrix }} - pr: ${{ steps.plan.outputs.pr }} - repo: ${{ steps.plan.outputs.repo }} - sha: ${{ steps.plan.outputs.sha }} - check_id: ${{ steps.plan.outputs.check_id }} - actor: ${{ steps.plan.outputs.actor }} - not_built: ${{ steps.plan.outputs.not_built }} - trusted: ${{ steps.plan.outputs.trusted }} - kind: ${{ steps.plan.outputs.kind }} - platforms: ${{ steps.plan.outputs.platforms }} - base_sha: ${{ steps.plan.outputs.base_sha }} - steps: - - id: plan - uses: actions/github-script@v9 - with: - script: | - const { owner, repo } = context.repo; - const MAX = 10; - core.setOutput('matrix', '[]'); - - // Trusted: a PR the push-app-to-main app opened on a branch of this - // repository, i.e. a script moving over from DevScripts. - const TRUSTED_AUTHOR = 'push-app-to-main[bot]'; - const isTrusted = p => !!p && p.user && p.user.login === TRUSTED_AUTHOR && p.user.type === 'Bot' && - !!p.head.repo && p.head.repo.full_name === `${owner}/${repo}`; - - let prNumber, requested = ''; - if (context.eventName === 'pull_request_target') { - const p = context.payload.pull_request; - prNumber = p.number; - // Exactly the pushed commit; a later push starts its own run. - requested = p.head.sha; - } else if (context.eventName === 'workflow_dispatch') { - prNumber = Number(context.payload.inputs.pr); - requested = `${context.payload.inputs.sha || ''} ${context.payload.inputs.scripts || ''}`; - } else if (context.eventName === 'issues') { - // "Ready For Testing" on a script issue: the app the issue form - // names, as `/test ` would. Only people who can triage set a - // label, and the default branch is what gets built. - const issue = context.payload.issue; - prNumber = issue.number; - // Triage can label too; building needs write, as for /test. The - // merge workflow labels through an app, i.e. a bot. - const sender = context.payload.sender; - if (sender.type !== 'Bot') { - const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({ owner, repo, username: sender.login }); - if (!['admin', 'maintain', 'write'].includes(perm.permission)) { - core.notice(`${sender.login} has ${perm.permission} access; a test needs write`); - return; - } - } - const body = (issue.body || '').replace(/\r/g, ''); - const field = name => { - const m = body.match(new RegExp(`^###[ \t]*${name}[ \t]*\n+([^\n#][^\n]*)`, 'm')); - return m && m[1].trim() !== '_No response_' ? m[1].trim() : ''; - }; - const type = field('Script Type'); - if (type && !/^(CT|VM)\b/.test(type)) { core.notice(`${type}: only ct/ and vm/ scripts are built`); return; } - const path = field('Script path').replace(/^\/+/, ''); - // Same slug rule as pr-merged-ready-for-testing.yml: the "Name of - // the Script" field, else the title, lowercased, spaces -> dashes. - const slug = (field('Name of the Script') || issue.title).trim().toLowerCase().replace(/\s+/g, '-'); - requested = /^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(path) ? path : slug; - core.info(`Ready For Testing: building ${requested}`); - } else { - prNumber = context.payload.issue.number; - const body = context.payload.comment.body.trim(); - if (!/^\/test(\s|$)/.test(body)) return; - // author_association hides private org members, so ask for the permission itself. - const user = context.payload.comment.user.login; - const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({ owner, repo, username: user }); - if (!['admin', 'maintain', 'write'].includes(perm.permission)) { - core.notice(`${user} has ${perm.permission} access; /test needs write`); - return; - } - await github.rest.reactions.createForIssueComment({ - owner, repo, comment_id: context.payload.comment.id, content: 'rocket', - }); - requested = body.split('\n')[0].slice('/test'.length); - } - - // `/test ` in an issue (not a PR) builds that app from the - // default branch as it is right now, e.g. to see whether a reported - // error reproduces. No PR, so no check run, no review, no approval. - const isIssue = ['issue_comment', 'issues'].includes(context.eventName) && !context.payload.issue.pull_request; - let pr = null, head, headSha; - if (isIssue) { - const { data: info } = await github.rest.repos.get({ owner, repo }); - const { data: branch } = await github.rest.repos.getBranch({ owner, repo, branch: info.default_branch }); - headSha = branch.commit.sha; - head = { owner: { login: owner }, name: repo, full_name: `${owner}/${repo}` }; - } else { - ({ data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber })); - if (pr.state !== 'open' || !pr.head.repo) { core.notice('PR is closed or its fork is gone'); return; } - head = pr.head.repo; - headSha = pr.head.sha; - if (context.eventName === 'pull_request_target' && headSha !== requested) { - core.notice('Pushed to again since; the newer push has its own run'); - return; - } - } - core.setOutput('kind', isIssue ? 'issue' : 'pr'); - core.setOutput('trusted', !isIssue && isTrusted(pr) ? 'true' : 'false'); - const MARK = ''; - const sticky = async body => { - const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number: prNumber }); - // Only our own comment: anyone can paste the marker into theirs. - const mine = comments.find(c => c.body.includes(MARK) && c.user.type === 'Bot'); - if (mine) await github.rest.issues.updateComment({ owner, repo, comment_id: mine.id, body }); - else await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body }); - }; - - // Which commit the approval covers. Reading pr.head.sha alone would - // build whatever was pushed between the comment and this job - // starting, i.e. code nobody looked at. Two ways to be sure: - // - // /test that exact commit, and only while it is the head. A - // full SHA: the author writes both commits, so a short - // prefix is a quick birthday collision away. - // /test the latest commit, if nothing reached the fork after - // the comment. pushed_at is set by GitHub on every push - // and read after the PR, so a push in between shows up. - // Branches of this repository belong to people with - // write access, who could build anything anyway. - const words = requested.split(/\s+/).filter(Boolean); - // Platform words pick where to build: `/test arm`, `/test pve incus - // bark-server`. None means every platform. A run on only some of - // them is reported under its own check name and never approves. - const PLATFORM = { pve: 'pve', proxmox: 'pve', incus: 'incus', arm: 'arm', arm64: 'arm', aarch64: 'arm' }; - const platformOf = t => PLATFORM[t.toLowerCase()]; - const only = new Set(words.map(platformOf).filter(Boolean)); - const onlyList = ['pve', 'incus', 'arm'].filter(p => only.has(p)); - const tokens = words.filter(t => !platformOf(t)); - const isSha = t => /^[0-9a-f]{7,40}$/.test(t); - // App names (bark-server) become ct/ paths; such a name without a - // ct/ script falls back to vm/ below. - const bare = new Set(); - const toPath = t => { - if (/^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(t) || !/^[A-Za-z0-9._-]+$/.test(t)) return t; - const p = `ct/${t.replace(/\.sh$/, '')}.sh`; - bare.add(p); - return p; - }; - const approved = isIssue ? [] : tokens.filter(isSha); - let entries = (isIssue ? tokens : tokens.filter(t => !isSha(t))).map(toPath); - const refuse = async (why, retry) => { - await sticky([MARK, '### Proxmox script test', '', `⚠️ ${why} Nothing was built.`, '', retry].join('\n')); - core.notice(why); - }; - if (isIssue) { - // The default branch is trusted; only which app is needed. - if (entries.length === 0) { - await refuse('In an issue, `/test` needs to know which app.', 'Comment `/test `, for example `/test bark-server` (the name of its `ct/` script).'); - return; - } - } else if (approved.length > 1) { - await refuse('`/test` takes at most one commit.', 'Comment `/test` to build the latest commit.'); - return; - } - if (approved.length === 1 && approved[0] !== headSha) { - await refuse( - approved[0].length !== 40 && headSha.startsWith(approved[0]) - ? 'A short SHA is not enough, it has to be all 40 characters.' - : `\`${approved[0]}\` is not the head of this PR (any more).`, - 'Comment `/test` to build the latest commit.'); - return; - } - if (!isIssue && approved.length === 0 && head.full_name !== `${owner}/${repo}`) { - const askedAt = context.eventName === 'workflow_dispatch' - ? (await github.rest.actions.getWorkflowRun({ owner, repo, run_id: context.runId })).data.created_at - : context.payload.comment.created_at; - const { data: fork } = await github.rest.repos.get({ owner: head.owner.login, repo: head.name }); - if (new Date(fork.pushed_at) >= new Date(askedAt)) { - await refuse( - `Something was pushed to ${head.full_name} after the \`/test\`, so it is not certain which commit was meant.`, - 'Comment `/test` again once you have looked at the latest commit.'); - return; - } - } - if (entries.some(e => !/^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(e))) { - await refuse( - isIssue ? 'Only app names (as in `ct/.sh` or `vm/.sh`) and platforms (`pve`, `incus`, `arm`) can be requested.' : 'Only app names, `ct/.sh` and `vm/.sh` paths, platforms (`pve`, `incus`, `arm`) and the commit can be requested.', - isIssue ? 'For example: `/test bark-server`, `/test arm bark-server` or `/test debian-vm`' : 'For example: `/test`, `/test arm`, or `/test pve incus bark-server`.'); - return; - } - // Everything the PR changes that this test cannot build is named in - // the comment, so a /test never ends in silence. - const notBuilt = []; - const code = s => '`' + String(s).replace(/[`\r\n]/g, '') + '`'; - if (entries.length === 0) { - const files = await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number: prNumber }); - const set = new Set(); - for (const f of files) { - if (f.status === 'removed') continue; - let m; - if ((m = f.filename.match(/^install\/([A-Za-z0-9._-]+)-install\.sh$/))) set.add(`ct/${m[1]}.sh`); - else if (/^(ct|vm)\/[A-Za-z0-9._-]+\.sh$/.test(f.filename)) set.add(f.filename); - else if (/^(tools|turnkey)\/.*\.sh$/.test(f.filename)) notBuilt.push(`${code(f.filename)}: only \`ct/\` and \`vm/\` scripts are built`); - } - entries = [...set].sort(); - } - - const include = []; - // Upgrade test: install the ct/ script from the PR's base branch, - // then update with the PR, as a user who installed before the merge - // would. Only in ProxmoxVE, where users have installed it, and only - // when the PR changes update_script(): with the same update code an - // existing install updates exactly as it does today. New scripts - // have nothing to upgrade from. Proxmox only, once per script. - const UPGRADE_REPO = 'community-scripts/ProxmoxVE'; - let baseSha = ''; - if (!isIssue && `${owner}/${repo}` === UPGRADE_REPO) { - const { data: b } = await github.rest.repos.getBranch({ owner, repo, branch: pr.base.ref }); - baseSha = b.commit.sha; - } - // From `function update_script() {` to the first `}` at the start of - // a line; null without one. - const updateCode = t => ((t || '').match(/^(function[ \t]+)?update_script[ \t]*\(\)[ \t]*\{[\s\S]*?^\}/m) || [null])[0]; - const updateChanged = async (p, headText) => { - if (!baseSha) return false; - let baseText; - try { - const res = await github.rest.repos.getContent({ owner, repo, path: p, ref: baseSha }); - baseText = Buffer.from(res.data.content || '', 'base64').toString('utf8'); - } catch (e) { return false; } - const before = updateCode(baseText), after = updateCode(headText); - return before !== null && after !== null && before !== after; - }; - const fetchScript = async p => { - try { - const res = await github.rest.repos.getContent({ owner: head.owner.login, repo: head.name, path: p, ref: headSha }); - return Buffer.from(res.data.content || '', 'base64').toString('utf8'); - } catch (e) { return null; } - }; - for (let script of entries.slice(0, MAX)) { - let text = await fetchScript(script); - // /test debian-vm in an issue: there is no ct/debian-vm.sh, but a vm/ one. - const wasBare = isIssue || bare.has(script); - if (text === null && wasBare && script.startsWith('ct/')) { - const vm = `vm/${script.slice(3)}`; - if ((text = await fetchScript(vm)) !== null) script = vm; - } - if (text === null) { - const where = isIssue ? 'on the default branch' : 'in this PR'; - notBuilt.push(wasBare - ? (script.startsWith('ct/') ? `${code(script.slice(3, -3))}: neither \`ct/\` nor \`vm/\` has it ${where}` : `${code(script)} does not exist ${where}`) - : `\`${script}\` does not exist in this PR (an \`install/\` script is built through the \`ct/\` script of the same name)`); - continue; - } - const isVm = script.startsWith('vm/'); - // The test hosts are amd64; an ARM-only VM script refuses to run there. - if (isVm && /^\s*vm_require_arch\s+arm64\b/m.test(text)) { - notBuilt.push(`${code(script)}: ARM64 only, the test hosts are amd64`); - continue; - } - // Scripts that ask for the OS get built once per OS. - const oses = !isVm && /msg_menu\s+"Choose the container OS"/.test(text) ? ['debian', 'alpine'] : ['']; - const name = isVm ? `vm-${script.slice(3, -3)}` : script.slice(3, -3); - // Every ct/ script on every platform: the same files run on Proxmox - // and, through community-scripts/core's Incus backend, on Incus. - // VMs are Proxmox only. ARM (Incus on GitHub's arm64 runner) is - // information only and never decides the result; a script that - // says var_arm64=no is not built there. Unset is built with - // var_arm64=yes, to see whether it would work. - const arm = (text.match(/^[ \t]*var_arm64="\$\{var_arm64:-(yes|no)\}"/m) || [])[1] || 'unset'; - let platforms = isVm ? ['pve'] : arm === 'no' ? ['pve', 'incus'] : ['pve', 'incus', 'arm']; - if (!isVm && arm === 'no' && (!only.size || only.has('arm'))) notBuilt.push(`${code(script)} on ARM64: the script sets \`var_arm64=no\``); - if (only.size) { - if (isVm && !only.has('pve')) notBuilt.push(`${code(script)}: VM scripts are built on \`pve\` only`); - platforms = platforms.filter(p => only.has(p)); - } - for (const platform of platforms) { - for (const os of oses) include.push({ script, os, platform, arm64: platform === 'arm' ? arm : '', mode: '', name: `${os ? `${name}-${os}` : name}-${platform}` }); - } - if (!isVm && platforms.includes('pve') && await updateChanged(script, text)) { - const os = oses[0]; - include.push({ script, os, platform: 'pve', arm64: '', mode: 'upgrade', name: `${os ? `${name}-${os}` : name}-pve-upgrade` }); - } - } - for (const script of entries.slice(MAX)) notBuilt.push(`\`${script}\`: at most ${MAX} scripts per run, name it with \`/test ${script}\``); - if (include.length === 0) { - await sticky([ - MARK, '### Proxmox script test', '', - isIssue || onlyList.length - ? `ℹ️ Nothing to build${onlyList.length ? ` on ${onlyList.join(', ')}` : ''}:` - : 'ℹ️ Nothing to build: this test builds `ct/` and `vm/` scripts (and changed `install/` scripts through their `ct/` script).', '', - ...(notBuilt.length ? notBuilt.map(n => `- ${n}`) : ['- this PR changes no `ct/`, `vm/` or `install/` script']), - ].join('\n')); - core.notice('No ct/ or vm/ script to build'); - return; - } - - const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; - // A check run belongs to a PR's commit; an issue has none. - let checkId = ''; - if (!isIssue) { - const { data: check } = await github.rest.checks.create({ - // A partial run gets its own name: it must not stand in for the - // full test, e.g. as a required check. - owner, repo, name: `Proxmox script test${onlyList.length ? ` (${onlyList.join(', ')} only)` : ''}`, head_sha: headSha, status: 'in_progress', - details_url: runUrl, - output: { title: `Building ${include.length}`, summary: include.map(i => `- \`${i.script}\` ${[i.platform, i.os, i.mode].filter(Boolean).join(' · ')}`).join('\n') }, - }); - checkId = String(check.id); - } - - core.setOutput('matrix', JSON.stringify(include)); - core.setOutput('pr', String(prNumber)); - core.setOutput('repo', head.full_name); - core.setOutput('sha', headSha); - core.setOutput('check_id', checkId); - core.setOutput('not_built', JSON.stringify(notBuilt)); - core.setOutput('platforms', onlyList.join(' ')); - core.setOutput('base_sha', baseSha); - // Who asked: the /test commenter, or whoever dispatched the run. - core.setOutput('actor', context.actor); - - await sticky([ - MARK, '### Proxmox script test', '', - `⏳ Building ${include.length} at ${headSha.slice(0, 7)}${isIssue ? ' (default branch)' : ''}${onlyList.length ? `, only on ${onlyList.join(', ')}` : ''} — [watch it live](${runUrl}) (open the \`test\` job)`, '', - ...include.map(i => `- \`${i.script}\` (${[i.platform, i.os, i.mode && 'upgrade from the base branch'].filter(Boolean).join(' · ')})`), - ...(notBuilt.length ? ['', '**Not built:**', ...notBuilt.map(n => `- ${n}`)] : []), - ].join('\n')); - - test: - needs: gate - if: needs.gate.outputs.matrix != '[]' && needs.gate.outputs.matrix != '' - # Only `proxmox-test` / `incus-test`, never `self-hosted`: the runners are - # registered without default labels so that no other workflow can be - # scheduled onto them. ARM runs on GitHub's own arm64 runner, a fresh VM - # per job (free for public repositories). - runs-on: ${{ matrix.platform == 'arm' && 'ubuntu-24.04-arm' || matrix.platform == 'incus' && 'incus-test' || 'proxmox-test' }} - # ARM is information only: a failed or missing ARM job (no runner, no - # minutes left, ...) never turns the run red. The report leaves it out of - # the result as well. - continue-on-error: ${{ matrix.platform == 'arm' }} - permissions: - contents: read - timeout-minutes: 120 # install (45) + update (30) + second update + checks - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(needs.gate.outputs.matrix) }} - name: test (${{ matrix.name }}) - steps: - # The default branch, never the PR: the harness itself must be trusted. - - uses: actions/checkout@v7 - with: - sparse-checkout: .github/workflows/scripts - persist-credentials: false - - # The arm64 runner is a plain Ubuntu VM: Incus and the screenshot - # browser are set up on it for this one job. - - name: Set up the ARM runner - if: matrix.platform == 'arm' - timeout-minutes: 20 # about 2 when healthy; a hang must not hold the report - run: sudo bash .github/workflows/scripts/arm-runner-setup.sh - - - name: Build and check ${{ matrix.script }} - env: - PR_REPO: ${{ needs.gate.outputs.repo }} - PR_SHA: ${{ needs.gate.outputs.sha }} - SCRIPT: ${{ matrix.script }} - OS: ${{ matrix.os }} - # ARM is the Incus harness on another machine. - PLATFORM: ${{ matrix.platform == 'arm' && 'incus' || matrix.platform }} - VAR_ARM64: ${{ matrix.platform == 'arm' && 'yes' || '' }} - # upgrade: install from the base branch, update with the PR. - TEST_MODE: ${{ matrix.mode || '' }} - BASE_SHA: ${{ needs.gate.outputs.base_sha }} - UPSTREAM: ${{ github.repository }} - CTID_BASE: ${{ github.repository == 'community-scripts/ProxmoxVE' && '9050' || '9000' }} - # Only so release lookups are not rate limited: unauthenticated, all - # test containers share the lab's one public IP and 60 calls an hour. - # This job's token can read public contents and expires with the job. - GH_API_TOKEN: ${{ github.token }} - OUT_DIR: ${{ runner.temp }}/result - # The self-hosted runners run as root; the GitHub one needs sudo (pct - # and incus do). - run: >- - ${{ matrix.platform == 'arm' && 'sudo --preserve-env=PR_REPO,PR_SHA,SCRIPT,OS,PLATFORM,VAR_ARM64,TEST_MODE,BASE_SHA,UPSTREAM,CTID_BASE,GH_API_TOKEN,OUT_DIR,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT' || '' }} - bash .github/workflows/scripts/pve-run-script.sh - - - if: always() && matrix.platform == 'arm' - run: sudo chmod -R a+rX "$RUNNER_TEMP/result" 2>/dev/null || true - - - if: always() - uses: actions/upload-artifact@v7 - with: - name: result-${{ matrix.name }} - path: ${{ runner.temp }}/result - if-no-files-found: ignore - retention-days: 14 - - report: - needs: [gate, test] - if: always() && needs.gate.result == 'success' && needs.gate.outputs.matrix != '[]' && needs.gate.outputs.matrix != '' - runs-on: ubuntu-latest + pr-script-test: + # The most any job in the shared workflow gets; each job there asks for + # less (the test job: contents read only). permissions: actions: read checks: write - contents: write # the ci-screenshots branch only; main is protected + contents: read issues: write pull-requests: write - steps: - - uses: actions/download-artifact@v8 - with: - pattern: result-* - path: results - - - uses: actions/github-script@v9 - env: - MATRIX: ${{ needs.gate.outputs.matrix }} - PR: ${{ needs.gate.outputs.pr }} - SHA: ${{ needs.gate.outputs.sha }} - CHECK_ID: ${{ needs.gate.outputs.check_id }} - TEST_RESULT: ${{ needs.test.result }} - ACTOR: ${{ needs.gate.outputs.actor }} - NOT_BUILT: ${{ needs.gate.outputs.not_built }} - TRUSTED: ${{ needs.gate.outputs.trusted }} - KIND: ${{ needs.gate.outputs.kind }} - PLATFORMS: ${{ needs.gate.outputs.platforms }} - with: - script: | - const fs = require('fs'); - const { owner, repo } = context.repo; - const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; - const matrix = JSON.parse(process.env.MATRIX); - - // download-artifact unpacks a single artifact straight into results/ - // and several into results//, so look for every result.json - // and match it by what it says it is. - const path = require('path'); - const found = new Map(); - const walk = dir => { - if (!fs.existsSync(dir)) return; - for (const e of fs.readdirSync(dir, { withFileTypes: true })) { - const p = path.join(dir, e.name); - if (e.isDirectory()) walk(p); - else if (e.name === 'result.json') { - try { - const r = JSON.parse(fs.readFileSync(p, 'utf8')); - // Written on the host that ran the PR's code: anything but a - // known status is an error, never something new. - if (!['pass', 'fail', 'unsupported', 'error'].includes(r.status)) r.status = 'error'; - const log = path.join(dir, 'install.log'); - r.install_log = fs.existsSync(log) ? fs.readFileSync(log, 'utf8') : ''; - const shot = path.join(dir, 'screenshot.png'); - r.shot_file = fs.existsSync(shot) ? shot : ''; - if (!['updated', 'no-update', 'fail', 'error', ''].includes(r.update_status || '')) r.update_status = ''; - // Installed fine, but the update could not be tested (e.g. a - // GitHub rate limit): not a pass, and not the PR's fault. - if (r.status === 'pass' && r.update_status === 'error') { r.status = 'error'; r.step = `update ${r.update_step}`; } - const ulog = path.join(dir, 'update.log'); - r.update_log = fs.existsSync(ulog) ? fs.readFileSync(ulog, 'utf8') : ''; - const u2log = path.join(dir, 'update2.log'); - r.update2_log = fs.existsSync(u2log) ? fs.readFileSync(u2log, 'utf8') : ''; - if (!['upgrade', ''].includes(r.mode || '')) r.mode = ''; - r.mode = r.mode || ''; - r.notes = Array.isArray(r.notes) ? r.notes.filter(n => typeof n === 'string').slice(0, 8) : []; - r.resources = typeof r.resources === 'string' ? r.resources : ''; - const ushot = path.join(dir, 'screenshot-update.png'); - r.shot_update_file = fs.existsSync(ushot) ? ushot : ''; - // ARM runs the Incus harness on an arm64 machine; result.json - // says so in arch. Only the platforms this workflow knows: an - // unknown one would never match a matrix row anyway. - if (r.arch === 'aarch64') r.platform = 'arm'; - else if (!['pve', 'incus'].includes(r.platform)) r.platform = 'pve'; - found.set(`${r.script}|${r.platform}|${r.os}|${r.mode}`, r); - } catch (err) { core.warning(`unreadable ${p}: ${err.message}`); } - } - } - }; - walk('results'); - - // A job that died before writing result.json still gets a row. - const rows = matrix.map(m => ({ ...(found.get(`${m.script}|${m.platform || 'pve'}|${m.os || 'default'}|${m.mode || ''}`) || { - script: m.script, os: m.os || 'default', platform: m.platform || 'pve', mode: m.mode || '', notes: [], resources: '', update2_log: '', - status: process.env.TEST_RESULT === 'cancelled' ? 'cancelled' : 'error', - step: 'no result (runner or job failure)', duration: 0, log_tail: '', install_log: '', shot_file: '', - update_status: '', update_log: '', shot_update_file: '', - }), name: m.name })); - // Where a row ran: "pve", "incus · alpine", "arm64", "pve · upgrade", or "VM" (Proxmox only). - const isVm = r => r.script.startsWith('vm/'); - const isArm = r => r.platform === 'arm'; - const where = r => { - if (isVm(r)) return 'VM'; - const p = isArm(r) ? 'arm64' : r.platform; - const w = r.os && r.os !== 'default' ? `${p} · ${r.os}` : p; - return r.mode === 'upgrade' ? `${w} · upgrade` : w; - }; - // ARM is information only: shown, never counted. The result, the - // check, the review and the approval come from Proxmox and Incus. - const coreRows = rows.filter(r => !isArm(r)); - const armRows = rows.filter(isArm); - - // Screenshots go to an orphan branch, because a comment can only show - // an image that has a URL. One file per PR and script, overwritten on - // every run; the link names the commit, so no cache shows an old one. - // The PNG comes from the host that ran the PR's code: check it again. - const BRANCH = 'ci-screenshots'; - const prNum = Number(process.env.PR); - const isPng = buf => buf.length > 8 && buf.length < 3000000 && - buf.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])); - const ensureBranch = async () => { - try { await github.rest.git.getRef({ owner, repo, ref: `heads/${BRANCH}` }); return; } - catch (e) { if (e.status !== 404) throw e; } - const { data: blob } = await github.rest.git.createBlob({ owner, repo, encoding: 'utf-8', - content: 'Screenshots taken by pr-script-test.yml for the PR comments. Not code; never merge this branch.\n' }); - const { data: tree } = await github.rest.git.createTree({ owner, repo, tree: [{ path: 'README.md', mode: '100644', type: 'blob', sha: blob.sha }] }); - const { data: commit } = await github.rest.git.createCommit({ owner, repo, message: 'Start ci-screenshots', tree: tree.sha, parents: [] }); - await github.rest.git.createRef({ owner, repo, ref: `refs/heads/${BRANCH}`, sha: commit.sha }); - }; - const uploadShot = async (r, local, suffix) => { - const buf = fs.readFileSync(local); - if (!isPng(buf)) { core.warning(`${r.script}: screenshot is not a plausible PNG, skipped`); return ''; } - const file = `${process.env.KIND === 'issue' ? 'issue' : 'pr'}-${prNum}/${r.name}${suffix}.png`; - for (let attempt = 1; attempt <= 3; attempt++) { - try { - await ensureBranch(); - let sha; - try { sha = (await github.rest.repos.getContent({ owner, repo, path: file, ref: BRANCH })).data.sha; } - catch (e) { if (e.status !== 404) throw e; } - const { data } = await github.rest.repos.createOrUpdateFileContents({ - owner, repo, branch: BRANCH, path: file, sha, content: buf.toString('base64'), - message: `Screenshot: ${r.script} (${where(r)})${suffix} for #${prNum} at ${process.env.SHA.slice(0, 7)}`, - }); - return `https://raw.githubusercontent.com/${owner}/${repo}/${data.commit.sha}/${file}`; - } catch (e) { - // Two reports at once race on the branch head; try again. - core.warning(`${r.script}: screenshot upload attempt ${attempt} failed: ${e.message}`); - await new Promise(res => setTimeout(res, 2000 * attempt)); - } - } - return ''; - }; - for (const r of rows) { - if (r.shot_file) r.shot_url = await uploadShot(r, r.shot_file, ''); - if (r.shot_update_file) r.shot_update_url = await uploadShot(r, r.shot_update_file, '-after-update'); - } - - // unsupported: the script needs a CPU feature the test host lacks - // (AVX2 and the like). Not the PR's fault, so not a failure. - const icon = { pass: '✅', fail: '❌', error: '⚠️', cancelled: '⏹️', unsupported: '🖥️' }; - const dur = s => `${Math.floor(s / 60)}m${String(s % 60).padStart(2, '0')}s`; - // Text from the build lands outside code blocks here: no table breaks, - // no code spans, and no @mention of a stranger (zero-width space). - const clean = s => String(s || '').replace(/[|\r\n]/g, ' ').replace(/`/g, "'").replace(/@/g, '@​').slice(0, 200); - const fence = s => String(s || '').replace(/~~~~/g, '~~~'); - - // fail: the app broke, the PR's business. error/cancelled: our test - // setup did, not the PR's. unsupported: the host's CPU is too old. - const count = s => coreRows.filter(r => r.status === s).length; - const passed = count('pass'); - const untestable = count('unsupported'); - const broken = count('fail'); - const setupTrouble = count('error') + count('cancelled'); - const allPass = passed === coreRows.length; - // `/test arm`: nothing that counts was built, so the result is - // information only as a whole. - const armOnly = coreRows.length === 0; - // Built on only some platforms (`/test pve`): a pass there says - // nothing about the others, so it neither approves nor lifts an - // earlier "changes requested". A failure is still a failure. - const partial = !!process.env.PLATFORMS; - const cancelled = (armOnly ? rows : coreRows).every(r => r.status === 'cancelled'); - const of = `**${passed}/${coreRows.length} passed**`; - const armPassed = armRows.filter(r => r.status === 'pass').length; - const armLine = armRows.length && !armOnly - ? `ARM64 (information only, not counted): ${armPassed}/${armRows.length} passed${armPassed < armRows.length ? `; failures there do not block this PR` : ''}.` - : ''; - const headline = armOnly - ? `ℹ️ ARM64 only, information only: **${armPassed}/${armRows.length} passed**` - : broken - ? `❌ ${of}` - : setupTrouble - ? `⚠️ ${of}, ${setupTrouble} could not be tested because of a problem with the test setup (not the PR)` - : untestable - ? `🖥️ ${of}, ${untestable} need${untestable === 1 ? 's' : ''} a newer CPU than the test host has (nothing failed)` - : `✅ ${of}`; - - // Update: the installed app is set back a version and updated with - // the PR's update_script. ➖ = it ran but had nothing to update. - const updateIcon = { updated: '✅', 'no-update': '➖', fail: '❌', error: '⚠️' }; - const detail = r => { - if (r.status !== 'pass') return clean(r.step); - const ip = /^[0-9.]{7,15}$/.test(r.ip || '') ? r.ip : ''; - if (isVm(r) && !r.url) return ip ? `booted, got ${ip}` : 'booted, no DHCP lease (an installer ISO, or no network)'; - const reach = r.url ? `${clean(r.url)} → ${r.http_code}` : 'no URL to check'; - return r.update_status === 'no-update' ? `${reach}; update: nothing to update${r.update_step ? ` (${clean(r.update_step)})` : ''}` : reach; - }; - const table = [ - '| Script | Platform | Install | Update | Duration | Detail |', - '|---|---|---|---|---|---|', - ...rows.map(r => `| \`${r.script}\` | ${where(r)} | ${r.update_status ? '✅' : icon[r.status] || '❔'} | ${updateIcon[r.update_status] || '–'} | ${dur(r.duration || 0)} | ${detail(r)} |`), - ].join('\n'); - - // A comment holds 65536 characters. Each script gets an equal share - // and a log that does not fit loses its beginning: the end is where - // a failure shows. - const budget = Math.floor(52000 / rows.length); - // A check run summary holds 65535 BYTES, not characters, and log - // emoji take 3-4 bytes each. Cut at a character boundary and close - // a ~~~~ block left open. - const fitBytes = (text, max) => { - if (Buffer.byteLength(text) <= max) return text; - let cut = Buffer.from(text).subarray(0, max - 300).toString('utf8').replace(/�+$/, ''); - if ((cut.match(/^~~~~/gm) || []).length % 2) cut += '\n~~~~'; - return `${cut}\n\n… cut to fit GitHub's size limit; the full report is in the PR comment and the logs in the run artifacts.`; - }; - const tail = (text, max) => { - text = fence(text).trimEnd(); - if (text.length <= max) return text; - const kept = text.slice(text.length - max); - const cut = text.slice(0, text.length - max).split('\n').length; - return `… ${cut} earlier lines cut, the full log is in the run artifacts\n${kept.slice(kept.indexOf('\n') + 1)}`; - }; - const details = (summaryLine, text, max) => [ - `
${summaryLine}`, '', - '~~~~text', tail(text, max), '~~~~', '
', - ].join('\n'); - - const logs = rows.map(r => { - const parts = []; - const failed = r.status !== 'pass'; - // A VM script's own output is all there is: no install log inside. - const buildShare = (failed || isVm(r)) && r.log_tail ? Math.min(r.log_tail.length, Math.floor(budget / 3)) : 0; - if (buildShare) parts.push(details(`${icon[r.status] || '❔'} ${r.script} (${where(r)}) — build log`, r.log_tail, buildShare)); - if (isVm(r)) return parts.join('\n\n'); - // The update log gets up to a third of what is left, the install log the rest. - const updateShare = r.update_log ? Math.min(r.update_log.length, Math.floor((budget - buildShare) / 3)) : 0; - if (r.install_log) { - const lines = r.install_log.trimEnd().split('\n').length; - parts.push(details(`📄 ${r.script} (${where(r)}) — install log (${lines} lines)`, r.install_log, budget - buildShare - updateShare - 400)); - } else if (!buildShare) { - parts.push(`${r.script} (${where(r)}): no install log was captured`); - } - if (updateShare) { - const lines = r.update_log.trimEnd().split('\n').length; - parts.push(details(`🔄 ${r.script} (${where(r)}) — update log (${lines} lines)`, r.update_log, updateShare)); - } - if (r.update2_log) { - parts.push(details(`🔁 ${r.script} (${where(r)}) — second update log`, r.update2_log, Math.min(r.update2_log.length, Math.floor(budget / 8)))); - } - return parts.join('\n\n'); - }); - - const shots = rows.filter(r => r.shot_url || r.shot_update_url).map(r => [ - `
📸 ${r.script} (${where(r)}) — ${r.url ? clean(r.url) : 'console'}${r.shot_url && r.shot_update_url ? ': after install / after update' : r.shot_update_url ? ': after update' : ''}`, '', - [r.shot_url, r.shot_update_url].filter(Boolean) - .map((u, i, all) => `Screenshot of ${r.script}`).join(' '), - '', '
', - ].join('\n')); - - const summary = [ - `${headline} at \`${process.env.SHA.slice(0, 7)}\``, '', - table, '', - ...(armLine ? [`${armLine}`, ''] : []), - ...(rows.some(r => r.mode === 'upgrade') - ? ['`upgrade`: installed from the base branch as it is now, then updated with this PR, as a user who installed before the merge would.', ''] - : []), - ...(() => { - const noted = rows.filter(r => r.notes.length); - return noted.length - ? ['**Notes** (information only, not counted):', ...noted.flatMap(r => r.notes.map(n => `- \`${r.script}\` (${where(r)}): ${clean(n)}`)), ''] - : []; - })(), - ...(() => { - const measured = rows.filter(r => r.resources); - return measured.length - ? ['
📊 Resources', '', '| Script | Platform | Usage |', '|---|---|---|', - ...measured.map(r => `| \`${r.script}\` | ${where(r)} | ${clean(r.resources)} |`), '', '
', ''] - : []; - })(), - ...(partial ? [`Built only on ${process.env.PLATFORMS.split(' ').join(', ')}; comment \`/test\` for every platform.`, ''] : []), - ...shots, '', - ...(() => { try { const n = JSON.parse(process.env.NOT_BUILT || '[]'); return n.length ? ['**Not built:**', ...n.map(x => `- ${x}`), ''] : []; } catch (e) { return []; } })(), - ...logs, '', - `Full logs: [workflow run](${runUrl}) (artifacts \`result-*\`). Comment \`/test${partial ? ` ${process.env.PLATFORMS}` : ''}${process.env.KIND === 'issue' ? ' ' + [...new Set(rows.map(r => r.script.slice(3, -3)))].join(' ') : ''}\` to build again.`, - ].join('\n'); - - // Issues have no check run (there is no PR commit to attach it to). - if (process.env.CHECK_ID) { - await github.rest.checks.update({ - owner, repo, check_run_id: Number(process.env.CHECK_ID), status: 'completed', - // Untestable here is "test it by hand", not a pass: action_required, - // because a required check would count neutral as passed. - conclusion: cancelled ? 'cancelled' : armOnly ? 'neutral' : allPass ? 'success' : broken ? 'failure' : 'action_required', - output: { - title: armOnly - ? `ARM64 ${armPassed}/${armRows.length} passed (information only)` - : `${passed}/${coreRows.length} passed${armRows.length ? ` (+ ARM ${armPassed}/${armRows.length}, not counted)` : ''}`, - summary: fitBytes(summary, 65000), - }, - }); - } - - // The result pings whoever started the test. A mention added by - // editing a comment does not reliably notify, so the old comment - // goes and the result is posted as a new one, at the end of the - // thread where it is looked for. - // Whoever started the test and the PR's author, once each; no bots. - const pr = Number(process.env.PR); - const isIssue = process.env.KIND === 'issue'; - const { data: prInfo } = isIssue - ? await github.rest.issues.get({ owner, repo, issue_number: pr }) - : await github.rest.pulls.get({ owner, repo, pull_number: pr }); - const author = prInfo.user && prInfo.user.type === 'User' ? prInfo.user.login : ''; - const login = /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$/; - const ping = [...new Set([process.env.ACTOR || '', author])].filter(l => login.test(l)).map(l => `@${l} `).join(''); - const MARK = ''; - const body = `${MARK}\n### Proxmox script test\n\n${ping}${summary}`.slice(0, 65000); - const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number: pr }); - for (const c of comments.filter(c => c.body.includes(MARK) && c.user.type === 'Bot')) { - await github.rest.issues.deleteComment({ owner, repo, comment_id: c.id }); - } - const { data: posted } = await github.rest.issues.createComment({ owner, repo, issue_number: pr, body }); - // An issue has nothing to review or approve: the comment is all. - if (isIssue) return; - - // A real failure requests changes on the tested commit. Untestable - // (CPU), runner errors and cancelled runs are not the PR's fault and - // leave the review state alone. Our previous "changes requested" is - // always dismissed first: a pass lifts it, a new failure replaces it. - const REVIEW_MARK = ''; - const failedRows = cancelled ? [] : coreRows.filter(r => r.status === 'fail'); - const sha7 = process.env.SHA.slice(0, 7); - // Trusted PRs (scripts moving over from DevScripts) are approved when - // every script passes -- not on untestable or setup trouble, which - // means "not tested", not "good". That approval counts toward the - // ruleset's required review; a push dismisses it as stale, and a - // later run that does not fully pass withdraws it. - // Anyone with write access can push to a migration branch, and the - // approval counts toward the required review. So it only covers - // what a migration contains: ct/ and install/ scripts. A PR that - // also changes anything else (workflows, misc/, ...) is tested but - // never approved. - let approve = process.env.TRUSTED === 'true' && allPass && !partial; - // A partial run that passed leaves every review as it is. - const leaveReviews = partial && !failedRows.length; - let noApproveWhy = ''; - if (approve) { - const changed = await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number: pr }); - const other = changed.map(f => f.filename).filter(n => !/^(ct|install)\/[A-Za-z0-9._-]+\.sh$/.test(n)); - if (other.length) { - approve = false; - noApproveWhy = `Not approved automatically: besides \`ct/\` and \`install/\` scripts this PR changes ${other.slice(0, 5).map(n => '`' + n.replace(/[`\r\n]/g, '') + '`').join(', ')}${other.length > 5 ? ', …' : ''}. That needs a human review.`; - } - } - if (noApproveWhy) { - await github.rest.issues.updateComment({ owner, repo, comment_id: posted.id, body: `${body}\n\n> ${noApproveWhy}`.slice(0, 65000) }); - } - try { - const reviews = await github.paginate(github.rest.pulls.listReviews, { owner, repo, pull_number: pr }); - const ours = state => reviews.filter(v => v.user && v.user.type === 'Bot' && v.state === state && (v.body || '').includes(REVIEW_MARK)); - for (const v of leaveReviews ? [] : ours('CHANGES_REQUESTED')) { - await github.rest.pulls.dismissReview({ - owner, repo, pull_number: pr, review_id: v.id, - message: failedRows.length ? 'Replaced by a newer Proxmox script test.' : `Proxmox script test passed at ${sha7}.`, - }); - } - if (!approve && !cancelled && !leaveReviews) { - for (const v of ours('APPROVED')) { - await github.rest.pulls.dismissReview({ - owner, repo, pull_number: pr, review_id: v.id, - message: `Withdrawn: the Proxmox script test at ${sha7} did not fully pass.`, - }); - } - } - if (approve && !ours('APPROVED').some(v => v.commit_id === process.env.SHA)) { - await github.rest.pulls.createReview({ - owner, repo, pull_number: pr, commit_id: process.env.SHA, event: 'APPROVE', - body: [ - REVIEW_MARK, `**Proxmox script test passed** at \`${sha7}\`: ${rows.map(r => `\`${r.script}\``).join(', ')}.`, '', - `Script moved over from DevScripts, built and checked on a real Proxmox host. [Test result](${posted.html_url}).`, - ].join('\n'), - }); - } - if (failedRows.length) { - await github.rest.pulls.createReview({ - owner, repo, pull_number: pr, commit_id: process.env.SHA, event: 'REQUEST_CHANGES', - body: [ - REVIEW_MARK, `**Proxmox script test failed** at \`${sha7}\`:`, '', - ...failedRows.map(r => `- \`${r.script}\` (${where(r)}): ${clean(r.step)}`), '', - `Logs and details: [test result](${posted.html_url}). Push a fix, then ask for \`/test\` again.`, - ].join('\n'), - }); - } - } catch (e) { - core.warning(`Could not update the review state: ${e.message}`); - } + uses: community-scripts/ci/.github/workflows/pr-script-test.yml@main + with: + screenshots_app_id: ${{ vars.GHAPP_SCREENSHOTS_ID }} + # Only the screenshot App, and only the report job uses it: never + # `secrets: inherit`, the test jobs run PR code. + secrets: + screenshots_app_key: ${{ secrets.GHAPP_SCREENSHOTS_PRIVATE_KEY }} diff --git a/.github/workflows/scripts/arm-runner-setup.sh b/.github/workflows/scripts/arm-runner-setup.sh deleted file mode 100644 index e3b33055a..000000000 --- a/.github/workflows/scripts/arm-runner-setup.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env bash -# Turns a GitHub-hosted arm64 runner (ubuntu-24.04-arm) into a throwaway Incus -# test host for pve-run-script.sh (PLATFORM=incus): the same layout as -# incus-ci-host-setup.sh, minus what a VM that lives for one job does not need -# (services, sweeper, runner registration). Run with sudo. -# -# The runner is a fresh VM per job and is discarded afterwards; nothing here -# has to be undone. - -set -euo pipefail - -NET_ADDR=10.98.0.1 -NET_CIDR=10.98.0.0/24 -SERVE_ROOT=/srv/ci -SERVE_PORT=18080 -start=$(date +%s) - -# --- Incus ---------------------------------------------------------------------- -# Ubuntu 24.04's own incus is 6.0.0, which lacks `-f csv,noheader`; the engine -# then finds no storage pool. Zabbly (the Incus maintainers' packages) has the -# current 6.0 LTS, as Debian 13 does. -mkdir -p /etc/apt/keyrings -curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc -# The key Zabbly publishes in its install instructions, not whatever the -# server returns today. -gpg --show-keys --with-colons /etc/apt/keyrings/zabbly.asc 2>/dev/null | - awk -F: '/^fpr/ {print $10; exit}' | grep -qx 4EFC590696CB15B87C73A3AD82CC8797C838DCFD || - { echo "::error::unexpected Zabbly signing key"; exit 1; } -# Only the Incus packages may come from there; everything else stays Ubuntu's. -printf '%s\n' 'Package: *' 'Pin: origin pkgs.zabbly.com' 'Pin-Priority: 1' '' \ - 'Package: incus* liblxc* lxcfs* qemu* edk2*' 'Pin: origin pkgs.zabbly.com' 'Pin-Priority: 600' \ - >/etc/apt/preferences.d/zabbly -printf '%s\n' 'Enabled: yes' 'Types: deb' 'URIs: https://pkgs.zabbly.com/incus/lts-6.0' \ - "Suites: $(. /etc/os-release && echo "$VERSION_CODENAME")" 'Components: main' \ - "Architectures: $(dpkg --print-architecture)" 'Signed-By: /etc/apt/keyrings/zabbly.asc' \ - >/etc/apt/sources.list.d/zabbly-incus-lts-6.0.sources -# Each slow step has its own limit, so a hang names itself in the log instead -# of holding the PR report until the job times out. -step() { local t=$1; shift; timeout "$t" "$@" || { echo "::error::'$*' failed or hung (limit ${t})"; return 1; }; } -# What the runner's network looks like when apt can't get through. -netdiag() { - echo "::group::Network on this runner" - resolvectl status 2>&1 | grep -E 'DNS Servers|Current DNS|resolv.conf mode' || cat /etc/resolv.conf - local h - for h in ports.ubuntu.com pkgs.zabbly.com github.com; do - echo "$h -> $(timeout 10 getent ahostsv4 "$h" | awk '{print $1; exit}')" - timeout 15 curl -s -o /dev/null -w " http %{http_code} in %{time_total}s\n" "http://$h/" || echo " no answer" - done - echo "::endgroup::" -} -# The runner's resolver (Azure's 168.63.129.16) stopped resolving the arm64 -# mirror on 2026-11-11 while every other name worked. Then the name is -# looked up over DNS-over-HTTPS and pinned in /etc/hosts for this job. -for h in ports.ubuntu.com; do - timeout 10 getent ahostsv4 "$h" >/dev/null && continue - ip=$(curl -fsS --max-time 10 -H 'accept: application/dns-json' "https://1.1.1.1/dns-query?name=$h&type=A" | - jq -r '[.Answer[]? | select(.type == 1) | .data][0] // empty') - if [[ "$ip" =~ ^[0-9]+(\.[0-9]+){3}$ ]]; then - echo "$ip $h" >>/etc/hosts - echo "::warning::the runner's DNS can't resolve $h; using $ip from 1.1.1.1" - fi -done -step 5m apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 update -qq || { netdiag; exit 1; } -DEBIAN_FRONTEND=noninteractive step 10m apt-get install -y -qq incus jq >/dev/null - -# Docker on the runner sets the FORWARD policy to DROP, which cuts the Incus -# containers off from the internet. -iptables -P FORWARD ACCEPT - -step 2m incus admin init --preseed </tmp/ci-serve.log 2>&1 & -echo "Incus $(incus --version) ready after $(($(date +%s) - start))s" - -# --- Screenshots ---------------------------------------------------------------- -# Optional: without them the build is still tested. The harness takes one only -# when `chromium` exists, the user ci-shot exists and the CI_SHOT chain keeps -# that user off private addresses (here: the cloud metadata service at -# 169.254.169.254 and the runner itself). -shots() { - useradd --system --create-home --shell /usr/sbin/nologin ci-shot - # Ubuntu's apt chromium is a snap wrapper; Playwright ships a plain arm64 - # build. Started directly with --screenshot, that build crashes on this - # runner (sandbox or not), so `chromium` here is a stand-in that takes the - # harness's arguments and drives the browser through Playwright instead. - mkdir -p /opt/pw - (cd /opt/pw && step 5m npm install --silent --no-audit --no-fund playwright-core@1.49.1 >/dev/null) || return 1 - PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers DEBIAN_FRONTEND=noninteractive \ - step 10m /opt/pw/node_modules/.bin/playwright-core install --with-deps chromium >/dev/null || return 1 - chmod -R a+rX /opt/pw /opt/pw-browsers - cat >/opt/pw/shot.js <<'EOF' -// node shot.js -const { chromium } = require('/opt/pw/node_modules/playwright-core'); -(async () => { - const [url, out, w, h] = process.argv.slice(2); - const browser = await chromium.launch({ chromiumSandbox: true }); - try { - const page = await browser.newPage({ viewport: { width: Number(w), height: Number(h) }, ignoreHTTPSErrors: true }); - await page.goto(url, { waitUntil: 'load', timeout: 30000 }).catch(() => {}); - await page.waitForTimeout(5000); - await page.screenshot({ path: out }); - } finally { - await browser.close(); - } -})().catch(e => { console.error(e.message); process.exit(1); }); -EOF - cat >/usr/local/bin/chromium <<'EOF' -#!/usr/bin/env bash -# Stand-in for `chromium --headless ... --screenshot= --window-size=W,H ` -# on the ARM runner (see arm-runner-setup.sh). Other flags are ignored. -url="" out="" w=1280 h=800 -for a in "$@"; do - case "$a" in - --version) echo "Chromium (Playwright $(node -p "require('/opt/pw/node_modules/playwright-core/package.json').version"))"; exit 0 ;; - --screenshot=*) out="${a#*=}" ;; - --window-size=*) s="${a#*=}"; w="${s%,*}"; h="${s#*,}" ;; - -*) ;; - *) url="$a" ;; - esac -done -[[ -n "$url" && -n "$out" ]] || { echo "usage: chromium --screenshot= " >&2; exit 2; } -PLAYWRIGHT_BROWSERS_PATH=/opt/pw-browsers exec node /opt/pw/shot.js "$url" "$out" "$w" "$h" -EOF - chmod 755 /usr/local/bin/chromium - # Ubuntu 24.04 forbids unprivileged user namespaces through AppArmor, and - # Chromium's sandbox needs them. The sandbox stays on; the restriction goes, - # on a VM that exists for this one job. - sysctl -qw kernel.apparmor_restrict_unprivileged_userns=0 2>/dev/null || true - - local uid fw net ns - uid=$(id -u ci-shot) - for fw in iptables ip6tables; do - $fw -w -N CI_SHOT - $fw -w -I OUTPUT -m owner --uid-owner "$uid" -j CI_SHOT - done - iptables -w -A CI_SHOT -d "$NET_ADDR" -j REJECT - iptables -w -A CI_SHOT -d "$NET_CIDR" -j ACCEPT - for ns in 127.0.0.53 $(awk '/^nameserver/ && $2 !~ /:/ {print $2}' /run/systemd/resolve/resolv.conf 2>/dev/null); do - iptables -w -A CI_SHOT -d "$ns" -p udp --dport 53 -j ACCEPT - iptables -w -A CI_SHOT -d "$ns" -p tcp --dport 53 -j ACCEPT - done - for net in 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.168.0.0/16 224.0.0.0/3; do - iptables -w -A CI_SHOT -d "$net" -j REJECT - done - for net in ::1/128 fc00::/7 fe80::/10 ff00::/8; do - ip6tables -w -A CI_SHOT -d "$net" -j REJECT - done - - # The same call the harness makes, on a page that needs no network. - local dir - dir=$(mktemp -d) - chown ci-shot "$dir" - # HOME: runuser keeps root's, which ci-shot cannot write (the harness does the same). - runuser -u ci-shot -- env HOME="$dir" timeout 60 chromium --headless=new --disable-gpu --hide-scrollbars \ - --no-first-run --window-size=320,200 --user-data-dir="$dir/profile" \ - --screenshot="$dir/shot.png" 'data:text/html,

ci

' &>"$dir/chromium.log" || true - if [[ ! -s "$dir/shot.png" ]]; then - echo "Chromium could not take a test screenshot:" - head -n 15 "$dir/chromium.log" - echo "userns: $(sysctl -n kernel.apparmor_restrict_unprivileged_userns 2>&1)" - rm -rf -- "$dir" - return 1 - fi - rm -rf -- "$dir" - echo "$(chromium --version) ready for screenshots" -} -shots || echo "::warning::no screenshots on this ARM run (the build is still tested)" -echo "Runner ready after $(($(date +%s) - start))s" diff --git a/.github/workflows/scripts/pve-run-script.sh b/.github/workflows/scripts/pve-run-script.sh deleted file mode 100644 index 16a5ca5c3..000000000 --- a/.github/workflows/scripts/pve-run-script.sh +++ /dev/null @@ -1,863 +0,0 @@ -#!/usr/bin/env bash -# Build one ct/ or vm/ script from a pull request on this test host, check -# that the container or VM really works, destroy it, and leave a result.json -# behind. The vm/ part lives in pve-run-vm.sh, sourced from here. -# -# PLATFORM says what this host is: pve (Proxmox VE, pct) or incus. The ct/ -# scripts are the same for both; community-scripts/core picks its Incus -# backend by itself when it finds incus instead of pveversion. Only the way -# this harness reaches into the container differs (guest_* below). -# -# Runs on the self-hosted runner, which *is* the disposable test host. The -# harness comes from the default branch; only the script under test comes from -# the pull request, pinned to the commit a maintainer approved. -# -# The engine hardcodes raw.githubusercontent.com//main for every fetch -# (ct -> build.func -> install script -> install.func -> tools.func). Pointing it -# at a fork branch therefore means rewriting those URLs: the PR tree is unpacked -# under SERVE_ROOT, every upstream main URL in it is replaced by this host's own -# HTTP server, and the containers fetch from there. Engine changes in the PR are -# tested along with the script, which is what a reviewer wants anyway. - -set -uo pipefail - -: "${PR_REPO:?}" "${PR_SHA:?}" "${SCRIPT:?}" "${UPSTREAM:?}" "${CTID_BASE:?}" "${OUT_DIR:?}" -OS="${OS:-}" -PLATFORM="${PLATFORM:-pve}" -CTID_COUNT="${CTID_COUNT:-50}" -SERVE_ROOT="${SERVE_ROOT:-/srv/ci}" -# The containers' own network: vmbr1 + dnsmasq on Proxmox, incusbr0 on Incus. -if [[ "$PLATFORM" == "incus" ]]; then - SERVE_URL="${SERVE_URL:-http://10.98.0.1:18080}" - BRIDGE="${BRIDGE:-incusbr0}" -else - SERVE_URL="${SERVE_URL:-http://10.99.0.1:18080}" - BRIDGE="${BRIDGE:-vmbr1}" -fi -BUILD_TIMEOUT="${BUILD_TIMEOUT:-45m}" -URL_WAIT="${URL_WAIT:-120}" - -# Every value below reaches a path, a URL or a command line. -[[ "$PR_REPO" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad PR_REPO" >&2; exit 2; } -[[ "$UPSTREAM" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad UPSTREAM" >&2; exit 2; } -[[ "$PR_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "bad PR_SHA" >&2; exit 2; } -[[ "$SCRIPT" =~ ^(ct|vm)/[A-Za-z0-9._-]+\.sh$ ]] || { echo "bad SCRIPT" >&2; exit 2; } -[[ -z "$OS" || "$OS" =~ ^(debian|ubuntu|alpine)$ ]] || { echo "bad OS" >&2; exit 2; } -[[ "$PLATFORM" =~ ^(pve|incus)$ ]] || { echo "bad PLATFORM" >&2; exit 2; } -# yes: build with var_arm64=yes. On arm64 the engine refuses to guess for a -# script that leaves var_arm64 unset; the ARM run builds those anyway, to see. -VAR_ARM64="${VAR_ARM64:-}" -[[ -z "$VAR_ARM64" || "$VAR_ARM64" == "yes" ]] || { echo "bad VAR_ARM64" >&2; exit 2; } -# upgrade: install the script as it is on the base branch (BASE_SHA of -# UPSTREAM), then update with the PR. That is what a user does who installed -# before the PR is merged and runs `update` after it. Empty: install and -# update both from the PR. -TEST_MODE="${TEST_MODE:-}" -[[ -z "$TEST_MODE" || "$TEST_MODE" == "upgrade" ]] || { echo "bad TEST_MODE" >&2; exit 2; } -if [[ -n "$TEST_MODE" ]]; then - [[ "${BASE_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || { echo "bad BASE_SHA" >&2; exit 2; } - [[ "$SCRIPT" == ct/* ]] || { echo "upgrade tests are for ct/ scripts" >&2; exit 2; } -fi -[[ "$PLATFORM" == "pve" || "$SCRIPT" == ct/* ]] || { echo "vm/ scripts run on pve only" >&2; exit 2; } -[[ "$CTID_BASE" =~ ^[0-9]+$ ]] || { echo "bad CTID_BASE" >&2; exit 2; } - -mkdir -p "$OUT_DIR" -RAW_LOG="$OUT_DIR/build.raw.log" -LOG="$OUT_DIR/build.log" -RESULT="$OUT_DIR/result.json" -INSTALL_LOG_OUT="$OUT_DIR/install.log" -UPDATE_RAW="$OUT_DIR/update.raw.log" -UPDATE_LOG="$OUT_DIR/update.log" -UPDATE_TIMEOUT="${UPDATE_TIMEOUT:-30m}" -UPDATE2_RAW="$OUT_DIR/update2.raw.log" -UPDATE2_LOG="$OUT_DIR/update2.log" -# How long the app has to keep running without a service restarting. -STABLE_WAIT="${STABLE_WAIT:-60}" - -kind="${SCRIPT%%/*}" # ct or vm -app=$(basename "$SCRIPT" .sh) -job="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${PLATFORM}-${kind}-${app}-${OS:-default}${TEST_MODE:+-$TEST_MODE}" -tree="$SERVE_ROOT/$job" -base="$SERVE_URL/$job" -src_tree="$tree" # what gets installed; the base branch's tree in an upgrade test -started=$(date +%s) - -status="fail" -step="" -exit_code="" -ctid="" # the guest's ID (pve: 9000, incus: ci-9000), container or VM - -# The container, whatever the platform. ctid is set once a slot is reserved. -guest_exec() { - if [[ "$PLATFORM" == "incus" ]]; then incus exec "$ctid" -- "$@"; else pct exec "$ctid" -- "$@"; fi -} -guest_exists() { - if [[ "$PLATFORM" == "incus" ]]; then incus info "$ctid" &>/dev/null; else pct status "$ctid" &>/dev/null; fi -} -guest_running() { - if [[ "$PLATFORM" == "incus" ]]; then - [[ "$(incus list "^${ctid}\$" -c s -f csv 2>/dev/null)" == "RUNNING" ]] - else - pct status "$ctid" 2>/dev/null | grep -q running - fi -} -guest_destroy() { - if [[ "$PLATFORM" == "incus" ]]; then - incus delete "$ctid" --force &>/dev/null || true - else - pct stop "$ctid" --skiplock 1 &>/dev/null || true - pct destroy "$ctid" --purge 1 --force 1 &>/dev/null || true - fi -} -ct_ip="" -url="" -http_code="" -failed_units="" -# updated: a real update ran; no-update: update ran, nothing to update; -# fail: the update broke; empty: not attempted (the install failed). -update_status="" -update_step="" -# install: building and checking what was installed; update: from the first -# update on. In an upgrade test a failure while installing is the base -# branch's, not the PR's. -phase="install" -# Findings that do not fail the build, shown under the table on the PR. -notes=() -# Disk, RAM and OOM kills of the container, measured at the end. -resources="" -# Second `update` right after the first: no-update, updated-again, or empty. -update2_status="" - -# Spinner frames: the engine redraws "⠋ Starting VM" a few times a second -# without a tty too. An alternation, not a bracket: these are multibyte and -# the runner's locale may be C. -SPINNER_RE='^\s*(⠋|⠙|⠹|⠸|⠼|⠴|⠦|⠧|⠇|⠏)' - -strip_ansi() { - sed -E 's/\x1B\[[0-9;?]*[A-Za-z]//g; s/\x1B[()][A-Z0-9]//g; s/\r/\n/g' | grep -vE '^\s*$' | grep -vE "$SPINNER_RE" -} - -# The install script's own output: everything $STD hides from the build log. -# Both engines write it inside the container as /root/.install-.log; -# dev_mode=keep makes sure a failed container is still there to read it from. -# If it is gone anyway, the copy the engine leaves on the host is the fallback. -collect_install_log() { - [[ -n "$ctid" ]] || return 0 - # Exactly one build ran in this container, so the newest install log is ours. - if guest_exists && - guest_exec sh -c 'cat "$(ls -t /root/.install-*.log 2>/dev/null | head -n1)"' 2>/dev/null | - strip_ansi >"$INSTALL_LOG_OUT" && [[ -s "$INSTALL_LOG_OUT" ]]; then - return 0 - fi - [[ "$PLATFORM" == "pve" ]] || { rm -f "$INSTALL_LOG_OUT"; return 0; } - # The engine names its host-side files after the session: pct_create___.log - local sid f - sid=$(find /tmp -maxdepth 1 -name "pct_create_${ctid}_*.log" -newer "$RAW_LOG.start" -printf '%f\n' 2>/dev/null | - sed -E 's/.*_([0-9a-f]+)\.log$/\1/' | head -n1) - [[ -n "$sid" ]] || { rm -f "$INSTALL_LOG_OUT"; return 0; } - for f in /tmp/*-"${ctid}"-"${sid}".log /var/log/community-scripts/install-"${sid}"-combined-*.log; do - [[ -s "$f" ]] && { strip_ansi <"$f" >"$INSTALL_LOG_OUT"; return 0; } - done - rm -f "$INSTALL_LOG_OUT" -} - -# "Requires AVX2" and friends: the script may be fine, the test host's CPU is -# not (Xeon E5-2650, Sandy Bridge: AVX, no AVX2). Shown as untestable rather -# than as a failure of the PR -- but the log is written by the code under test, -# so its word is not enough. Only the engine's error lines count, only when the -# build or install itself failed, and the feature they name must really be -# missing from this host's /proc/cpuinfo. Claiming a feature the host has -# changes nothing. -CPU_RE='(requir|need)[a-z]*[^.]{0,60}(AVX(2|-?512)?|SSE4[._]?2|x86-64-v[234])\b|\b(AVX(2|-?512)?|x86-64-v[234])\b[^.]{0,60}(not (supported|available)|does not have|is missing|unsupported)' - -classify_cpu_failure() { - local line feature flags - line=$(strip_ansi <"$RAW_LOG" | grep -E '✖|\[ERROR\]' | grep -m1 -iE "$CPU_RE" | - sed -E 's/^\s*(✖️?|\[[^]]*\]|\s)*//' | cut -c1-200) - [[ -n "$line" ]] || return 0 - # The CPU flag that would have to be missing for the message to be true. - case "${line,,}" in - *avx-512* | *avx512* | *x86-64-v4*) feature=avx512f ;; - *avx2* | *x86-64-v3*) feature=avx2 ;; - *sse4.2* | *sse4_2* | *x86-64-v2*) feature=sse4_2 ;; - *) return 0 ;; - esac - flags=" $(grep -m1 '^flags' /proc/cpuinfo | cut -d: -f2) " - [[ "$flags" == *" ${feature} "* ]] && return 0 - status="unsupported" - step="needs a newer CPU than the test host has (no ${feature}): ${line}" -} - -# Disk used against var_disk, RAM in use at rest (anonymous memory, without -# the page cache that fills any limit) against the limit, and how often the -# kernel killed something for lack of memory, over the container's whole life, -# the build included. A dir-backed Incus root has no size of its own: df shows -# the host's disk. Then what the container holds is counted with du, and the -# script's var_disk is the size to compare with. -measure_resources() { - local out disk_size disk_used anon limit oom want_disk want_ram line=() - out=$(guest_exec sh -c 'df -Pm / | awk "NR==2 {print \$2, \$3}" - awk "/^anon / {print int(\$2 / 1048576)}" /sys/fs/cgroup/memory.stat 2>/dev/null || echo - - cat /sys/fs/cgroup/memory.max 2>/dev/null || echo - - awk "/^oom_kill / {print \$2}" /sys/fs/cgroup/memory.events 2>/dev/null || echo -' 2>/dev/null) || return 0 - read -r disk_size disk_used <<<"$(sed -n 1p <<<"$out")" - anon=$(sed -n 2p <<<"$out"); limit=$(sed -n 3p <<<"$out"); oom=$(sed -n 4p <<<"$out") - want_disk=$(grep -m1 -oE 'var_disk="\$\{var_disk:-[0-9]+\}"' "$src_tree/$SCRIPT" | grep -oE '[0-9]+' | head -n1) - want_ram=$(grep -m1 -oE 'var_ram="\$\{var_ram:-[0-9]+\}"' "$src_tree/$SCRIPT" | grep -oE '[0-9]+' | head -n1) - [[ "$disk_size" =~ ^[0-9]+$ && "$disk_used" =~ ^[0-9]+$ ]] || return 0 - if [[ -n "$want_disk" ]] && ((disk_size > want_disk * 1024 * 3 / 2)); then - disk_size=$((want_disk * 1024)) - disk_used=$(guest_exec sh -c 'du -sxm / 2>/dev/null | cut -f1' 2>/dev/null) - [[ "$disk_used" =~ ^[0-9]+$ ]] || return 0 - fi - line+=("disk $(awk -v u="$disk_used" 'BEGIN {printf "%.1f", u / 1024}') of $(awk -v s="$disk_size" 'BEGIN {printf "%.0f", s / 1024}') GB") - if ((disk_used * 100 >= disk_size * 90)); then - notes+=("disk ${disk_used} of ${disk_size} MB used after the install: var_disk may be too small") - fi - [[ "$limit" =~ ^[0-9]+$ ]] && limit=$((limit / 1048576)) || limit="${want_ram:-}" - if [[ "$anon" =~ ^[0-9]+$ ]]; then - line+=("RAM at rest ${anon}${limit:+ of ${limit}} MB") - if [[ "$limit" =~ ^[0-9]+$ ]] && ((anon * 100 >= limit * 80)); then - notes+=("${anon} of ${limit} MB RAM in use at rest: var_ram may be too small") - fi - fi - if [[ "$oom" =~ ^[0-9]+$ ]]; then - line+=("OOM kills ${oom}") - ((oom > 0)) && notes+=("the kernel killed ${oom} process(es) for lack of memory: var_ram is too small for the build or the app") - fi - resources=$( - IFS='|' - echo "${line[*]}" - ) - resources=${resources//|/ · } -} - -finish() { - [[ -f "$RAW_LOG" ]] && strip_ansi <"$RAW_LOG" >"$LOG" - [[ -f "$RAW_LOG" ]] && collect_install_log - [[ -f "$UPDATE_RAW" ]] && strip_ansi <"$UPDATE_RAW" >"$UPDATE_LOG" - [[ -f "$UPDATE2_RAW" ]] && strip_ansi <"$UPDATE2_RAW" >"$UPDATE2_LOG" - rm -f "$UPDATE_RAW" "$UPDATE2_RAW" - [[ "$kind" == ct && -n "$ctid" && -z "$resources" ]] && guest_running && measure_resources - jq -n \ - --arg script "$SCRIPT" --arg kind "$kind" --arg platform "$PLATFORM" --arg os "${OS:-default}" --arg status "$status" \ - --arg mode "$TEST_MODE" --arg resources "$resources" --arg update2_status "$update2_status" \ - --arg notes "$(printf '%s\n' "${notes[@]}")" \ - --arg step "$step" --arg exit_code "$exit_code" --arg sha "$PR_SHA" \ - --arg ctid "$ctid" --arg url "$url" --arg http_code "$http_code" \ - --arg failed_units "$failed_units" --arg ip "$ct_ip" --arg arch "$(uname -m)" \ - --arg update_status "$update_status" --arg update_step "$update_step" \ - --argjson duration "$(($(date +%s) - started))" \ - --arg tail "$([[ -f "$LOG" ]] && tail -n 60 "$LOG")" \ - '{script: $script, kind: $kind, platform: $platform, arch: $arch, os: $os, status: $status, step: $step, exit_code: $exit_code, - sha: $sha, ctid: $ctid, ip: $ip, url: $url, http_code: $http_code, - failed_units: $failed_units, update_status: $update_status, update_step: $update_step, - mode: $mode, resources: $resources, update2_status: $update2_status, - notes: ($notes | split("\n") | map(select(. != ""))), - duration: $duration, log_tail: $tail}' >"$RESULT" - echo "::group::result" - jq 'del(.log_tail)' "$RESULT" - echo "::endgroup::" -} - -# --- Live output ------------------------------------------------------------------ -# The job log on GitHub streams while the job runs, so mirroring the build there -# is all "live" takes. Two streams: the engine's own output, and the install -# log inside the container, which holds everything $STD hides (apt, npm, ...). -# -# Both are written by the code under test. A line starting with `::` would be -# read by the runner as a workflow command, so command processing is switched -# off for as long as they run, with a token the build cannot know. -stream_token="" -log_pid="" -install_pid="" - -# Colours stay, cursor movement and spinner redraws go. -live_filter() { - sed -u -E 's/\r/\n/g; s/\x1B\[[0-9;?]*[ABCDEFGHJKSTfhilnsu]//g' | sed -u -E "/^\s*\$/d; /${SPINNER_RE}/d" -} - -start_streams() { - stream_token=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n') - echo "::stop-commands::${stream_token}" - tail -n +1 -F --pid="$build_pid" "$RAW_LOG" 2>/dev/null > >(live_filter) & - log_pid=$! - # A VM has no install log to follow: its script runs on this host only. - [[ "$kind" == "ct" ]] || return 0 - # setsid: its own process group, so stop_streams can take down the - # pct/incus exec + tail pair inside it in one go. - setsid bash -c ' - gx() { if [ "$2" = incus ]; then incus exec "$1" -- "${@:3}"; else pct exec "$1" -- "${@:3}"; fi; } - for _ in $(seq 1 900); do - f=$(gx "$1" "$2" sh -c "ls -t /root/.install-*.log 2>/dev/null | head -n1" 2>/dev/null) - if [ -n "$f" ]; then - echo "──────── install log of CT $1 (live) ────────" - if [ "$2" = incus ]; then exec incus exec "$1" -- tail -n +1 -F "$f"; fi - exec pct exec "$1" -- tail -n +1 -F "$f" - fi - sleep 2 - done' _ "$ctid" "$PLATFORM" 2>/dev/null > >(live_filter | sed -u 's/^/ │ /') & - install_pid=$! -} - -stop_streams() { - [[ -n "$stream_token" ]] || return 0 - [[ -n "$install_pid" ]] && kill -TERM -- "-$install_pid" &>/dev/null - [[ -n "$log_pid" ]] && kill -TERM "$log_pid" &>/dev/null - wait "$log_pid" "$install_pid" &>/dev/null - sleep 1 # let the filters flush - install_pid="" - log_pid="" - echo "::${stream_token}::" - stream_token="" -} - -build_pid="" -update_pid="" -cleanup() { - stop_streams - local pid - for pid in "$build_pid" "$update_pid"; do - [[ -n "$pid" ]] || continue - pkill -TERM -P "$pid" &>/dev/null || true - kill -TERM "$pid" &>/dev/null || true - sleep 2 - done - if [[ -n "$ctid" && "$kind" == "vm" ]]; then - vm_cleanup - elif [[ -n "$ctid" ]]; then - guest_exists && guest_destroy - rm -f "/run/ci-jobs/$ctid" - # What the engine leaves on the host for this container and its session. - local f sid - [[ "$PLATFORM" == "pve" ]] && for f in /tmp/pct_create_"${ctid}"_*.log; do - [[ -f "$f" && "$f" -nt "$RAW_LOG.start" ]] || continue - sid=$(sed -E 's/.*_([0-9a-f]+)\.log$/\1/' <<<"$f") - rm -f -- "$f" /tmp/create-lxc-"${sid}".log /tmp/*-"${ctid}"-"${sid}".log - done - fi - rm -f "$RAW_LOG.start" - rm -rf -- "$tree" "$tree-base" -} -trap cleanup EXIT -# A cancelled job sends TERM; without this the EXIT trap would not run. -trap 'exit 143' TERM -trap 'exit 130' INT - -# fail [fail|unsupported]: only the build/install branches pass the -# second argument, after classify_cpu_failure has checked the host. -# -# The exit code says whether *this job* worked, not whether the app did. A -# script that fails to build is a result: it is written down, reported on the -# PR (check, comment, review) and the job exits 0 -- a red run per broken app -# would only be notification noise. Only trouble with the test setup itself -# (setup_error) fails the job, because that is ours to look at. -fail() { - step="$1" - status="${2:-fail}" - [[ "$status" == "unsupported" ]] || status="fail" - if [[ "$TEST_MODE" == "upgrade" && "$phase" == "install" && "$status" == "fail" ]]; then - status="error" - step="the install before this PR (${UPSTREAM}@${BASE_SHA:0:7}) failed, so the upgrade was not tested: $1" - fi - finish - echo "::warning title=${SCRIPT}::${step}" - exit 0 -} - -setup_error() { - step="$1" - status="error" - finish - echo "::error title=${SCRIPT}::test setup: ${step}" - exit 1 -} - -# Any answer means the app is up, unless it says it isn't: nothing at all, -# 404/410 (the printed URL leads nowhere), 5xx (server error, proxy without -# backend), or a 3xx left after 5 redirects (a loop). 401/403 are a login -# wall (warpgate answers 401), 400 a quirk of the app: both pass. -url_ok() { [[ "$1" =~ ^[1-4][0-9][0-9]$ && ! "$1" =~ ^(3[0-9][0-9]|404|410)$ ]]; } -url_why() { - case "$1" in - 404 | 410) echo "the printed URL leads nowhere" ;; - 5??) echo "server error" ;; - 3??) echo "still redirecting after 5 hops" ;; - *) echo "the app never answered" ;; - esac -} - -# --- Reserve a container ID --------------------------------------------------- -# A lock file per ID, not just "first free one", because the engine only creates -# the container minutes later and two jobs would otherwise pick the same ID. -# VMs take the range 100 above the containers' (DevScripts 9100-9149, -# ProxmoxVE 9150-9199). -mkdir -p /run/ci-jobs -if [[ "$kind" == "vm" ]]; then - source "$(dirname "$0")/pve-run-vm.sh" - CTID_BASE=$((CTID_BASE + 100)) - vm_wait_turn -fi -exec 9>/run/ci-jobs/.lock -flock 9 -# On Incus the ID is an instance name: var_ctid=ci-9000 names the container. -for ((id = CTID_BASE; id < CTID_BASE + CTID_COUNT; id++)); do - name=$id - [[ "$PLATFORM" == "incus" ]] && name="ci-$id" - [[ -e "/run/ci-jobs/$name" ]] && continue - if [[ "$PLATFORM" == "incus" ]]; then - incus info "$name" &>/dev/null && continue - else - pvesh get /cluster/nextid --vmid "$id" &>/dev/null || continue - fi - echo "$job" >"/run/ci-jobs/$name" - ctid=$name - break -done -flock -u 9 -exec 9>&- -[[ -n "$ctid" ]] || setup_error "no free ${kind} ID in ${CTID_BASE}-$((CTID_BASE + CTID_COUNT - 1))" - -# --- Fetch the approved commit and point it at ourselves ------------------------ -mkdir -p "$tree" -curl -fsSL --retry 3 "https://codeload.github.com/${PR_REPO}/tar.gz/${PR_SHA}" | - tar -xz -C "$tree" --strip-components=1 || setup_error "could not download ${PR_REPO}@${PR_SHA:0:7}" -[[ -f "$tree/$SCRIPT" ]] || fail "${SCRIPT} does not exist at ${PR_SHA:0:7}" - -upstream_re="https://raw\.githubusercontent\.com/${UPSTREAM//./\\.}/(refs/heads/)?main" -grep -rlE "$upstream_re" "$tree" --include='*.sh' --include='*.func' | - xargs -r sed -i -E "s#${upstream_re}#${base}#g" - -# What gets installed: the PR, or in an upgrade test the base branch. -src_tree="$tree" -src_base="$base" -if [[ "$TEST_MODE" == "upgrade" ]]; then - src_tree="$tree-base" - src_base="$base-base" - mkdir -p "$src_tree" - curl -fsSL --retry 3 "https://codeload.github.com/${UPSTREAM}/tar.gz/${BASE_SHA}" | - tar -xz -C "$src_tree" --strip-components=1 || setup_error "could not download ${UPSTREAM}@${BASE_SHA:0:7}" - [[ -f "$src_tree/$SCRIPT" ]] || fail "${SCRIPT} does not exist at ${UPSTREAM}@${BASE_SHA:0:7}" - grep -rlE "$upstream_re" "$src_tree" --include='*.sh' --include='*.func' | - xargs -r sed -i -E "s#${upstream_re}#${src_base}#g" -fi - -if [[ "$kind" == "vm" ]]; then - vm_build_and_check - exit 0 -fi - -# --- Build ---------------------------------------------------------------------- -if [[ "$TEST_MODE" == "upgrade" ]]; then - echo "Upgrade test: installing ${SCRIPT} from ${UPSTREAM}@${BASE_SHA:0:7} (before this PR), then updating with ${PR_REPO}@${PR_SHA:0:7}" -fi -echo "Building ${SCRIPT} (${OS:-default OS}) on ${PLATFORM} as CT ${ctid} from $([[ "$TEST_MODE" == upgrade ]] && echo "${UPSTREAM}@${BASE_SHA:0:7}" || echo "${PR_REPO}@${PR_SHA:0:7}")" -hostname_safe="ci-$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}" | cut -c1-50)" -build_env=( - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - HOME=/root TERM=dumb LANG=C.UTF-8 - mode=default MODE=default PHS_SILENT=1 DIAGNOSTICS=no - var_ctid="$ctid" var_hostname="$hostname_safe" var_brg="$BRIDGE" - "dev_mode=timing,keep" COMMUNITY_SCRIPTS_URL="$src_base" -) -if [[ "$PLATFORM" == "incus" ]]; then - build_env+=(var_container_storage=default) -else - build_env+=(var_container_storage=local var_template_storage=local) -fi -touch "$RAW_LOG.start" -[[ -n "$OS" ]] && build_env+=(var_os="$OS") -[[ -n "$VAR_ARM64" ]] && build_env+=(var_arm64=yes) -# A script that hands over to upstream's own installer (run_external_installer) -# asks first; nobody can answer in CI, and testing the script means running it. -build_env+=(var_external_installer=yes) -# Both engines turn var_github_token into GITHUB_TOKEN inside the container. -[[ -n "${GH_API_TOKEN:-}" ]] && build_env+=(var_github_token="$GH_API_TOKEN") -# How much API quota this build starts with, with the token and without (the -# lab's one address). /rate_limit itself costs nothing. A "rate limit" in the -# build is then easy to place. -gh_quota() { - curl -s --max-time 10 ${1:+-H "Authorization: Bearer $1"} https://api.github.com/rate_limit 2>/dev/null | - jq -r '.rate | "\(.remaining) of \(.limit) left, resets in \(((.reset - now) / 60) | floor) min"' 2>/dev/null || echo "unknown" -} -echo "GitHub API quota: token $(gh_quota "${GH_API_TOKEN:-}") · no token $(gh_quota "")" - -# --- Answers for the script's questions ------------------------------------------ -# Nobody types into CI, so anything the install asks has to be answered up front. -# AGENTS.md rule 24: a script reads var_ first and prompts only when it is -# unset (or uses prompt_input_required ... "var_x"). Every such variable gets a -# CI value here. The names come from json/.json app_vars (DevScripts) and -# from the read/prompt_ lines of the install script (both repos). A bare `read` -# that ignores its variable still fails, and the report says why. -ENGINE_VARS=" var_ctid var_hostname var_os var_version var_cpu var_ram var_disk var_brg var_net var_container_storage var_template_storage var_unprivileged var_tags var_arm64 var_testurl var_verbose var_timezone var_http_proxy var_http_no_proxy var_appversion var_github_token " -host_app=$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}") -ci_value() { - case "${1,,}" in - *pass* | *secret* | *token* | *key*) head -c 32 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | cut -c1-20 ;; - *email* | *mail*) echo "ci@example.test" ;; - *url* | *endpoint*) echo "https://ci-${host_app}.example.test" ;; - # One name per variable: var_opencloud_collabora_fqdn -> - # ci-opencloud-collabora.example.test, so several of them cannot collide. - *host* | *domain* | *fqdn*) - local part="${1#var_}" - part=$(sed -E 's/_?(fqdn|hostname|host|domain)$//' <<<"${part,,}" | tr '_' '-') - echo "ci-${part:-$host_app}.example.test" - ;; - *user*) echo "ciadmin" ;; - *) echo "ci-test" ;; - esac -} -declare -A answers=() -json="$src_tree/json/${app}.json" -if [[ -f "$json" ]]; then - while IFS=$'\t' read -r name default; do - answers[$name]="$default" - done < <(jq -r '(if type == "array" then .[0] else . end).app_vars // [] | .[] | - select(.name | test("^var_[a-z0-9_]{1,40}$")) | [.name, (.default // "" | tostring)] | @tsv' "$json" 2>/dev/null) -fi -# Names built at run time from a loop, e.g. OpenCloud: -# servers=("opencloud" "collabora" "wopi") -# for server in "${servers[@]}"; do fqdn_var="var_opencloud_${server}_fqdn" -# Each var_...${x}... template is expanded with the words of `for x in ...` -# (or of the array that loop walks). Nothing is executed, only read. -expand_templates() { - local tpl x pre suf words arr w - while read -r tpl; do - x=$(sed -E 's/^[^$]*\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?.*$/\1/' <<<"$tpl") - pre="${tpl%%\$*}" - suf=$(sed -E 's/^[^$]*\$\{?[A-Za-z_][A-Za-z0-9_]*\}?//' <<<"$tpl") - words=$(grep -m1 -oE "for ${x} in [^;]*" "$1" | sed -E "s/^for ${x} in //") - if [[ "$words" =~ \$\{([A-Za-z_][A-Za-z0-9_]*)\[@\]\} ]]; then - arr="${BASH_REMATCH[1]}" - words=$(grep -m1 -oE "^\s*${arr}=\([^)]*\)" "$1" | sed -E "s/^\s*${arr}=\(//; s/\)$//") - fi - for w in $words; do - w="${w//[\"\']/}" - [[ "${pre}${w}${suf}" =~ ^var_[a-z0-9_]{1,40}$ ]] && echo "${pre}${w}${suf}" - done - done < <(grep -oE 'var_[a-z0-9_]*\$\{?[A-Za-z_][A-Za-z0-9_]*\}?[a-z0-9_]*' "$1" | sort -u) -} - -# The engine names the install script after APP (lower case, no spaces), which -# need not match the ct/ file name: ct/bluesky-pds.sh -> blueskypds-install.sh. -nsapp=$(grep -m1 -oE '^APP="[^"]+"' "$src_tree/$SCRIPT" | cut -d'"' -f2 | tr -d ' ' | tr '[:upper:]' '[:lower:]') -for f in "$src_tree/install/${app}-install.sh" "$src_tree/install/${nsapp}-install.sh" "$src_tree/$SCRIPT"; do - [[ -f "$f" ]] || continue - while read -r name; do - [[ -v "answers[$name]" ]] || answers[$name]="" - done < <({ - grep -E '\bread\b|prompt_(input|input_required|confirm|select|password)\b' "$f" | grep -oE '\bvar_[a-z0-9_]{1,40}\b' - expand_templates "$f" - } | sort -u) -done -for name in "${!answers[@]}"; do - [[ "$ENGINE_VARS" == *" $name "* ]] && continue - value="${answers[$name]}" - [[ -n "$value" ]] || value=$(ci_value "$name") - build_env+=("$name=$value") - case "${name,,}" in - *pass* | *secret* | *token* | *key*) echo "Pre-answered ${name}=(random)" ;; - *) echo "Pre-answered ${name}=${value}" ;; - esac -done - -# In the background so that a TERM from a cancelled job is handled at once: -# bash defers traps until a foreground command returns. -env -i "${build_env[@]}" timeout --kill-after=1m "$BUILD_TIMEOUT" \ - bash -c 'bash -c "$(curl -fsSL "$1")"' _ "$src_base/$SCRIPT" "$RAW_LOG" 2>&1 & -build_pid=$! -start_streams -wait "$build_pid" -exit_code=$? -build_pid="" -stop_streams - -if ((exit_code == 124 || exit_code == 137)); then - fail "build timed out after ${BUILD_TIMEOUT}" -fi -# The engine marks the step that broke with a cross. Its closing "Installation -# failed in container" line is generic, so the cross before it is the useful one. -failure_line() { - local line - line=$(strip_ansi <"$RAW_LOG" | grep -E '✖|\[ERROR\]|error:' | grep -v 'Installation failed in container' | - tail -n 1 | sed -E 's/^\s*(✖️?|\s)*//' | cut -c1-200) - # A prompt nobody can answer: say what to do instead. - # First, because the report keeps 200 characters. - if [[ "$line" == *"read exited with status"* || "$line" == *"executing command read "* ]]; then - local asked - asked=$(grep -oE '\bvar_[a-z0-9_]{1,40}\b' <<<"$line" | tail -n 1) - line="unanswerable prompt (AGENTS.md rule 24): ask only when ${asked:-var_} is unset, or use prompt_input_required. ${line}" - fi - echo "$line" -} -if ((exit_code != 0)); then - last=$(failure_line) - step="build exited ${exit_code}${last:+: $last}" - status="fail" - classify_cpu_failure - fail "$step" "$status" -fi -# With dev_mode=keep (needed to read the install log) the engine carries on -# after a failed install and still prints "Completed Successfully", exit 0. -# The container is a number on Proxmox (9000) and a name on Incus (ci-9000). -install_exit=$(strip_ansi <"$RAW_LOG" | grep -oE 'Installation failed in container [A-Za-z0-9._-]+ \(exit code: [0-9]+\)' | - grep -oE '[0-9]+\)$' | tr -d ')' | tail -n 1) -if [[ -n "$install_exit" ]]; then - last=$(failure_line) - step="install script exited ${install_exit}${last:+: $last}" - status="fail" - classify_cpu_failure - fail "$step" "$status" -fi -grep -qi 'completed successfully' "$RAW_LOG" || fail "build exited 0 but never reported success" - -# --- Checks --------------------------------------------------------------------- -guest_running || fail "container ${ctid} is not running" - -ct_ip=$(guest_exec sh -c "ip -4 -o addr show dev eth0 | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null | head -n1) - -# What the app's page looks like, for the PR comment. Chromium runs as its own -# unprivileged user with its sandbox on: the page is served by code under test. -# ci-shot-firewall (pve-ci-host-setup.sh) keeps that user off this host and the -# lab network; without it there is no screenshot rather than an unguarded one. -# $1: file name in OUT_DIR (screenshot.png after install, -update after update). -take_screenshot() { - command -v chromium >/dev/null && id ci-shot &>/dev/null || return 0 - iptables -w -S CI_SHOT 2>/dev/null | grep -q -- '-d 10.0.0.0/8 .*REJECT' || return 0 - local dir - dir=$(mktemp -d) - chown ci-shot "$dir" - # HOME: runuser keeps root's, which ci-shot cannot write; Playwright's - # Chromium (the ARM runner) then crashes in its crash handler. - runuser -u ci-shot -- env HOME="$dir" timeout 60 chromium --headless=new --disable-gpu --hide-scrollbars \ - --ignore-certificate-errors --no-first-run --window-size=1280,800 --virtual-time-budget=10000 \ - --user-data-dir="$dir/profile" --screenshot="$dir/shot.png" "$url" &>/dev/null - # Only a plausible PNG leaves this function. - if [[ -s "$dir/shot.png" && $(stat -c %s "$dir/shot.png") -lt 3000000 ]] && - [[ "$(head -c 8 "$dir/shot.png" | od -An -tx1 | tr -d ' \n')" == "89504e470d0a1a0a" ]]; then - cp "$dir/shot.png" "$OUT_DIR/${1:-screenshot.png}" - fi - rm -rf -- "$dir" -} - -# Is the app up? No failed services, and the URL the script tells the user to -# open, e.g. http://${IP}:1337/admin, answers without an error (see url_ok; -# redirects are followed, so a jump to /login is fine). A script that prints no URL has no web UI to check. -# Run after the install and again after the update. Sets health_problem. -app_url() { # the URL the script at $1 tells the user to open - local pattern - pattern=$(grep -oE 'https?://\$\{IP\}(:[0-9]+)?(/[A-Za-z0-9._~/?=&%-]*)?' "$1" | head -n1) - [[ -n "$pattern" && -n "$ct_ip" ]] && echo "${pattern/\$\{IP\}/$ct_ip}" -} -url=$(app_url "$src_tree/$SCRIPT") -health_problem="" -check_health() { - health_problem="" - failed_units="" - if guest_exec sh -c 'command -v systemctl' &>/dev/null; then - sleep 10 - failed_units=$(guest_exec systemctl --failed --no-legend --plain 2>/dev/null | awk '{print $1}' | paste -sd, -) - elif guest_exec sh -c 'command -v rc-status' &>/dev/null; then - failed_units=$(guest_exec rc-status --crashed 2>/dev/null | awk '{print $1}' | paste -sd, -) - fi - [[ -z "$failed_units" ]] || { health_problem="failed services: ${failed_units}"; return 1; } - [[ -n "$url" ]] || return 0 - local waited - for ((waited = 0; waited < URL_WAIT; waited += 5)); do - http_code=$(curl -sk -L --max-redirs 5 -o /dev/null -w '%{http_code}' --max-time 10 "$url" || true) - url_ok "$http_code" && break - sleep 5 - done - # Also when the check fails: an error page explains more than a status code. - [[ "$http_code" =~ ^[1-5][0-9][0-9]$ ]] && take_screenshot "$1" - url_ok "$http_code" || - { health_problem="${url} answered ${http_code/#000/nothing} within ${URL_WAIT}s ($(url_why "$http_code"))"; return 1; } -} - -check_health screenshot.png || fail "$health_problem" - -# --- Update ------------------------------------------------------------------------- -# A fresh install is already on the latest release, so a plain `update` would -# only say "No update available" and never run update_script's real path. The -# version files the install wrote (~/., one per fetch_and_deploy_gh_release) -# that update_script checks are set back first (see below); check_for_gh_release -# then sees an update and runs it all. Apps deployed from a branch -# (fetch_and_deploy_gh_branch, checked with check_for_gh_branch) store a commit -# hash instead of a version. `update` is the /usr/bin/update the install generated: it pulls -# ct/.sh from the URL this container was built from, i.e. the PR's tree. -UPDATE_FAKE="0.0.0-ci" -update_fail() { - update_status="fail" - update_step="$1" - fail "update: $1" -} - -version_files=$(guest_exec sh -c ' - for f in /root/.*; do - [ -f "$f" ] || continue - case "$f" in */.install-* | *.pending | *.bak | */.bash* | */.profile | */.hushlogin | */.viminfo | */.lesshst | */.wget-hsts | */.selected_editor) continue ;; esac - [ "$(wc -l <"$f")" -le 1 ] && grep -qE "^(v?[0-9]+(\.[0-9A-Za-z_+-]+)+|[0-9a-f]{7,40})$" "$f" && echo "$f" - done' 2>/dev/null | grep -E '^/root/\.[A-Za-z0-9._-]+$') -# Only set back what update_script checks: the names it passes to -# check_for_*release. The install can also deploy helpers the update never -# touches (immich: ~/.mise); those would stay 0.0.0-ci and fail the run. A -# name that is a variable can't be read here, so then all of them are set back. -tracked=$(grep -oE 'check_for_[a-z]*_?release[[:space:]]+("[^"]*"|[^[:space:]]+)' "$tree/$SCRIPT" | - sed -E 's/^check_for_[a-z]*_?release[[:space:]]+//; s/^"(.*)"$/\1/' | tr -d ' ' | tr '[:upper:]' '[:lower:]' | sort -u) -if [[ -n "$tracked" && "$tracked" != *'$'* ]]; then - version_files=$(grep -xF -f <(sed 's|^|/root/.|' <<<"$tracked") <<<"$version_files" || true) -fi -# A version keeps its digits and gets "-dev" (6.7.16.0 -> 6.7.16.0-dev): the -# engine ranks that below the release, so the update runs, and an -# update_script that reads the installed series (shopware: composer -# "~6.7.0" from ~/.shopware) still gets the real one. 0.0.0-ci broke that. -# A commit hash (or a bare number) has no series and gets 0.0.0-ci. -declare -A update_fake=() -for f in $version_files; do - cur=$(guest_exec cat "$f" 2>/dev/null | tr -d '\r\n') - if [[ "$cur" =~ ^v?[0-9]+(\.[0-9A-Za-z_+-]+)+$ ]]; then - update_fake[$f]="${cur}-dev" - else - update_fake[$f]="$UPDATE_FAKE" - fi - guest_exec sh -c "echo '${update_fake[$f]}' > '$f'; rm -f '$f.pending'" || - update_fail "could not reset $f" - echo "Update test: set back $f from ${cur} to ${update_fake[$f]}" -done - -phase="update" -# Upgrade test: the container pulls its update from where it was installed -# from (COMMUNITY_SCRIPTS_URL in /usr/bin/update); point that at the PR, as -# merging it would. -if [[ "$TEST_MODE" == "upgrade" ]]; then - guest_exec sed -i -E "s#^export COMMUNITY_SCRIPTS_URL=.*#export COMMUNITY_SCRIPTS_URL=\"${base}\"#" /usr/bin/update && - guest_exec grep -qxF "export COMMUNITY_SCRIPTS_URL=\"${base}\"" /usr/bin/update || - setup_error "could not point /usr/bin/update at the PR" - echo "Upgrade test: /usr/bin/update now pulls from ${PR_REPO}@${PR_SHA:0:7}" -fi - -# run_update : runs `update` in the container, live in the -# job log; sets update_rc. -run_update() { - local raw=$1 token - : >"$raw" - token=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n') - echo "──────── $2 (live) ────────" - echo "::stop-commands::${token}" - # timeout runs a program, not a shell function, so spell out the exec here. - if [[ "$PLATFORM" == "incus" ]]; then guest_cmd=(incus exec "$ctid" --); else guest_cmd=(pct exec "$ctid" --); fi - timeout --kill-after=1m "$UPDATE_TIMEOUT" "${guest_cmd[@]}" env PHS_SILENT=1 mode=default MODE=default \ - TERM=dumb DIAGNOSTICS=no var_ignore_breaking_changes=yes var_external_installer=yes ${GH_API_TOKEN:+GITHUB_TOKEN="$GH_API_TOKEN"} update </dev/null \ - > >(tee "$raw" | live_filter) 2>&1 & - update_pid=$! - wait "$update_pid" - update_rc=$? - update_pid="" - sleep 1 - echo "::${token}::" -} -run_update "$UPDATE_RAW" "update test" - -update_failure_line() { - strip_ansi <"${1:-$UPDATE_RAW}" | grep -E '✖|\[ERROR\]|error:' | tail -n 1 | sed -E 's/^\s*(✖️?|\s)*//' | cut -c1-180 -} -if ((update_rc == 124 || update_rc == 137)); then - update_fail "timed out after ${UPDATE_TIMEOUT}" -elif ((update_rc != 0)); then - last=$(update_failure_line) - update_fail "exited ${update_rc}${last:+: $last}" -fi -# The engine can report an error and still exit 0 (bark-server: "github: rate -# limit reached", then "Cleaned"). A rate limit is ours, not the PR's: the -# update could not be tested. Any other error line is a failed update. -update_error=$(strip_ansi <"$UPDATE_RAW" | grep -E '✖|\[ERROR\]' | tail -n 1 | sed -E 's/^\s*(✖️?|\s)*//' | cut -c1-180) -if [[ -n "$update_error" ]]; then - if [[ "${update_error,,}" == *"rate limit"* ]]; then - echo "GitHub API quota now: token $(gh_quota "${GH_API_TOKEN:-}") · no token $(gh_quota "")" - update_status="error" - update_step="could not be tested: ${update_error}" - status="pass" - finish - exit 0 - fi - update_fail "$update_error" -fi -if strip_ansi <"$UPDATE_RAW" | grep -qi 'updated successfully'; then - update_status="updated" - for f in $version_files; do - [[ "$(guest_exec cat "$f" 2>/dev/null)" != "${update_fake[$f]}" ]] || - update_fail "reported success, but $f is still ${update_fake[$f]}" - done -else - # Ran cleanly without updating anything: no version file to fake (apt/npm - # based), or an update_script that decides differently. Not a failure. - update_status="no-update" - update_step=$(strip_ansi <"$UPDATE_RAW" | grep -iE 'no update|up to date|up-to-date|already' | tail -n 1 | sed -E 's/^\s*(✔️?|ℹ️?|\s)*//' | cut -c1-160) -fi - -url=$(app_url "$tree/$SCRIPT") -check_health screenshot-update.png || update_fail "after the update: $health_problem" - -# --- Second update ------------------------------------------------------------------ -# Right after an update, `update` has nothing to do: it must say so and leave -# the app alone. One that breaks on a second run breaks for every user on -# their next update; one that updates again although nothing changed -# rebuilds on every run (often a version file it never writes). -if [[ "$update_status" == "updated" ]]; then - run_update "$UPDATE2_RAW" "second update (must have nothing to do)" - if ((update_rc == 124 || update_rc == 137)); then - update_fail "second run timed out after ${UPDATE_TIMEOUT}" - elif ((update_rc != 0)); then - last=$(update_failure_line "$UPDATE2_RAW") - update_fail "second run exited ${update_rc}${last:+: $last}" - fi - update_error=$(strip_ansi <"$UPDATE2_RAW" | grep -E '✖|\[ERROR\]' | tail -n 1 | sed -E 's/^\s*(✖️?|\s)*//' | cut -c1-180) - if [[ -n "$update_error" && "${update_error,,}" != *"rate limit"* ]]; then - update_fail "second run: $update_error" - fi - if strip_ansi <"$UPDATE2_RAW" | grep -qi 'updated successfully'; then - update2_status="updated-again" - notes+=("a second \`update\` right after the first updated again: update_script does not notice it is current (e.g. a version file it never writes)") - else - update2_status="no-update" - fi - check_health screenshot-update.png || update_fail "after the second update: $health_problem" -fi - -# --- Does it stay up? ----------------------------------------------------------- -# A service that crash-loops can be "active" at every single look. Count the -# restarts of every service (and Docker container) over STABLE_WAIT seconds: -# any that went up was restarted while nobody touched the container. -restart_counts() { - guest_exec sh -c ' - if command -v systemctl >/dev/null; then - systemctl list-units --type=service --all --no-legend --plain | awk "{print \$1}" | - while read -r u; do echo "$u $(systemctl show -p NRestarts --value "$u" 2>/dev/null)"; done - fi - if command -v docker >/dev/null; then - docker ps -aq | xargs -r docker inspect -f "docker:{{.Name}} {{.RestartCount}}" 2>/dev/null - fi' 2>/dev/null | awk '$2 ~ /^[0-9]+$/' | sort -} -before=$(restart_counts) -sleep "$STABLE_WAIT" -after=$(restart_counts) -restarting=$(join <(echo "$before") <(echo "$after") | awk '$3 > $2 {printf "%s%s (+%d)", sep, $1, $3 - $2; sep=", "}') -[[ -z "$restarting" ]] || fail "not stable: restarted within ${STABLE_WAIT}s without anyone touching it: ${restarting}" -check_health screenshot-update.png || fail "not stable: ${STABLE_WAIT}s later, ${health_problem}" - -# Error lines in the journal since boot: often harmless, sometimes the reason -# a feature is broken. Information only. -if guest_exec sh -c 'command -v journalctl' &>/dev/null; then - # Without the noise every container produces: udev may not trigger devices - # in an unprivileged container, and logind finds no VTs. - journal_errors=$(guest_exec journalctl -b -p err -q -o cat --no-pager 2>/dev/null | grep -vE '^\s*$' | - grep -vE "Failed to write 'add' to '/sys/.*/uevent'|Seat has no VTs but VT number not 0" | sort | uniq -c | sort -rn) - if [[ -n "$journal_errors" ]]; then - n=$(awk '{s += $1} END {print s}' <<<"$journal_errors") - top=$(head -n 2 <<<"$journal_errors" | sed -E 's/^\s*[0-9]+\s+//' | cut -c1-120 | paste -sd'|' - | sed 's/|/; /g') - notes+=("${n} error line(s) in the journal since boot, e.g. ${top}") - fi -fi - -status="pass" -finish diff --git a/.github/workflows/scripts/pve-run-vm.sh b/.github/workflows/scripts/pve-run-vm.sh deleted file mode 100644 index 56137a9fd..000000000 --- a/.github/workflows/scripts/pve-run-vm.sh +++ /dev/null @@ -1,140 +0,0 @@ -# shellcheck shell=bash -# The vm/ half of pve-run-script.sh, sourced from it: same inputs, same -# result.json, same live output. Only building and checking differ. -# -# A VM script runs entirely on this host (there is no install script inside -# the guest to follow), unattended through the core engine's VM_UNATTENDED=1: -# the script's defaults, with ID, name, bridge and storage set from here. The -# guest then boots on vmbr1 like the containers do, so its address is whatever -# this host's dnsmasq leased to its MAC; no guest agent needed. -# -# What passes: the script exits 0 and says "Completed successfully", the VM is -# running and keeps running through the boot wait, and where the script prints -# a URL built from the VM's address (http://${VM_IP}:9000) that URL answers 200. -# An installer ISO or an image without DHCP gets no address; that is reported, -# not failed. There is no update test: a VM has no `update`. - -VM_BOOT_WAIT="${VM_BOOT_WAIT:-300}" -VM_URL_WAIT="${VM_URL_WAIT:-600}" -LEASES="${LEASES:-/var/lib/misc/dnsmasq.leases}" - -# One VM build per host at a time: a VM takes 2-8 GB of RAM and a disk image, -# where a container takes a fraction. Held until the job exits; the build is -# started with this descriptor closed, so a QEMU process can never inherit it. -vm_wait_turn() { - exec 7>/run/ci-jobs/.vm-turn - flock -n 7 && return 0 - echo "Another VM build is running on this host; waiting for it (one VM at a time)" - flock -w 3600 7 || setup_error "waited an hour for this host's VM slot" -} - -vm_cleanup() { - if qm status "$ctid" &>/dev/null; then - qm stop "$ctid" --skiplock 1 --timeout 30 &>/dev/null || true - qm destroy "$ctid" --purge 1 --destroy-unreferenced-disks 1 --skiplock 1 &>/dev/null || true - fi - rm -f "/run/ci-jobs/$ctid" "/run/ci-jobs/console-${ctid}.png" - # Cloud-Init network snippets the script wrote for this VM. - [[ -f "$RAW_LOG.start" ]] && - find /var/lib/vz/snippets -maxdepth 1 -type f -name "*${ctid}*" -newer "$RAW_LOG.start" -delete 2>/dev/null - true -} - -# The VM's screen, straight from QEMU. Pixels only, but they come from the -# guest, so the same checks as for a browser screenshot apply. -vm_console_shot() { - local f="/run/ci-jobs/console-${ctid}.png" - rm -f "$f" - printf 'screendump %s -f png\n' "$f" | timeout 30 qm monitor "$ctid" &>/dev/null - if [[ -s "$f" && $(stat -c %s "$f") -lt 3000000 ]] && - [[ "$(head -c 8 "$f" | od -An -tx1 | tr -d ' \n')" == "89504e470d0a1a0a" ]]; then - cp "$f" "$OUT_DIR/screenshot.png" - fi - rm -f "$f" -} - -vm_running() { - qm status "$ctid" 2>/dev/null | grep -q 'status: running' -} - -vm_build_and_check() { - local hn mac pattern waited last - hn="ci-$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}" | cut -c1-40)-${ctid}" - echo "Building ${SCRIPT} as VM ${ctid} from ${PR_REPO}@${PR_SHA:0:7}" - local vm_env=( - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - HOME=/root TERM=dumb LANG=C.UTF-8 DIAGNOSTICS=no - VM_UNATTENDED=1 VM_VMID="$ctid" VM_HOSTNAME="$hn" VM_BRIDGE="$BRIDGE" - VM_STORAGE=local VM_ISO_STORAGE=local VM_START=yes - COMMUNITY_SCRIPTS_URL="$base" - ) - # Release lookups (vm_latest_from_github and the like). - [[ -n "${GH_API_TOKEN:-}" ]] && vm_env+=(GITHUB_TOKEN="$GH_API_TOKEN") - touch "$RAW_LOG.start" - - env -i "${vm_env[@]}" timeout --kill-after=1m "$BUILD_TIMEOUT" \ - bash -c 'bash -c "$(curl -fsSL "$1")"' _ "$base/$SCRIPT" </dev/null >"$RAW_LOG" 2>&1 7>&- & - build_pid=$! - start_streams - wait "$build_pid" - exit_code=$? - build_pid="" - stop_streams - - ((exit_code == 124 || exit_code == 137)) && fail "build timed out after ${BUILD_TIMEOUT}" - if ((exit_code != 0)); then - last=$(failure_line) - step="script exited ${exit_code}${last:+: $last}" - status="fail" - classify_cpu_failure - fail "$step" "$status" - fi - # vm_finish ends on a msg_ok of the script's choosing ("VM created. ..."), - # older scripts on "Completed successfully". exit_script and friends leave - # with 0 too, but on a cross. - last=$(strip_ansi <"$RAW_LOG" | tail -n 1 | cut -c1-200) - if ! grep -qi 'completed successfully' "$RAW_LOG" && ! [[ "$last" =~ ^[[:space:]]*✔ ]]; then - fail "script exited 0 but never reported success${last:+ (last line: ${last})}" - fi - vm_running || fail "VM ${ctid} is not running after the script finished" - - # The address dnsmasq handed to this VM's MAC. The VM must stay up while it - # boots: one that powers itself off or crashes is a failure. - mac=$(qm config "$ctid" 2>/dev/null | sed -nE 's/^net0:[^=]*=([0-9A-Fa-f:]{17}).*/\1/p' | tr 'A-F' 'a-f') - echo "Waiting up to ${VM_BOOT_WAIT}s for VM ${ctid} (${mac:-no NIC}) to boot and get an address" - for ((waited = 0; waited < VM_BOOT_WAIT; waited += 5)); do - vm_running || fail "VM ${ctid} stopped by itself while booting" - [[ -n "$mac" ]] && ct_ip=$(awk -v m="$mac" '$2 == m {print $3}' "$LEASES" 2>/dev/null | tail -n 1) - [[ -n "$ct_ip" ]] && break - sleep 5 - done - [[ -n "$ct_ip" ]] && echo "VM ${ctid} got ${ct_ip}" || echo "VM ${ctid} got no DHCP lease within ${VM_BOOT_WAIT}s" - - # Only a URL the script builds from the address it found; <VM-IP> in a - # summary is a hint for later (after an installer ran), not a promise. - # ${VM_IP}, $VM_IP and ${VM_IP:-<VM-IP>} all count. - local ip_re='\$\{?VM_IP(:-[^}]*)?\}?' - pattern=$(grep -oE "https?://${ip_re}(:[0-9]+)?(/[A-Za-z0-9._~/?=&%-]*)?" "$tree/$SCRIPT" | head -n1) - if [[ -n "$pattern" ]]; then - [[ -n "$ct_ip" ]] || fail "no IP address within ${VM_BOOT_WAIT}s, so ${pattern} could not be checked" - url=$(sed -E "s#${ip_re}#${ct_ip}#" <<<"$pattern") - for ((waited = 0; waited < VM_URL_WAIT; waited += 10)); do - vm_running || fail "VM ${ctid} stopped by itself while its web UI was awaited" - http_code=$(curl -sk -L --max-redirs 5 -o /dev/null -w '%{http_code}' --max-time 10 "$url" || true) - url_ok "$http_code" && break - sleep 10 - done - [[ "$http_code" =~ ^[1-5][0-9][0-9]$ ]] && take_screenshot screenshot.png - else - # Let the console settle on a login prompt rather than a boot message. - sleep 20 - fi - [[ -s "$OUT_DIR/screenshot.png" ]] || vm_console_shot - if [[ -n "$url" ]] && ! url_ok "$http_code"; then - fail "${url} answered ${http_code/#000/nothing} within ${VM_URL_WAIT}s" - fi - vm_running || fail "VM ${ctid} stopped by itself" - - status="pass" - finish -}