From db30bcd960122557211ae39d81c59dc1633c4f8d Mon Sep 17 00:00:00 2001 From: Michel Roegl-Brunner <73236783+michelroegl-brunner@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:04:32 +0100 Subject: [PATCH] Build PR scripts on a Proxmox test host via /test (#18594) A maintainer comments `/test` on a pull request and a self-hosted runner on a disposable Proxmox VE host builds every ct/ script the PR touches (and the ct/ entry point of every changed install/ script), checks the result and reports back on the PR. - Gate: only users with write access; builds the latest commit only if nothing was pushed to the fork after the comment, or a pinned `/test `. - Build: the PR tree at that commit is served from the test host with every raw.githubusercontent.com/community-scripts/ProxmoxVE/main URL rewritten to it, so engine changes in the PR are tested too. Unattended (mode=default); prompts declared the AGENTS.md way are pre-answered with CI values. - Checks: build and install succeed, container running, no failed units, the URL the script prints answers 200. Screenshot of that URL. - Report: check run, one comment (results, screenshot, install log, mentions of the PR author and whoever asked), changes requested on a real failure, dismissed again on a pass. CPU-feature failures (no AVX2 on the test host) are marked untestable, not failed. - The test job has a read-only token and no secrets; its runner carries only the proxmox-test label. Container IDs 9050-9099. Same file as in DevScripts; github.repository selects the URLs and the container ID range. --- .github/workflows/pr-script-test.yml | 475 ++++++++++++++++++ .../workflows/scripts/pve-ci-host-setup.sh | 203 ++++++++ .github/workflows/scripts/pve-run-script.sh | 419 +++++++++++++++ 3 files changed, 1097 insertions(+) create mode 100644 .github/workflows/pr-script-test.yml create mode 100644 .github/workflows/scripts/pve-ci-host-setup.sh create mode 100644 .github/workflows/scripts/pve-run-script.sh diff --git a/.github/workflows/pr-script-test.yml b/.github/workflows/pr-script-test.yml new file mode 100644 index 000000000..1bca56cde --- /dev/null +++ b/.github/workflows/pr-script-test.yml @@ -0,0 +1,475 @@ +name: PR script test + +# Builds the ct/ scripts a pull request touches on a real Proxmox host and +# reports the outcome on the PR, as a check run and as one comment kept up to +# date. +# +# A self-hosted runner that executes pull request code as root is only +# acceptable if a human decides what runs on it. Nothing here starts on a push: +# someone with write access comments `/test` (optionally followed by a full SHA +# to pin, and the ct/ paths to build). Only a commit that existed when the +# comment was written gets built: a push after it is never covered, not even +# one that lands in the seconds before this job runs. +# +# gate ubuntu-latest who asked, which scripts, which commit +# test proxmox-test build + checks, one matrix job per script/OS +# report ubuntu-latest check run + PR comment +# +# The test job gets a read-only token and never sees a secret: the host it runs +# on executes untrusted code and has to be treated as compromised. +# +# This file is identical in DevScripts and ProxmoxVE; github.repository decides +# which URLs get rewritten and which container ID range is used. + +on: + issue_comment: + types: [created] + workflow_dispatch: + inputs: + pr: + description: "Pull request number" + required: true + type: number + sha: + description: "Pin a commit (full 40-character SHA); empty = the latest, if nothing was pushed since" + required: false + type: string + scripts: + description: "Space-separated ct/ paths (empty = everything the PR touches)" + required: false + type: string + +permissions: {} + +jobs: + gate: + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.issue.pull_request && startsWith(github.event.comment.body, '/test')) + runs-on: ubuntu-latest + permissions: + actions: read # when the run was dispatched, for a /test without SHA + checks: write + contents: read + issues: write + pull-requests: write + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + pr: ${{ steps.plan.outputs.pr }} + repo: ${{ steps.plan.outputs.repo }} + sha: ${{ steps.plan.outputs.sha }} + check_id: ${{ steps.plan.outputs.check_id }} + actor: ${{ steps.plan.outputs.actor }} + steps: + - id: plan + uses: actions/github-script@v8 + with: + script: | + const { owner, repo } = context.repo; + const MAX = 10; + core.setOutput('matrix', '[]'); + + let prNumber, requested = ''; + if (context.eventName === 'workflow_dispatch') { + prNumber = Number(context.payload.inputs.pr); + requested = `${context.payload.inputs.sha || ''} ${context.payload.inputs.scripts || ''}`; + } else { + prNumber = context.payload.issue.number; + const body = context.payload.comment.body.trim(); + if (!/^\/test(\s|$)/.test(body)) return; + // author_association hides private org members, so ask for the permission itself. + const user = context.payload.comment.user.login; + const { data: perm } = await github.rest.repos.getCollaboratorPermissionLevel({ owner, repo, username: user }); + if (!['admin', 'maintain', 'write'].includes(perm.permission)) { + core.notice(`${user} has ${perm.permission} access; /test needs write`); + return; + } + await github.rest.reactions.createForIssueComment({ + owner, repo, comment_id: context.payload.comment.id, content: 'rocket', + }); + requested = body.split('\n')[0].slice('/test'.length); + } + + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); + if (pr.state !== 'open' || !pr.head.repo) { core.notice('PR is closed or its fork is gone'); return; } + const head = pr.head.repo; + const MARK = ''; + const sticky = async body => { + const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number: prNumber }); + // Only our own comment: anyone can paste the marker into theirs. + const mine = comments.find(c => c.body.includes(MARK) && c.user.type === 'Bot'); + if (mine) await github.rest.issues.updateComment({ owner, repo, comment_id: mine.id, body }); + else await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body }); + }; + + // Which commit the approval covers. Reading pr.head.sha alone would + // build whatever was pushed between the comment and this job + // starting, i.e. code nobody looked at. Two ways to be sure: + // + // /test that exact commit, and only while it is the head. A + // full SHA: the author writes both commits, so a short + // prefix is a quick birthday collision away. + // /test the latest commit, if nothing reached the fork after + // the comment. pushed_at is set by GitHub on every push + // and read after the PR, so a push in between shows up. + // Branches of this repository belong to people with + // write access, who could build anything anyway. + const tokens = requested.split(/\s+/).filter(Boolean); + const isSha = t => /^[0-9a-f]{7,40}$/.test(t); + const approved = tokens.filter(isSha); + let entries = tokens.filter(t => !isSha(t)); + const refuse = async (why, retry) => { + await sticky([MARK, '### Proxmox script test', '', `⚠️ ${why} Nothing was built.`, '', retry].join('\n')); + core.notice(why); + }; + if (approved.length > 1) { + await refuse('`/test` takes at most one commit.', 'Comment `/test` to build the latest commit.'); + return; + } + if (approved.length === 1 && approved[0] !== pr.head.sha) { + await refuse( + approved[0].length !== 40 && pr.head.sha.startsWith(approved[0]) + ? 'A short SHA is not enough, it has to be all 40 characters.' + : `\`${approved[0]}\` is not the head of this PR (any more).`, + 'Comment `/test` to build the latest commit.'); + return; + } + if (approved.length === 0 && head.full_name !== `${owner}/${repo}`) { + const askedAt = context.eventName === 'workflow_dispatch' + ? (await github.rest.actions.getWorkflowRun({ owner, repo, run_id: context.runId })).data.created_at + : context.payload.comment.created_at; + const { data: fork } = await github.rest.repos.get({ owner: head.owner.login, repo: head.name }); + if (new Date(fork.pushed_at) >= new Date(askedAt)) { + await refuse( + `Something was pushed to ${head.full_name} after the \`/test\`, so it is not certain which commit was meant.`, + 'Comment `/test` again once you have looked at the latest commit.'); + return; + } + } + if (entries.some(e => !/^ct\/[A-Za-z0-9._-]+\.sh$/.test(e))) { + core.setFailed(`Only ct/.sh paths can be requested, got: ${requested}`); + return; + } + if (entries.length === 0) { + const files = await github.paginate(github.rest.pulls.listFiles, { owner, repo, pull_number: prNumber }); + const set = new Set(); + for (const f of files) { + if (f.status === 'removed') continue; + let m; + if ((m = f.filename.match(/^install\/([A-Za-z0-9._-]+)-install\.sh$/))) set.add(`ct/${m[1]}.sh`); + else if (/^ct\/[A-Za-z0-9._-]+\.sh$/.test(f.filename)) set.add(f.filename); + } + entries = [...set].sort(); + } + + const include = []; + for (const script of entries.slice(0, MAX)) { + let text; + try { + const res = await github.rest.repos.getContent({ owner: head.owner.login, repo: head.name, path: script, ref: pr.head.sha }); + text = Buffer.from(res.data.content || '', 'base64').toString('utf8'); + } catch (e) { continue; } // install script without a ct/ entry point + // Scripts that ask for the OS get built once per OS. + const oses = /msg_menu\s+"Choose the container OS"/.test(text) ? ['debian', 'alpine'] : ['']; + const name = script.slice(3, -3); + for (const os of oses) include.push({ script, os, name: os ? `${name}-${os}` : name }); + } + if (entries.length > MAX) core.warning(`Only the first ${MAX} of ${entries.length} scripts are built`); + if (include.length === 0) { core.notice('No ct/ script to build'); return; } + + const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + const { data: check } = await github.rest.checks.create({ + owner, repo, name: 'Proxmox script test', head_sha: pr.head.sha, status: 'in_progress', + details_url: runUrl, + output: { title: `Building ${include.length} on Proxmox`, summary: include.map(i => `- \`${i.script}\` ${i.os}`).join('\n') }, + }); + + core.setOutput('matrix', JSON.stringify(include)); + core.setOutput('pr', String(prNumber)); + core.setOutput('repo', head.full_name); + core.setOutput('sha', pr.head.sha); + core.setOutput('check_id', String(check.id)); + // Who asked: the /test commenter, or whoever dispatched the run. + core.setOutput('actor', context.actor); + + await sticky([ + MARK, '### Proxmox script test', '', + `⏳ Building ${include.length} at ${pr.head.sha.slice(0, 7)} — [watch it live](${runUrl}) (open the \`test\` job)`, '', + ...include.map(i => `- \`${i.script}\`${i.os ? ` (${i.os})` : ''}`), + ].join('\n')); + + test: + needs: gate + if: needs.gate.outputs.matrix != '[]' && needs.gate.outputs.matrix != '' + # Only `proxmox-test`, never `self-hosted`: the runner is registered without + # default labels so that no other workflow can be scheduled onto it. + runs-on: proxmox-test + permissions: + contents: read + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.gate.outputs.matrix) }} + name: test (${{ matrix.name }}) + steps: + # The default branch, never the PR: the harness itself must be trusted. + - uses: actions/checkout@v5 + with: + sparse-checkout: .github/workflows/scripts + persist-credentials: false + + - name: Build and check ${{ matrix.script }} + env: + PR_REPO: ${{ needs.gate.outputs.repo }} + PR_SHA: ${{ needs.gate.outputs.sha }} + SCRIPT: ${{ matrix.script }} + OS: ${{ matrix.os }} + UPSTREAM: ${{ github.repository }} + CTID_BASE: ${{ github.repository == 'community-scripts/ProxmoxVE' && '9050' || '9000' }} + OUT_DIR: ${{ runner.temp }}/result + run: bash .github/workflows/scripts/pve-run-script.sh + + - if: always() + uses: actions/upload-artifact@v4 + with: + name: result-${{ matrix.name }} + path: ${{ runner.temp }}/result + if-no-files-found: ignore + retention-days: 14 + + report: + needs: [gate, test] + if: always() && needs.gate.result == 'success' && needs.gate.outputs.matrix != '[]' && needs.gate.outputs.matrix != '' + runs-on: ubuntu-latest + permissions: + actions: read + checks: write + contents: write # the ci-screenshots branch only; main is protected + issues: write + pull-requests: write + steps: + - uses: actions/download-artifact@v5 + with: + pattern: result-* + path: results + + - uses: actions/github-script@v8 + env: + MATRIX: ${{ needs.gate.outputs.matrix }} + PR: ${{ needs.gate.outputs.pr }} + SHA: ${{ needs.gate.outputs.sha }} + CHECK_ID: ${{ needs.gate.outputs.check_id }} + TEST_RESULT: ${{ needs.test.result }} + ACTOR: ${{ needs.gate.outputs.actor }} + with: + script: | + const fs = require('fs'); + const { owner, repo } = context.repo; + const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + const matrix = JSON.parse(process.env.MATRIX); + + // download-artifact unpacks a single artifact straight into results/ + // and several into results//, so look for every result.json + // and match it by what it says it is. + const path = require('path'); + const found = new Map(); + const walk = dir => { + if (!fs.existsSync(dir)) return; + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, e.name); + if (e.isDirectory()) walk(p); + else if (e.name === 'result.json') { + try { + const r = JSON.parse(fs.readFileSync(p, 'utf8')); + // Written on the host that ran the PR's code: anything but a + // known status is an error, never something new. + if (!['pass', 'fail', 'unsupported', 'error'].includes(r.status)) r.status = 'error'; + const log = path.join(dir, 'install.log'); + r.install_log = fs.existsSync(log) ? fs.readFileSync(log, 'utf8') : ''; + const shot = path.join(dir, 'screenshot.png'); + r.shot_file = fs.existsSync(shot) ? shot : ''; + found.set(`${r.script}|${r.os}`, r); + } catch (err) { core.warning(`unreadable ${p}: ${err.message}`); } + } + } + }; + walk('results'); + + // A job that died before writing result.json still gets a row. + const rows = matrix.map(m => ({ ...(found.get(`${m.script}|${m.os || 'default'}`) || { + script: m.script, os: m.os || 'default', + status: process.env.TEST_RESULT === 'cancelled' ? 'cancelled' : 'error', + step: 'no result (runner or job failure)', duration: 0, log_tail: '', install_log: '', shot_file: '', + }), name: m.name })); + + // Screenshots go to an orphan branch, because a comment can only show + // an image that has a URL. One file per PR and script, overwritten on + // every run; the link names the commit, so no cache shows an old one. + // The PNG comes from the host that ran the PR's code: check it again. + const BRANCH = 'ci-screenshots'; + const prNum = Number(process.env.PR); + const isPng = buf => buf.length > 8 && buf.length < 3000000 && + buf.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])); + const ensureBranch = async () => { + try { await github.rest.git.getRef({ owner, repo, ref: `heads/${BRANCH}` }); return; } + catch (e) { if (e.status !== 404) throw e; } + const { data: blob } = await github.rest.git.createBlob({ owner, repo, encoding: 'utf-8', + content: 'Screenshots taken by pr-script-test.yml for the PR comments. Not code; never merge this branch.\n' }); + const { data: tree } = await github.rest.git.createTree({ owner, repo, tree: [{ path: 'README.md', mode: '100644', type: 'blob', sha: blob.sha }] }); + const { data: commit } = await github.rest.git.createCommit({ owner, repo, message: 'Start ci-screenshots', tree: tree.sha, parents: [] }); + await github.rest.git.createRef({ owner, repo, ref: `refs/heads/${BRANCH}`, sha: commit.sha }); + }; + for (const r of rows.filter(r => r.shot_file)) { + const buf = fs.readFileSync(r.shot_file); + if (!isPng(buf)) { core.warning(`${r.script}: screenshot is not a plausible PNG, skipped`); continue; } + const file = `pr-${prNum}/${r.name}.png`; + for (let attempt = 1; attempt <= 3 && !r.shot_url; attempt++) { + try { + await ensureBranch(); + let sha; + try { sha = (await github.rest.repos.getContent({ owner, repo, path: file, ref: BRANCH })).data.sha; } + catch (e) { if (e.status !== 404) throw e; } + const { data } = await github.rest.repos.createOrUpdateFileContents({ + owner, repo, branch: BRANCH, path: file, sha, content: buf.toString('base64'), + message: `Screenshot: ${r.script} (${r.os}) for #${prNum} at ${process.env.SHA.slice(0, 7)}`, + }); + r.shot_url = `https://raw.githubusercontent.com/${owner}/${repo}/${data.commit.sha}/${file}`; + } catch (e) { + // Two reports at once race on the branch head; try again. + core.warning(`${r.script}: screenshot upload attempt ${attempt} failed: ${e.message}`); + await new Promise(res => setTimeout(res, 2000 * attempt)); + } + } + } + + // unsupported: the script needs a CPU feature the test host lacks + // (AVX2 and the like). Not the PR's fault, so not a failure. + const icon = { pass: '✅', fail: '❌', error: '⚠️', cancelled: '⏹️', unsupported: '🖥️' }; + const dur = s => `${Math.floor(s / 60)}m${String(s % 60).padStart(2, '0')}s`; + // Text from the build lands outside code blocks here: no table breaks, + // no code spans, and no @mention of a stranger (zero-width space). + const clean = s => String(s || '').replace(/[|\r\n]/g, ' ').replace(/`/g, "'").replace(/@/g, '@​').slice(0, 200); + const fence = s => String(s || '').replace(/~~~~/g, '~~~'); + + // fail: the app broke, the PR's business. error/cancelled: our test + // setup did, not the PR's. unsupported: the host's CPU is too old. + const count = s => rows.filter(r => r.status === s).length; + const passed = count('pass'); + const untestable = count('unsupported'); + const broken = count('fail'); + const setupTrouble = count('error') + count('cancelled'); + const allPass = passed === rows.length; + const cancelled = rows.every(r => r.status === 'cancelled'); + const of = `**${passed}/${rows.length} passed**`; + const headline = broken + ? `❌ ${of}` + : setupTrouble + ? `⚠️ ${of}, ${setupTrouble} could not be tested because of a problem with the test setup (not the PR)` + : untestable + ? `🖥️ ${of}, ${untestable} need${untestable === 1 ? 's' : ''} a newer CPU than the test host has (nothing failed)` + : `✅ ${of}`; + + const table = [ + '| Script | OS | Result | Duration | Detail |', + '|---|---|---|---|---|', + ...rows.map(r => `| \`${r.script}\` | ${r.os} | ${icon[r.status] || '❔'} | ${dur(r.duration || 0)} | ${r.status === 'pass' ? (r.url ? `${clean(r.url)} → ${r.http_code}` : 'no URL to check') : clean(r.step)} |`), + ].join('\n'); + + // A comment holds 65536 characters. Each script gets an equal share + // and a log that does not fit loses its beginning: the end is where + // a failure shows. + const budget = Math.floor(52000 / rows.length); + const tail = (text, max) => { + text = fence(text).trimEnd(); + if (text.length <= max) return text; + const kept = text.slice(text.length - max); + const cut = text.slice(0, text.length - max).split('\n').length; + return `… ${cut} earlier lines cut, the full log is in the run artifacts\n${kept.slice(kept.indexOf('\n') + 1)}`; + }; + const details = (summaryLine, text, max) => [ + `
${summaryLine}`, '', + '~~~~text', tail(text, max), '~~~~', '
', + ].join('\n'); + + const logs = rows.map(r => { + const parts = []; + const failed = r.status !== 'pass'; + const buildShare = failed && r.log_tail ? Math.min(r.log_tail.length, Math.floor(budget / 3)) : 0; + if (buildShare) parts.push(details(`${icon[r.status] || '❔'} ${r.script} (${r.os}) — build log`, r.log_tail, buildShare)); + if (r.install_log) { + const lines = r.install_log.trimEnd().split('\n').length; + parts.push(details(`📄 ${r.script} (${r.os}) — install log (${lines} lines)`, r.install_log, budget - buildShare - 300)); + } else if (!buildShare) { + parts.push(`${r.script} (${r.os}): no install log was captured`); + } + return parts.join('\n\n'); + }); + + const shots = rows.filter(r => r.shot_url).map(r => [ + `
📸 ${r.script} (${r.os}) — ${clean(r.url)}`, '', + `Screenshot of ${r.script}`, '', '
', + ].join('\n')); + + const summary = [ + `${headline} at \`${process.env.SHA.slice(0, 7)}\``, '', + table, '', + ...shots, '', + ...logs, '', + `Full logs: [workflow run](${runUrl}) (artifacts \`result-*\`). Comment \`/test\` to build again.`, + ].join('\n'); + + await github.rest.checks.update({ + owner, repo, check_run_id: Number(process.env.CHECK_ID), status: 'completed', + // Untestable here is "test it by hand", not a pass: action_required, + // because a required check would count neutral as passed. + conclusion: cancelled ? 'cancelled' : allPass ? 'success' : broken ? 'failure' : 'action_required', + output: { title: `${passed}/${rows.length} passed`, summary: summary.slice(0, 65000) }, + }); + + // The result pings whoever started the test. A mention added by + // editing a comment does not reliably notify, so the old comment + // goes and the result is posted as a new one, at the end of the + // thread where it is looked for. + // Whoever started the test and the PR's author, once each; no bots. + const pr = Number(process.env.PR); + const { data: prInfo } = await github.rest.pulls.get({ owner, repo, pull_number: pr }); + const author = prInfo.user && prInfo.user.type === 'User' ? prInfo.user.login : ''; + const login = /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$/; + const ping = [...new Set([process.env.ACTOR || '', author])].filter(l => login.test(l)).map(l => `@${l} `).join(''); + const MARK = ''; + const body = `${MARK}\n### Proxmox script test\n\n${ping}${summary}`.slice(0, 65000); + const comments = await github.paginate(github.rest.issues.listComments, { owner, repo, issue_number: pr }); + for (const c of comments.filter(c => c.body.includes(MARK) && c.user.type === 'Bot')) { + await github.rest.issues.deleteComment({ owner, repo, comment_id: c.id }); + } + const { data: posted } = await github.rest.issues.createComment({ owner, repo, issue_number: pr, body }); + + // A real failure requests changes on the tested commit. Untestable + // (CPU), runner errors and cancelled runs are not the PR's fault and + // leave the review state alone. Our previous "changes requested" is + // always dismissed first: a pass lifts it, a new failure replaces it. + const REVIEW_MARK = ''; + const failedRows = cancelled ? [] : rows.filter(r => r.status === 'fail'); + const sha7 = process.env.SHA.slice(0, 7); + try { + const reviews = await github.paginate(github.rest.pulls.listReviews, { owner, repo, pull_number: pr }); + for (const v of reviews.filter(v => v.user && v.user.type === 'Bot' && v.state === 'CHANGES_REQUESTED' && (v.body || '').includes(REVIEW_MARK))) { + await github.rest.pulls.dismissReview({ + owner, repo, pull_number: pr, review_id: v.id, + message: failedRows.length ? 'Replaced by a newer Proxmox script test.' : `Proxmox script test passed at ${sha7}.`, + }); + } + if (failedRows.length) { + await github.rest.pulls.createReview({ + owner, repo, pull_number: pr, commit_id: process.env.SHA, event: 'REQUEST_CHANGES', + body: [ + REVIEW_MARK, `**Proxmox script test failed** at \`${sha7}\`:`, '', + ...failedRows.map(r => `- \`${r.script}\` (${r.os}): ${clean(r.step)}`), '', + `Logs and details: [test result](${posted.html_url}). Push a fix, then ask for \`/test\` again.`, + ].join('\n'), + }); + } + } catch (e) { + core.warning(`Could not update the review state: ${e.message}`); + } diff --git a/.github/workflows/scripts/pve-ci-host-setup.sh b/.github/workflows/scripts/pve-ci-host-setup.sh new file mode 100644 index 000000000..e9485d3dc --- /dev/null +++ b/.github/workflows/scripts/pve-ci-host-setup.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash +# One-time setup of the Proxmox test host behind pr-script-test.yml. +# Safe to re-run. Run as root on the test host (a disposable PVE, never a +# production node: it builds pull request code as root). +# +# pve-ci-host-setup.sh host only +# pve-ci-host-setup.sh host + a runner for that repo +# +# The token is a runner *registration* token (valid one hour): +# gh api -X POST repos//actions/runners/registration-token --jq .token + +set -euo pipefail + +RUNNER_VERSION="${RUNNER_VERSION:-2.338.2}" +SERVE_ROOT=/srv/ci +SERVE_PORT=18080 +UPLINK="${UPLINK:-vmbr0}" + +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq jq curl python3 dnsmasq iptables chromium fonts-dejavu fonts-noto-color-emoji >/dev/null + +# Screenshots of the app under test are taken as this user, never as root. +id ci-shot &>/dev/null || useradd --system --create-home --shell /usr/sbin/nologin ci-shot + +# The page Chromium renders is served by code under test, and a redirect or an +# could point it at this host (the PVE UI on :8006) or the lab network; +# the screenshot would then publish that on the PR. So ci-shot may reach the +# test containers, DNS and the public internet (apps load fonts and scripts +# from CDNs), and nothing private. +cat >/usr/local/sbin/ci-shot-firewall <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +uid=$(id -u ci-shot) +for fw in iptables ip6tables; do + $fw -w -N CI_SHOT 2>/dev/null || $fw -w -F CI_SHOT + $fw -w -C OUTPUT -m owner --uid-owner "$uid" -j CI_SHOT 2>/dev/null || + $fw -w -I OUTPUT -m owner --uid-owner "$uid" -j CI_SHOT +done +iptables -w -A CI_SHOT -d 10.99.0.1 -j REJECT +iptables -w -A CI_SHOT -d 10.99.0.0/24 -j ACCEPT +for ns in $(awk '/^nameserver/ && $2 !~ /:/ {print $2}' /etc/resolv.conf); do + iptables -w -A CI_SHOT -d "$ns" -p udp --dport 53 -j ACCEPT + iptables -w -A CI_SHOT -d "$ns" -p tcp --dport 53 -j ACCEPT +done +for net in 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.168.0.0/16 224.0.0.0/3; do + iptables -w -A CI_SHOT -d "$net" -j REJECT +done +for net in ::1/128 fc00::/7 fe80::/10 ff00::/8; do + ip6tables -w -A CI_SHOT -d "$net" -j REJECT +done +EOF +chmod +x /usr/local/sbin/ci-shot-firewall +cat >/etc/systemd/system/ci-shot-firewall.service <<'EOF' +[Unit] +Description=Keep the screenshot browser away from this host and the lab network +After=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/ci-shot-firewall + +[Install] +WantedBy=multi-user.target +EOF + +# Containers get their own NATed bridge instead of the lab network: no DHCP +# lease surprises (a lab lease once handed a container the host's own address), +# and an app listening on 8080 can never be confused with this host. +cat >/etc/network/interfaces.d/ci-nat </dev/null || iptables -t nat -A POSTROUTING -s 10.99.0.0/24 -o ${UPLINK} -j MASQUERADE + post-down iptables -t nat -D POSTROUTING -s 10.99.0.0/24 -o ${UPLINK} -j MASQUERADE || true +EOF +ifreload -a +cat >/etc/dnsmasq.d/ci.conf <<'EOF' +interface=vmbr1 +bind-interfaces +except-interface=lo +dhcp-range=10.99.0.100,10.99.0.250,255.255.255.0,2h +dhcp-option=option:router,10.99.0.1 +dhcp-option=option:dns-server,10.99.0.1 +no-resolv +EOF +# Upstream = whatever this host resolves with. Debian's dnsmasq would otherwise +# ask resolvconf, which has nothing to offer on a PVE host. +awk '/^nameserver/ {print "server=" $2}' /etc/resolv.conf >>/etc/dnsmasq.d/ci.conf +systemctl enable dnsmasq >/dev/null +systemctl restart dnsmasq + +# A PVE ISO install writes /etc/timezone; a cloud-image install does not, and +# the engine's pct_timezone aborts without it. +[[ -s /etc/timezone ]] || { + timedatectl set-timezone Europe/Vienna + echo Europe/Vienna >/etc/timezone +} + +# Containers live on the root filesystem; there is no LVM in a cloud-image PVE. +pvesm set local --content iso,vztmpl,backup,rootdir,images + +# Templates are fetched once here rather than on every first build. +pveam update >/dev/null +for t in $(pveam available --section system | awk '/_amd64\./ && /debian-13-standard|alpine-3\.2[0-9]-default|ubuntu-24\.04-standard/ {print $2}'); do + pveam list local | grep -q "$t" || pveam download local "$t" >/dev/null +done + +# The rewritten PR trees, served to the engine and to the containers. +mkdir -p "$SERVE_ROOT" +cat >/etc/systemd/system/ci-serve.service </usr/local/sbin/ci-sweep <<'EOF' +#!/usr/bin/env bash +# Destroy CI containers (9000-9099) that no running job holds, and stale trees. +for id in $(pct list | awk 'NR>1 && $1>=9000 && $1<=9099 {print $1}'); do + [[ -e "/run/ci-jobs/$id" ]] && continue + pct stop "$id" --skiplock 1 &>/dev/null || true + pct destroy "$id" --purge 1 --force 1 &>/dev/null && echo "swept CT $id" +done +find /srv/ci -mindepth 1 -maxdepth 1 -type d -mmin +180 -exec rm -rf {} + +# Engine logs that a killed job never got to remove. +find /tmp -maxdepth 1 -type f \( -name 'create-lxc-*.log' -o -name 'pct_create_*.log' \ + -o -regex '/tmp/.*-9[0-9][0-9][0-9]-[0-9a-f]+\.log' \) -mmin +180 -delete +find /var/log/community-scripts -type f -mtime +2 -delete 2>/dev/null +find /opt/actions-runner/*/_diag -type f -mtime +14 -delete 2>/dev/null +true +EOF +chmod +x /usr/local/sbin/ci-sweep +cat >/etc/systemd/system/ci-sweep.service <<'EOF' +[Unit] +Description=Remove leftover CI containers + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ci-sweep +EOF +cat >/etc/systemd/system/ci-sweep.timer <<'EOF' +[Unit] +Description=Remove leftover CI containers every 3 hours + +[Timer] +OnCalendar=0/3:30 +Persistent=true + +[Install] +WantedBy=timers.target +EOF + +systemctl daemon-reload +systemctl enable --now ci-serve.service ci-sweep.timer ci-shot-firewall.service >/dev/null +systemctl restart ci-serve.service ci-shot-firewall.service + +# --- Runner (one per repository) ---------------------------------------------- +if [[ $# -eq 2 ]]; then + repo="$1" + token="$2" + [[ "$repo" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad repo" >&2; exit 2; } + dir="/opt/actions-runner/${repo#*/}" + mkdir -p "$dir" + cd "$dir" + if [[ ! -x ./config.sh ]]; then + curl -fsSL "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" | tar -xz + ./bin/installdependencies.sh >/dev/null + fi + # A registered runner is reconfigured from scratch: stop and remove the old + # registration (the token works for removal too). + if [[ -f .runner ]]; then + ./svc.sh stop >/dev/null 2>&1 || true + ./svc.sh uninstall >/dev/null 2>&1 || true + RUNNER_ALLOW_RUNASROOT=1 ./config.sh remove --token "$token" >/dev/null 2>&1 || true + fi + # Root, because pct needs it; the host is disposable by design. + # --no-default-labels: without it the runner also answers to `self-hosted`, + # and any other workflow with `runs-on: self-hosted` (and its secrets) could + # land on this host. + RUNNER_ALLOW_RUNASROOT=1 ./config.sh --unattended --replace --no-default-labels \ + --url "https://github.com/${repo}" --token "$token" \ + --name "pve-ci-${repo#*/}" --labels proxmox-test --work _work + ./svc.sh install root >/dev/null + ./svc.sh start >/dev/null + echo "runner for ${repo} registered" +fi + +echo "test host ready: ${SERVE_ROOT} on :${SERVE_PORT}, sweeper every 3h" diff --git a/.github/workflows/scripts/pve-run-script.sh b/.github/workflows/scripts/pve-run-script.sh new file mode 100644 index 000000000..622653e6d --- /dev/null +++ b/.github/workflows/scripts/pve-run-script.sh @@ -0,0 +1,419 @@ +#!/usr/bin/env bash +# Build one ct/ script from a pull request on this Proxmox host, check that the +# container really works, destroy it, and leave a result.json behind. +# +# Runs on the self-hosted runner, which *is* the disposable test host. The +# harness comes from the default branch; only the script under test comes from +# the pull request, pinned to the commit a maintainer approved. +# +# The engine hardcodes raw.githubusercontent.com//main for every fetch +# (ct -> build.func -> install script -> install.func -> tools.func). Pointing it +# at a fork branch therefore means rewriting those URLs: the PR tree is unpacked +# under SERVE_ROOT, every upstream main URL in it is replaced by this host's own +# HTTP server, and the containers fetch from there. Engine changes in the PR are +# tested along with the script, which is what a reviewer wants anyway. + +set -uo pipefail + +: "${PR_REPO:?}" "${PR_SHA:?}" "${SCRIPT:?}" "${UPSTREAM:?}" "${CTID_BASE:?}" "${OUT_DIR:?}" +OS="${OS:-}" +CTID_COUNT="${CTID_COUNT:-50}" +SERVE_ROOT="${SERVE_ROOT:-/srv/ci}" +SERVE_URL="${SERVE_URL:-http://10.99.0.1:18080}" +BRIDGE="${BRIDGE:-vmbr1}" +BUILD_TIMEOUT="${BUILD_TIMEOUT:-45m}" +URL_WAIT="${URL_WAIT:-120}" + +# Every value below reaches a path, a URL or a command line. +[[ "$PR_REPO" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad PR_REPO" >&2; exit 2; } +[[ "$UPSTREAM" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "bad UPSTREAM" >&2; exit 2; } +[[ "$PR_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "bad PR_SHA" >&2; exit 2; } +[[ "$SCRIPT" =~ ^ct/[A-Za-z0-9._-]+\.sh$ ]] || { echo "bad SCRIPT" >&2; exit 2; } +[[ -z "$OS" || "$OS" =~ ^(debian|ubuntu|alpine)$ ]] || { echo "bad OS" >&2; exit 2; } +[[ "$CTID_BASE" =~ ^[0-9]+$ ]] || { echo "bad CTID_BASE" >&2; exit 2; } + +mkdir -p "$OUT_DIR" +RAW_LOG="$OUT_DIR/build.raw.log" +LOG="$OUT_DIR/build.log" +RESULT="$OUT_DIR/result.json" +INSTALL_LOG_OUT="$OUT_DIR/install.log" + +app=$(basename "$SCRIPT" .sh) +job="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${app}-${OS:-default}" +tree="$SERVE_ROOT/$job" +base="$SERVE_URL/$job" +started=$(date +%s) + +status="fail" +step="" +exit_code="" +ctid="" +ct_ip="" +url="" +http_code="" +failed_units="" + +strip_ansi() { + sed -E 's/\x1B\[[0-9;?]*[A-Za-z]//g; s/\x1B[()][A-Z0-9]//g; s/\r/\n/g' | grep -vE '^\s*$' +} + +# The install script's own output: everything $STD hides from the build log. +# Both engines write it inside the container as /root/.install-.log; +# dev_mode=keep makes sure a failed container is still there to read it from. +# If it is gone anyway, the copy the engine leaves on the host is the fallback. +collect_install_log() { + [[ -n "$ctid" ]] || return 0 + # Exactly one build ran in this container, so the newest install log is ours. + if pct status "$ctid" &>/dev/null && + pct exec "$ctid" -- sh -c 'cat "$(ls -t /root/.install-*.log 2>/dev/null | head -n1)"' 2>/dev/null | + strip_ansi >"$INSTALL_LOG_OUT" && [[ -s "$INSTALL_LOG_OUT" ]]; then + return 0 + fi + # The engine names its host-side files after the session: pct_create___.log + local sid f + sid=$(find /tmp -maxdepth 1 -name "pct_create_${ctid}_*.log" -newer "$RAW_LOG.start" -printf '%f\n' 2>/dev/null | + sed -E 's/.*_([0-9a-f]+)\.log$/\1/' | head -n1) + [[ -n "$sid" ]] || { rm -f "$INSTALL_LOG_OUT"; return 0; } + for f in /tmp/*-"${ctid}"-"${sid}".log /var/log/community-scripts/install-"${sid}"-combined-*.log; do + [[ -s "$f" ]] && { strip_ansi <"$f" >"$INSTALL_LOG_OUT"; return 0; } + done + rm -f "$INSTALL_LOG_OUT" +} + +# "Requires AVX2" and friends: the script may be fine, the test host's CPU is +# not (Xeon E5-2650, Sandy Bridge: AVX, no AVX2). Shown as untestable rather +# than as a failure of the PR -- but the log is written by the code under test, +# so its word is not enough. Only the engine's error lines count, only when the +# build or install itself failed, and the feature they name must really be +# missing from this host's /proc/cpuinfo. Claiming a feature the host has +# changes nothing. +CPU_RE='(requir|need)[a-z]*[^.]{0,60}(AVX(2|-?512)?|SSE4[._]?2|x86-64-v[234])\b|\b(AVX(2|-?512)?|x86-64-v[234])\b[^.]{0,60}(not (supported|available)|does not have|is missing|unsupported)' + +classify_cpu_failure() { + local line feature flags + line=$(strip_ansi <"$RAW_LOG" | grep -E '✖|\[ERROR\]' | grep -m1 -iE "$CPU_RE" | + sed -E 's/^\s*(✖️?|\[[^]]*\]|\s)*//' | cut -c1-200) + [[ -n "$line" ]] || return 0 + # The CPU flag that would have to be missing for the message to be true. + case "${line,,}" in + *avx-512* | *avx512* | *x86-64-v4*) feature=avx512f ;; + *avx2* | *x86-64-v3*) feature=avx2 ;; + *sse4.2* | *sse4_2* | *x86-64-v2*) feature=sse4_2 ;; + *) return 0 ;; + esac + flags=" $(grep -m1 '^flags' /proc/cpuinfo | cut -d: -f2) " + [[ "$flags" == *" ${feature} "* ]] && return 0 + status="unsupported" + step="needs a newer CPU than the test host has (no ${feature}): ${line}" +} + +finish() { + [[ -f "$RAW_LOG" ]] && strip_ansi <"$RAW_LOG" >"$LOG" + [[ -f "$RAW_LOG" ]] && collect_install_log + jq -n \ + --arg script "$SCRIPT" --arg os "${OS:-default}" --arg status "$status" \ + --arg step "$step" --arg exit_code "$exit_code" --arg sha "$PR_SHA" \ + --arg ctid "$ctid" --arg url "$url" --arg http_code "$http_code" \ + --arg failed_units "$failed_units" \ + --argjson duration "$(($(date +%s) - started))" \ + --arg tail "$([[ -f "$LOG" ]] && tail -n 60 "$LOG")" \ + '{script: $script, os: $os, status: $status, step: $step, exit_code: $exit_code, + sha: $sha, ctid: $ctid, url: $url, http_code: $http_code, + failed_units: $failed_units, duration: $duration, log_tail: $tail}' >"$RESULT" + echo "::group::result" + jq 'del(.log_tail)' "$RESULT" + echo "::endgroup::" +} + +# --- Live output ------------------------------------------------------------------ +# The job log on GitHub streams while the job runs, so mirroring the build there +# is all "live" takes. Two streams: the engine's own output, and the install +# log inside the container, which holds everything $STD hides (apt, npm, ...). +# +# Both are written by the code under test. A line starting with `::` would be +# read by the runner as a workflow command, so command processing is switched +# off for as long as they run, with a token the build cannot know. +stream_token="" +log_pid="" +install_pid="" + +# Colours stay, cursor movement and spinner redraws go. +live_filter() { + sed -u -E 's/\r/\n/g; s/\x1B\[[0-9;?]*[ABCDEFGHJKSTfhilnsu]//g' | sed -u -E '/^\s*$/d' +} + +start_streams() { + stream_token=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n') + echo "::stop-commands::${stream_token}" + tail -n +1 -F --pid="$build_pid" "$RAW_LOG" 2>/dev/null > >(live_filter) & + log_pid=$! + # setsid: its own process group, so stop_streams can take down the + # pct exec / tail pair inside it in one go. + setsid bash -c ' + for _ in $(seq 1 900); do + f=$(pct exec "$1" -- sh -c "ls -t /root/.install-*.log 2>/dev/null | head -n1" 2>/dev/null) + if [ -n "$f" ]; then + echo "──────── install log of CT $1 (live) ────────" + exec pct exec "$1" -- tail -n +1 -F "$f" + fi + sleep 2 + done' _ "$ctid" 2>/dev/null > >(live_filter | sed -u 's/^/ │ /') & + install_pid=$! +} + +stop_streams() { + [[ -n "$stream_token" ]] || return 0 + [[ -n "$install_pid" ]] && kill -TERM -- "-$install_pid" &>/dev/null + [[ -n "$log_pid" ]] && kill -TERM "$log_pid" &>/dev/null + wait "$log_pid" "$install_pid" &>/dev/null + sleep 1 # let the filters flush + install_pid="" + log_pid="" + echo "::${stream_token}::" + stream_token="" +} + +build_pid="" +cleanup() { + stop_streams + if [[ -n "$build_pid" ]]; then + pkill -TERM -P "$build_pid" &>/dev/null || true + kill -TERM "$build_pid" &>/dev/null || true + sleep 2 + fi + if [[ -n "$ctid" ]]; then + if pct status "$ctid" &>/dev/null; then + pct stop "$ctid" --skiplock 1 &>/dev/null || true + pct destroy "$ctid" --purge 1 --force 1 &>/dev/null || true + fi + rm -f "/run/ci-jobs/$ctid" + # What the engine leaves on the host for this container and its session. + local f sid + for f in /tmp/pct_create_"${ctid}"_*.log; do + [[ -f "$f" && "$f" -nt "$RAW_LOG.start" ]] || continue + sid=$(sed -E 's/.*_([0-9a-f]+)\.log$/\1/' <<<"$f") + rm -f -- "$f" /tmp/create-lxc-"${sid}".log /tmp/*-"${ctid}"-"${sid}".log + done + fi + rm -f "$RAW_LOG.start" + rm -rf -- "$tree" +} +trap cleanup EXIT +# A cancelled job sends TERM; without this the EXIT trap would not run. +trap 'exit 143' TERM +trap 'exit 130' INT + +# fail [fail|unsupported]: only the build/install branches pass the +# second argument, after classify_cpu_failure has checked the host. +# +# The exit code says whether *this job* worked, not whether the app did. A +# script that fails to build is a result: it is written down, reported on the +# PR (check, comment, review) and the job exits 0 -- a red run per broken app +# would only be notification noise. Only trouble with the test setup itself +# (setup_error) fails the job, because that is ours to look at. +fail() { + step="$1" + status="${2:-fail}" + [[ "$status" == "unsupported" ]] || status="fail" + finish + echo "::warning title=${SCRIPT}::${step}" + exit 0 +} + +setup_error() { + step="$1" + status="error" + finish + echo "::error title=${SCRIPT}::test setup: ${step}" + exit 1 +} + +# --- Reserve a container ID --------------------------------------------------- +# A lock file per ID, not just "first free one", because the engine only creates +# the container minutes later and two jobs would otherwise pick the same ID. +mkdir -p /run/ci-jobs +exec 9>/run/ci-jobs/.lock +flock 9 +for ((id = CTID_BASE; id < CTID_BASE + CTID_COUNT; id++)); do + [[ -e "/run/ci-jobs/$id" ]] && continue + pvesh get /cluster/nextid --vmid "$id" &>/dev/null || continue + echo "$job" >"/run/ci-jobs/$id" + ctid=$id + break +done +flock -u 9 +exec 9>&- +[[ -n "$ctid" ]] || setup_error "no free container ID in ${CTID_BASE}-$((CTID_BASE + CTID_COUNT - 1))" + +# --- Fetch the approved commit and point it at ourselves ------------------------ +mkdir -p "$tree" +curl -fsSL --retry 3 "https://codeload.github.com/${PR_REPO}/tar.gz/${PR_SHA}" | + tar -xz -C "$tree" --strip-components=1 || setup_error "could not download ${PR_REPO}@${PR_SHA:0:7}" +[[ -f "$tree/$SCRIPT" ]] || fail "${SCRIPT} does not exist at ${PR_SHA:0:7}" + +upstream_re="https://raw\.githubusercontent\.com/${UPSTREAM//./\\.}/(refs/heads/)?main" +grep -rlE "$upstream_re" "$tree" --include='*.sh' --include='*.func' | + xargs -r sed -i -E "s#${upstream_re}#${base}#g" + +# --- Build ---------------------------------------------------------------------- +echo "Building ${SCRIPT} (${OS:-default OS}) as CT ${ctid} from ${PR_REPO}@${PR_SHA:0:7}" +hostname_safe="ci-$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}" | cut -c1-50)" +build_env=( + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + HOME=/root TERM=dumb LANG=C.UTF-8 + mode=default MODE=default PHS_SILENT=1 DIAGNOSTICS=no + var_ctid="$ctid" var_hostname="$hostname_safe" + var_container_storage=local var_template_storage=local var_brg="$BRIDGE" + "dev_mode=timing,keep" COMMUNITY_SCRIPTS_URL="$base" +) +touch "$RAW_LOG.start" +[[ -n "$OS" ]] && build_env+=(var_os="$OS") + +# --- Answers for the script's questions ------------------------------------------ +# Nobody types into CI, so anything the install asks has to be answered up front. +# AGENTS.md rule 24: a script reads var_ first and prompts only when it is +# unset (or uses prompt_input_required ... "var_x"). Every such variable gets a +# CI value here. The names come from json/.json app_vars (DevScripts) and +# from the read/prompt_ lines of the install script (both repos). A bare `read` +# that ignores its variable still fails, and the report says why. +ENGINE_VARS=" var_ctid var_hostname var_os var_version var_cpu var_ram var_disk var_brg var_net var_container_storage var_template_storage var_unprivileged var_tags var_arm64 var_testurl var_verbose var_timezone var_http_proxy var_http_no_proxy var_appversion " +host_app=$(tr -c 'a-z0-9-\n' '-' <<<"${app,,}") +ci_value() { + case "${1,,}" in + *pass* | *secret* | *token* | *key*) head -c 32 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | cut -c1-20 ;; + *email* | *mail*) echo "ci@example.test" ;; + *url* | *endpoint*) echo "https://ci-${host_app}.example.test" ;; + *host* | *domain* | *fqdn*) echo "ci-${host_app}.example.test" ;; + *user*) echo "ciadmin" ;; + *) echo "ci-test" ;; + esac +} +declare -A answers=() +json="$tree/json/${app}.json" +if [[ -f "$json" ]]; then + while IFS=$'\t' read -r name default; do + answers[$name]="$default" + done < <(jq -r '(if type == "array" then .[0] else . end).app_vars // [] | .[] | + select(.name | test("^var_[a-z0-9_]{1,40}$")) | [.name, (.default // "" | tostring)] | @tsv' "$json" 2>/dev/null) +fi +# The engine names the install script after APP (lower case, no spaces), which +# need not match the ct/ file name: ct/bluesky-pds.sh -> blueskypds-install.sh. +nsapp=$(grep -m1 -oE '^APP="[^"]+"' "$tree/$SCRIPT" | cut -d'"' -f2 | tr -d ' ' | tr '[:upper:]' '[:lower:]') +for f in "$tree/install/${app}-install.sh" "$tree/install/${nsapp}-install.sh" "$tree/$SCRIPT"; do + [[ -f "$f" ]] || continue + while read -r name; do + [[ -v "answers[$name]" ]] || answers[$name]="" + done < <(grep -E '\bread\b|prompt_(input|input_required|confirm|select|password)\b' "$f" | grep -oE '\bvar_[a-z0-9_]{1,40}\b' | sort -u) +done +for name in "${!answers[@]}"; do + [[ "$ENGINE_VARS" == *" $name "* ]] && continue + value="${answers[$name]}" + [[ -n "$value" ]] || value=$(ci_value "$name") + build_env+=("$name=$value") + case "${name,,}" in + *pass* | *secret* | *token* | *key*) echo "Pre-answered ${name}=(random)" ;; + *) echo "Pre-answered ${name}=${value}" ;; + esac +done + +# In the background so that a TERM from a cancelled job is handled at once: +# bash defers traps until a foreground command returns. +env -i "${build_env[@]}" timeout --kill-after=1m "$BUILD_TIMEOUT" \ + bash -c 'bash -c "$(curl -fsSL "$1")"' _ "$base/$SCRIPT" "$RAW_LOG" 2>&1 & +build_pid=$! +start_streams +wait "$build_pid" +exit_code=$? +build_pid="" +stop_streams + +if ((exit_code == 124 || exit_code == 137)); then + fail "build timed out after ${BUILD_TIMEOUT}" +fi +# The engine marks the step that broke with a cross. Its closing "Installation +# failed in container" line is generic, so the cross before it is the useful one. +failure_line() { + local line + line=$(strip_ansi <"$RAW_LOG" | grep -E '✖|\[ERROR\]|error:' | grep -v 'Installation failed in container' | + tail -n 1 | sed -E 's/^\s*(✖️?|\s)*//' | cut -c1-200) + # A prompt nobody can answer: say what to do instead. + # First, because the report keeps 200 characters. + if [[ "$line" == *"read exited with status"* || "$line" == *"executing command read "* ]]; then + local asked + asked=$(grep -oE '\bvar_[a-z0-9_]{1,40}\b' <<<"$line" | tail -n 1) + line="unanswerable prompt (AGENTS.md rule 24): ask only when ${asked:-var_} is unset, or use prompt_input_required. ${line}" + fi + echo "$line" +} +if ((exit_code != 0)); then + last=$(failure_line) + step="build exited ${exit_code}${last:+: $last}" + status="fail" + classify_cpu_failure + fail "$step" "$status" +fi +# With dev_mode=keep (needed to read the install log) the engine carries on +# after a failed install and still prints "Completed Successfully", exit 0. +install_exit=$(strip_ansi <"$RAW_LOG" | grep -oE 'Installation failed in container [0-9]+ \(exit code: [0-9]+\)' | + grep -oE '[0-9]+\)$' | tr -d ')' | tail -n 1) +if [[ -n "$install_exit" ]]; then + last=$(failure_line) + step="install script exited ${install_exit}${last:+: $last}" + status="fail" + classify_cpu_failure + fail "$step" "$status" +fi +grep -qi 'completed successfully' "$RAW_LOG" || fail "build exited 0 but never reported success" + +# --- Checks --------------------------------------------------------------------- +pct status "$ctid" 2>/dev/null | grep -q running || fail "container ${ctid} is not running" + +ct_ip=$(pct exec "$ctid" -- sh -c "ip -4 -o addr show dev eth0 | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null | head -n1) + +if pct exec "$ctid" -- sh -c 'command -v systemctl' &>/dev/null; then + sleep 10 + failed_units=$(pct exec "$ctid" -- systemctl --failed --no-legend --plain 2>/dev/null | awk '{print $1}' | paste -sd, -) +elif pct exec "$ctid" -- sh -c 'command -v rc-status' &>/dev/null; then + failed_units=$(pct exec "$ctid" -- rc-status --crashed 2>/dev/null | awk '{print $1}' | paste -sd, -) +fi +[[ -z "$failed_units" ]] || fail "failed services: ${failed_units}" + +# What the app's page looks like, for the PR comment. Chromium runs as its own +# unprivileged user with its sandbox on: the page is served by code under test. +# ci-shot-firewall (pve-ci-host-setup.sh) keeps that user off this host and the +# lab network; without it there is no screenshot rather than an unguarded one. +take_screenshot() { + command -v chromium >/dev/null && id ci-shot &>/dev/null || return 0 + iptables -w -S CI_SHOT 2>/dev/null | grep -q -- '-d 10.0.0.0/8 .*REJECT' || return 0 + local dir + dir=$(mktemp -d) + chown ci-shot "$dir" + runuser -u ci-shot -- timeout 60 chromium --headless=new --disable-gpu --hide-scrollbars \ + --ignore-certificate-errors --no-first-run --window-size=1280,800 --virtual-time-budget=10000 \ + --user-data-dir="$dir/profile" --screenshot="$dir/shot.png" "$url" &>/dev/null + # Only a plausible PNG leaves this function. + if [[ -s "$dir/shot.png" && $(stat -c %s "$dir/shot.png") -lt 3000000 ]] && + [[ "$(head -c 8 "$dir/shot.png" | od -An -tx1 | tr -d ' \n')" == "89504e470d0a1a0a" ]]; then + cp "$dir/shot.png" "$OUT_DIR/screenshot.png" + fi + rm -rf -- "$dir" +} + +# The URL the script tells the user to open, e.g. http://${IP}:1337/admin, has +# to answer 200 (after redirects, so a jump to /login is fine). A script that +# prints no URL has no web UI to check. +pattern=$(grep -oE 'https?://\$\{IP\}(:[0-9]+)?(/[A-Za-z0-9._~/?=&%-]*)?' "$tree/$SCRIPT" | head -n1) +if [[ -n "$pattern" && -n "$ct_ip" ]]; then + url="${pattern/\$\{IP\}/$ct_ip}" + for ((waited = 0; waited < URL_WAIT; waited += 5)); do + http_code=$(curl -sk -L --max-redirs 5 -o /dev/null -w '%{http_code}' --max-time 10 "$url" || true) + [[ "$http_code" =~ ^2[0-9][0-9]$ ]] && break + sleep 5 + done + # Also when the check fails: an error page explains more than a status code. + [[ "$http_code" =~ ^[1-5][0-9][0-9]$ ]] && take_screenshot + [[ "$http_code" =~ ^2[0-9][0-9]$ ]] || fail "${url} answered ${http_code/#000/nothing} instead of 200 within ${URL_WAIT}s" +fi + +status="pass" +finish