diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 595ec14..2b45ef4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,6 +31,7 @@ concurrency: permissions: contents: read + issues: write # open/close the publish alert (scripts/ci-alert.sh) jobs: publish: @@ -194,3 +195,30 @@ jobs: printf '%s\n' "$GITHUB_SHA" > "$REPO_DIR/.published-sha" printf '%s\n' "$GITHUB_SHA" | rclone rcat "r2:$R2_BUCKET/.published-sha" \ --header-upload "Cache-Control: public, max-age=0, must-revalidate" + + # This workflow is the one nobody watches: it runs on a push that a bot + # already auto-merged, so a failure here means the catalog silently stops + # tracking upstream. Turn it into an issue, which notifies, and close that + # issue on the next green publish. + - name: Alert on a failed publish + if: failure() + env: + GH_TOKEN: ${{ github.token }} + run: | + ./scripts/ci-alert.sh open publish "publish failed" </dev/null || \ + sudo apt-get install -y flatpak bubblewrap + flatpak remote-add --user --if-not-exists flathub \ + https://dl.flathub.org/repo/flathub.flatpakrepo + for id in $stale; do + if INSTALL_RUNTIME=1 ./scripts/check-apply-extra.sh "$id"; then + ./scripts/ci-alert.sh resolve "update-check/$id" \ + "The pinned payload for \`$id\` unpacks again." + fi + done + fi + # Last, so a held-back app still gets its PR opened for the apps that passed. # A red daily run is the point: an upstream that repackaged needs a human. - name: Fail if any pin was held back diff --git a/docs/packaging-playbook.md b/docs/packaging-playbook.md index 7f17b3d..208fdb5 100644 --- a/docs/packaging-playbook.md +++ b/docs/packaging-playbook.md @@ -298,6 +298,12 @@ said no. - **Re-crawl (§1) periodically** to catch new releases and freshly-rejected Flathub PRs. - Each app's `resolve-update.sh` already lets FlatPark re-pin & rebuild on new upstream releases — no manual version bumps. +- **A broken upstream repackage opens an issue.** When `update-check`'s payload gate rejects a new + artifact it holds that app's pin (users stay on the last version known to install) and + `scripts/ci-alert.sh` opens a `ci-alert`-labelled issue naming the app — a red run in the Actions + tab notifies nobody, an issue does. Fix `registry//apply_extra.sh`, verify with + `./scripts/check-apply-extra.sh `, and the next run closes the issue itself. A failed + `publish` opens the same kind of issue under the `publish` key. - **Runtime majors are NOT bumped by CI.** `update-check` only re-pins extra-data and the metainfo release; `runtime-version` / `base-version` are hand-pinned per manifest. When a new major lands, the maintainer kicks off an **AI-led batch update** that bumps and re-tests the whole catalog at diff --git a/scripts/ci-alert.sh b/scripts/ci-alert.sh new file mode 100755 index 0000000..2548b07 --- /dev/null +++ b/scripts/ci-alert.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Turn a CI failure into something that reaches a human. +# +# Nothing else in the pipeline notifies: a held-back pin or a failed publish is +# only a red run in the Actions tab, and GitHub emails those out for *scheduled* +# runs alone — a workflow_dispatch or a push-triggered publish that dies after +# an auto-merge tells nobody. This opens a labelled issue instead, and closes it +# again once the same key succeeds. +# +# Deduped by a hidden marker in the issue body rather than by title, so an issue +# someone renamed is still found, and re-titling one never opens a second. A +# repeat failure is deliberately quiet: a daily job that keeps failing keeps ONE +# open issue and does not comment on every run. +# +# Usage: +# ci-alert.sh open [body-file] # body on stdin if omitted +# ci-alert.sh resolve <key> [comment] +# ci-alert.sh list [key-prefix] # keys with an open alert +# +# Needs GH_TOKEN (or GITHUB_TOKEN) with issues:write. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +. "$ROOT/scripts/lib/common.sh" +need gh + +LABEL="ci-alert" + +run_url() { + [ -n "${GITHUB_RUN_ID:-}" ] && [ -n "${GITHUB_REPOSITORY:-}" ] || return 0 + printf '%s/%s/actions/runs/%s\n' \ + "${GITHUB_SERVER_URL:-https://github.com}" "${GITHUB_REPOSITORY:-}" "$GITHUB_RUN_ID" +} + +marker() { printf '<!-- ci-alert:%s -->' "$1"; } + +# The open issue carrying this key, or nothing. Listed and filtered locally +# rather than via `--search`: the search index lags issue creation by up to a +# minute, long enough for two runs in a row to each open their own copy. +find_open() { + gh issue list --state open --label "$LABEL" --limit 100 \ + --json number,body -q \ + "map(select(.body != null and (.body | contains(\"$(marker "$1")\")))) | .[0].number // empty" +} + +cmd_open() { + local key="$1" title="$2" body_file="${3:-}" body existing url + [ -n "$key" ] && [ -n "$title" ] || die "usage: ci-alert.sh open <key> <title> [body-file]" + if [ -n "$body_file" ]; then + body="$(cat "$body_file")" + else + body="$(cat)" + fi + + existing="$(find_open "$key")" + if [ -n "$existing" ]; then + log "ci-alert: #$existing already open for $key" + return 0 + fi + + url="$(run_url)" + [ -z "$url" ] || body="$(printf '%s\n\nFailing run: %s' "$body" "$url")" + body="$(printf '%s\n\n%s\n' "$body" "$(marker "$key")")" + + # The label has to exist before it can be applied, and a fresh clone of this + # repo has never created it. + gh label create "$LABEL" --color B60205 \ + --description "Opened automatically by a failing workflow" >/dev/null 2>&1 || true + gh issue create --title "$title" --label "$LABEL" --body "$body" >&2 + log "ci-alert: opened for $key" +} + +cmd_resolve() { + local key="$1" comment="${2:-}" existing url + [ -n "$key" ] || die "usage: ci-alert.sh resolve <key> [comment]" + existing="$(find_open "$key")" + [ -n "$existing" ] || return 0 + url="$(run_url)" + [ -n "$comment" ] || comment="Fixed — this succeeded again." + [ -z "$url" ] || comment="$(printf '%s\n\nSucceeding run: %s' "$comment" "$url")" + gh issue close "$existing" --comment "$comment" >&2 + log "ci-alert: closed #$existing for $key" +} + +# Keys that currently have an open alert, so a caller can ask whether the +# thing that failed is healthy again. GitHub hands bodies back with CRLF line +# endings, hence the trailing-whitespace trim. +cmd_list() { + local prefix="${1:-}" + gh issue list --state open --label "$LABEL" --limit 100 \ + --json body -q '.[].body // ""' \ + | sed -n 's/.*<!-- ci-alert:\([^>]*\) -->.*/\1/p' \ + | sed 's/[[:space:]]*$//' \ + | awk -v p="$prefix" 'p == "" || index($0, p) == 1' +} + +case "${1:-}" in + open) shift; cmd_open "${1:-}" "${2:-}" "${3:-}" ;; + resolve) shift; cmd_resolve "${1:-}" "${2:-}" ;; + list) shift; cmd_list "${1:-}" ;; + *) die "usage: ci-alert.sh open|resolve|list ..." ;; +esac diff --git a/tests/test_ci_alert.sh b/tests/test_ci_alert.sh new file mode 100755 index 0000000..f1b5dad --- /dev/null +++ b/tests/test_ci_alert.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# ci-alert.sh must open exactly ONE issue per key no matter how often the +# failure repeats, and close that issue again when the same key succeeds. +# A second issue per red run would be worse than no notification at all. +# +# Hermetic: `gh` is a stand-in backed by a JSON file. The dedupe query is the +# part worth testing, so the stand-in evaluates the real -q expression with jq +# rather than faking the answer. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +. "$ROOT/tests/lib/assert.sh" + +command -v jq >/dev/null || { echo "test_ci_alert: SKIP (no jq)"; exit 0; } + +tmp="$(mktemp -d)"; trap 'rm -rf "$tmp"' EXIT +export CI_ALERT_TEST_STATE="$tmp/issues.json" +export CI_ALERT_TEST_LOG="$tmp/gh.log" +printf '[]\n' > "$CI_ALERT_TEST_STATE" +: > "$CI_ALERT_TEST_LOG" + +mkdir -p "$tmp/bin" +cat > "$tmp/bin/gh" <<'GH' +#!/usr/bin/env bash +set -euo pipefail +state="$CI_ALERT_TEST_STATE"; printf '%s\n' "$*" >> "$CI_ALERT_TEST_LOG" +sub="${1:-} ${2:-}"; shift 2 || true +write() { local f; f="$(mktemp)"; cat > "$f"; mv "$f" "$state"; } +case "$sub" in + "label create") ;; + "issue list") + expr="" + while [ $# -gt 0 ]; do + case "$1" in -q|--jq) expr="$2"; shift 2 ;; *) shift ;; esac + done + jq '[.[] | select(.state == "open")]' "$state" | jq -r "$expr" + ;; + "issue create") + title=""; body="" + while [ $# -gt 0 ]; do + case "$1" in + --title) title="$2"; shift 2 ;; + --body) body="$2"; shift 2 ;; + *) shift ;; + esac + done + n=$(( $(jq 'length' "$state") + 100 )) + jq --argjson n "$n" --arg t "$title" --arg b "$body" \ + '. + [{number: $n, title: $t, body: $b, state: "open"}]' "$state" | write + echo "issue #$n" + ;; + "issue close") + n="$1" + jq --argjson n "$n" 'map(if .number == $n then .state = "closed" else . end)' "$state" | write + ;; + *) echo "unexpected gh call: $sub $*" >&2; exit 1 ;; +esac +GH +chmod +x "$tmp/bin/gh" +export PATH="$tmp/bin:$PATH" +export GITHUB_SERVER_URL="https://github.com" GITHUB_REPOSITORY="flatpark/flatpark" GITHUB_RUN_ID="42" + +open_count() { jq '[.[] | select(.state == "open")] | length' "$CI_ALERT_TEST_STATE"; } +creates() { grep -c '^issue create' "$CI_ALERT_TEST_LOG" || true; } + +# First failure opens the issue, and it carries a link to the failing run. +printf 'the payload does not unpack\n' \ + | "$ROOT/scripts/ci-alert.sh" open "update-check/com.example.App" "held back: com.example.App" >/dev/null +assert_eq "$(open_count)" "1" +body="$(jq -r '.[0].body' "$CI_ALERT_TEST_STATE")" +case "$body" in + *"actions/runs/42"*) ;; + *) echo "FAIL: issue body lacks the run URL"; exit 1 ;; +esac + +# Same failure tomorrow, and the day after: still one issue, no new create call. +for _ in 1 2; do + printf 'the payload still does not unpack\n' \ + | "$ROOT/scripts/ci-alert.sh" open "update-check/com.example.App" "held back: com.example.App" >/dev/null +done +assert_eq "$(open_count)" "1" +assert_eq "$(creates)" "1" + +# A different key is a different issue. +printf 'publish died\n' | "$ROOT/scripts/ci-alert.sh" open "publish" "publish failed" >/dev/null +assert_eq "$(open_count)" "2" + +# Success closes only the key that recovered. +"$ROOT/scripts/ci-alert.sh" resolve "update-check/com.example.App" >/dev/null +assert_eq "$(open_count)" "1" +assert_eq "$(jq -r '[.[] | select(.state == "open")][0].body | contains("ci-alert:publish")' "$CI_ALERT_TEST_STATE")" "true" + +# `list` reports what is still alerting, filtered by key prefix, so a caller +# can re-check exactly the things that have not recovered. +assert_eq "$("$ROOT/scripts/ci-alert.sh" list)" "publish" +assert_eq "$("$ROOT/scripts/ci-alert.sh" list update-check/)" "" + +# Resolving a key with nothing open is a no-op, not an error. +"$ROOT/scripts/ci-alert.sh" resolve "update-check/com.example.App" >/dev/null +assert_eq "$(open_count)" "1"