mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 10:45:59 -10:00
feat(ci): ask reporters to verify on the release that carries their fix
A PR writing "Fixes #N" auto-closes its issue on merge. One writing "Refs", "Related" or "Addresses" does not, so the report stays open waiting for a follow-up after the release that nobody sends. Auditing the last 400 merged PRs found 33 open issues referenced by shipped work; only three could be closed on the evidence, and most of the rest were blocked on nothing but an unanswered "please re-test". On every published release this now diffs against the previous tag in the same series, collects the issues those PRs referenced, and asks each still-open reporter to verify. It never closes anything — deciding a report is resolved stays human. Two traps the audit hit are pinned by tests rather than left to be rediscovered: - "owner/repo#123" is an upstream reference. Matching it once pulled a wailsapp/wails issue number in as if it were ours. - GitHub's issues API answers for pull requests too, so a consolidation PR listing "PR #5576 by @x" reads as a perfectly good open issue. A dry run against desktop-v1.18.0 showed 26 of 40 references were pull requests; without the check the bot would have asked their authors to re-test. The PR template now says where to put the keyword, since a "- Fixes #123" list item is exactly the silent no-op that produced this backlog.
This commit is contained in:
yhh committed
1 parent
fc42c619d8
commit
88925b5caa
4 files changed
+331
No files matched your search
@@ -2,6 +2,15 @@
|
||||
|
||||
-
|
||||
|
||||
## Issues
|
||||
|
||||
<!--
|
||||
If this resolves a report, put `Fixes #123` on its own line — GitHub only
|
||||
auto-closes from a bare line, so `- Fixes #123` in a list does nothing and the
|
||||
report stays open. If it only relates to one, use `Refs #123` instead: the
|
||||
release workflow then asks that reporter to verify once the fix ships.
|
||||
-->
|
||||
|
||||
## Verification
|
||||
|
||||
-
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
name: Request verification on released issues
|
||||
|
||||
# A PR that writes "Fixes #N" auto-closes its issue on merge. One that writes
|
||||
# "Refs"/"Related"/"Addresses" does not, so the report stays open waiting for a
|
||||
# follow-up nobody remembers to send after the release. This sends it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: Release tag to announce (e.g. desktop-v1.18.0)
|
||||
required: true
|
||||
type: string
|
||||
dry_run:
|
||||
description: Print the targets without commenting
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
release:
|
||||
types: [published]
|
||||
push:
|
||||
branches:
|
||||
- main-v2
|
||||
paths:
|
||||
- '.github/workflows/release-verify-issues.yml'
|
||||
- 'scripts/release-verify-issues.mjs'
|
||||
- 'scripts/release-verify-issues.test.mjs'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
concurrency:
|
||||
group: release-verify-issues-${{ github.event.inputs.tag || github.event.release.tag_name || github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
if: github.repository == 'esengine/DeepSeek-Reasonix'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '22'
|
||||
- name: Test notifier
|
||||
run: node --test scripts/release-verify-issues.test.mjs
|
||||
# A push that only touched the script has no release to announce; the test
|
||||
# above is the whole point of that trigger.
|
||||
- name: Request verification
|
||||
if: github.event_name != 'push'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ github.event.inputs.tag || github.event.release.tag_name }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
args=(--tag "$TAG")
|
||||
if [ "$DRY_RUN" = "true" ]; then args+=(--dry-run); fi
|
||||
node scripts/release-verify-issues.mjs "${args[@]}"
|
||||
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env node
|
||||
// Ask reporters to verify, on the release that actually carries their fix.
|
||||
// A PR that writes "Fixes #N" auto-closes its issue on merge; one that writes
|
||||
// "Refs"/"Related"/"Addresses" does not, so the report sits open until someone
|
||||
// remembers to come back after the release. Nobody does. This posts that
|
||||
// follow-up for every still-open issue referenced by a PR in the release.
|
||||
//
|
||||
// It never closes anything — deciding a report is resolved stays human.
|
||||
//
|
||||
// Usage:
|
||||
// GH_TOKEN=... node scripts/release-verify-issues.mjs --tag desktop-v1.18.0 [--dry-run]
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
// A bare "#123" is ours. "owner/repo#123" is upstream — matching those once
|
||||
// filed a Wails issue number as if it were one of ours.
|
||||
const ISSUE_REF = /(^|[^\w/#-])#(\d{1,6})\b/g;
|
||||
const MERGE_SUBJECT = /Merge pull request #(\d+)/g;
|
||||
const SQUASH_SUBJECT = /\(#(\d+)\)\s*$/gm;
|
||||
|
||||
export function parseIssueRefs(text) {
|
||||
const out = new Set();
|
||||
for (const [, , n] of (text || "").matchAll(ISSUE_REF)) out.add(Number(n));
|
||||
return out;
|
||||
}
|
||||
|
||||
export function parsePullRequestNumbers(gitLog) {
|
||||
const out = new Set();
|
||||
for (const [, n] of (gitLog || "").matchAll(MERGE_SUBJECT)) out.add(Number(n));
|
||||
for (const [, n] of (gitLog || "").matchAll(SQUASH_SUBJECT)) out.add(Number(n));
|
||||
return [...out].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
// Releases share a repo but not a series: desktop-v1.18.0 follows
|
||||
// desktop-v1.17.21, not npm-v1.18.0.
|
||||
export function tagSeries(tag) {
|
||||
const m = /^(.*?)v\d/.exec(tag);
|
||||
return m ? m[1] : "";
|
||||
}
|
||||
|
||||
export function previousTag(tags, current) {
|
||||
const series = tagSeries(current);
|
||||
const peers = tags.filter((t) => tagSeries(t) === series);
|
||||
const i = peers.indexOf(current);
|
||||
return i >= 0 && i + 1 < peers.length ? peers[i + 1] : null;
|
||||
}
|
||||
|
||||
export function verificationMarker(tag) {
|
||||
return `<!-- release-verify:${tag} -->`;
|
||||
}
|
||||
|
||||
export function renderComment({ tag, pullNumbers }) {
|
||||
const prs = pullNumbers.map((n) => `#${n}`).join(", ");
|
||||
const source = pullNumbers.length === 1 ? `${prs} is` : `${prs} are`;
|
||||
return [
|
||||
verificationMarker(tag),
|
||||
`A change referencing this report shipped in **\`${tag}\`** — ${source} in that release.`,
|
||||
"",
|
||||
"The PR referenced this issue without claiming to close it, so this is a request to verify rather than a fix announcement: the change may resolve what you reported, address only part of it, or turn out to be unrelated.",
|
||||
"",
|
||||
`Could you re-test on \`${tag}\` and reply either way? "Still happening" is as useful as "fixed" — an unverified fix is why this thread stayed open.`,
|
||||
"",
|
||||
"No action needed if you have moved on; this stays open until someone says otherwise.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
// GitHub's issues API returns pull requests too, so a reference to a sibling PR
|
||||
// looks like a perfectly good open issue until you check for this key.
|
||||
export function isNotifiable(record, tag) {
|
||||
if (!record || record.isPullRequest) return false;
|
||||
if (record.state !== "open") return false;
|
||||
return !(record.commentBodies || []).some((body) => (body || "").includes(verificationMarker(tag)));
|
||||
}
|
||||
|
||||
export function selectTargets({ refsByIssue, records, tag }) {
|
||||
return [...refsByIssue.entries()]
|
||||
.filter(([issue]) => isNotifiable(records.get(issue), tag))
|
||||
.map(([issue, pulls]) => ({ issue, pullNumbers: [...pulls].sort((a, b) => a - b) }))
|
||||
.sort((a, b) => a.issue - b.issue);
|
||||
}
|
||||
|
||||
function gh(args, { json = true } = {}) {
|
||||
const out = execFileSync("gh", args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
|
||||
return json ? JSON.parse(out) : out;
|
||||
}
|
||||
|
||||
function git(args) {
|
||||
return execFileSync("git", args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
async function main() {
|
||||
const argv = process.argv.slice(2);
|
||||
const dryRun = argv.includes("--dry-run");
|
||||
const tag = argv[argv.indexOf("--tag") + 1];
|
||||
if (!tag || tag.startsWith("--")) {
|
||||
console.error("usage: release-verify-issues.mjs --tag <release-tag> [--dry-run]");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const tags = git(["tag", "--list", "--sort=-v:refname"]).split("\n").filter(Boolean);
|
||||
const from = previousTag(tags, tag);
|
||||
if (!from) {
|
||||
console.log(`no previous tag in the ${tagSeries(tag) || "root"} series; nothing to compare`);
|
||||
return;
|
||||
}
|
||||
|
||||
const pulls = parsePullRequestNumbers(git(["log", `${from}..${tag}`, "--pretty=%s%n%b"]));
|
||||
console.log(`${from}..${tag}: ${pulls.length} pull request(s)`);
|
||||
|
||||
const refsByIssue = new Map();
|
||||
for (const pr of pulls) {
|
||||
let data;
|
||||
try {
|
||||
data = gh(["pr", "view", String(pr), "--json", "title,body"]);
|
||||
} catch {
|
||||
continue; // a "#N" that is not a PR in this repo
|
||||
}
|
||||
for (const issue of parseIssueRefs(`${data.title}\n${data.body || ""}`)) {
|
||||
if (issue === pr) continue;
|
||||
if (!refsByIssue.has(issue)) refsByIssue.set(issue, new Set());
|
||||
refsByIssue.get(issue).add(pr);
|
||||
}
|
||||
}
|
||||
|
||||
const repo = gh(["repo", "view", "--json", "nameWithOwner"]).nameWithOwner;
|
||||
const records = new Map();
|
||||
for (const issue of refsByIssue.keys()) {
|
||||
let head;
|
||||
try {
|
||||
head = gh(["api", `repos/${repo}/issues/${issue}`]);
|
||||
} catch {
|
||||
continue; // referenced number does not exist in this repo
|
||||
}
|
||||
const record = { state: head.state, isPullRequest: Boolean(head.pull_request), commentBodies: [] };
|
||||
if (isNotifiable(record, tag)) {
|
||||
const comments = gh(["api", `repos/${repo}/issues/${issue}/comments`, "--paginate"]);
|
||||
record.commentBodies = comments.map((c) => c.body || "");
|
||||
}
|
||||
records.set(issue, record);
|
||||
}
|
||||
|
||||
const targets = selectTargets({ refsByIssue, records, tag });
|
||||
const skippedPulls = [...records.values()].filter((r) => r.isPullRequest).length;
|
||||
console.log(`${refsByIssue.size} referenced (${skippedPulls} were pull requests), ${targets.length} to notify`);
|
||||
|
||||
for (const { issue, pullNumbers } of targets) {
|
||||
const body = renderComment({ tag, pullNumbers });
|
||||
if (dryRun) {
|
||||
console.log(`[dry-run] #${issue} <- ${pullNumbers.map((n) => `#${n}`).join(", ")}`);
|
||||
continue;
|
||||
}
|
||||
gh(["issue", "comment", String(issue), "--body", body], { json: false });
|
||||
console.log(`commented on #${issue}`);
|
||||
await sleep(3000); // stay under GitHub's secondary content-creation limit
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
parseIssueRefs,
|
||||
parsePullRequestNumbers,
|
||||
previousTag,
|
||||
renderComment,
|
||||
isNotifiable,
|
||||
selectTargets,
|
||||
tagSeries,
|
||||
verificationMarker,
|
||||
} from "./release-verify-issues.mjs";
|
||||
|
||||
test("upstream references are not mistaken for our issue numbers", () => {
|
||||
const body = "Follows wailsapp/wails#5544 and ScoopInstaller/Extras#18363. Refs #3470, fixes (#4092).";
|
||||
assert.deepEqual([...parseIssueRefs(body)].sort((a, b) => a - b), [3470, 4092]);
|
||||
});
|
||||
|
||||
test("issue refs survive the punctuation PR bodies actually use", () => {
|
||||
const refs = parseIssueRefs("Addresses #6607 (also the mechanism behind #6346 / #5760).\n- #6225\nRelated: #6259, #6333");
|
||||
assert.deepEqual([...refs].sort((a, b) => a - b), [5760, 6225, 6259, 6333, 6346, 6607]);
|
||||
});
|
||||
|
||||
test("pull request numbers come from both merge and squash subjects", () => {
|
||||
const log = [
|
||||
"Merge pull request #7053 from esengine/fix/cli-colour-profile",
|
||||
"fix(cli): resolve colour support from colorprofile",
|
||||
"polish(desktop): tighten spacing (#7019)",
|
||||
].join("\n");
|
||||
assert.deepEqual(parsePullRequestNumbers(log), [7019, 7053]);
|
||||
});
|
||||
|
||||
test("a release only follows its own tag series", () => {
|
||||
const tags = ["desktop-v1.18.0", "npm-v1.18.0", "v1.18.0", "desktop-v1.17.21", "npm-v1.17.21", "v1.17.21"];
|
||||
assert.equal(previousTag(tags, "desktop-v1.18.0"), "desktop-v1.17.21");
|
||||
assert.equal(previousTag(tags, "npm-v1.18.0"), "npm-v1.17.21");
|
||||
assert.equal(previousTag(tags, "v1.18.0"), "v1.17.21");
|
||||
assert.equal(tagSeries("desktop-v1.18.0"), "desktop-");
|
||||
assert.equal(tagSeries("v1.18.0"), "");
|
||||
});
|
||||
|
||||
test("the oldest tag in a series has nothing to compare against", () => {
|
||||
assert.equal(previousTag(["desktop-v1.0.0"], "desktop-v1.0.0"), null);
|
||||
assert.equal(previousTag(["desktop-v1.0.0"], "desktop-v9.9.9"), null);
|
||||
});
|
||||
|
||||
test("a referenced pull request is never treated as a reportable issue", () => {
|
||||
const tag = "desktop-v1.18.0";
|
||||
// GitHub's issues API answers for pull requests too, so this is the only
|
||||
// thing separating a consolidation PR's "PR #5576 by @x" from a real report.
|
||||
assert.equal(isNotifiable({ state: "open", isPullRequest: true, commentBodies: [] }, tag), false);
|
||||
assert.equal(isNotifiable({ state: "open", isPullRequest: false, commentBodies: [] }, tag), true);
|
||||
});
|
||||
|
||||
test("closed and already-notified issues are skipped", () => {
|
||||
const tag = "desktop-v1.18.0";
|
||||
assert.equal(isNotifiable({ state: "closed", isPullRequest: false, commentBodies: [] }, tag), false);
|
||||
assert.equal(
|
||||
isNotifiable({ state: "open", isPullRequest: false, commentBodies: [`x ${verificationMarker(tag)} y`] }, tag),
|
||||
false,
|
||||
);
|
||||
// a marker from a different release must not suppress this one
|
||||
assert.equal(
|
||||
isNotifiable({ state: "open", isPullRequest: false, commentBodies: [verificationMarker("desktop-v1.17.21")] }, tag),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("selectTargets keeps only notifiable issues, sorted, with their source PRs", () => {
|
||||
const tag = "desktop-v1.18.0";
|
||||
const refsByIssue = new Map([
|
||||
[300, new Set([30])],
|
||||
[100, new Set([10])],
|
||||
[200, new Set([21, 20])],
|
||||
]);
|
||||
const records = new Map([
|
||||
[100, { state: "open", isPullRequest: false, commentBodies: [verificationMarker(tag)] }],
|
||||
[200, { state: "open", isPullRequest: false, commentBodies: [] }],
|
||||
[300, { state: "open", isPullRequest: true, commentBodies: [] }],
|
||||
]);
|
||||
assert.deepEqual(selectTargets({ refsByIssue, records, tag }), [{ issue: 200, pullNumbers: [20, 21] }]);
|
||||
});
|
||||
|
||||
test("the comment carries a per-tag marker that the skip check matches", () => {
|
||||
const body = renderComment({ tag: "desktop-v1.18.0", pullNumbers: [7019] });
|
||||
assert.ok(body.includes(verificationMarker("desktop-v1.18.0")));
|
||||
assert.ok(!body.includes(verificationMarker("desktop-v1.17.21")));
|
||||
assert.match(body, /#7019 is in that release/);
|
||||
});
|
||||
|
||||
test("the comment asks for verification and never claims the issue is fixed", () => {
|
||||
const body = renderComment({ tag: "desktop-v1.18.0", pullNumbers: [1, 2] });
|
||||
assert.match(body, /#1, #2 are in that release/);
|
||||
assert.match(body, /request to verify rather than a fix announcement/);
|
||||
assert.doesNotMatch(body, /\bclosing\b|\bfixed in\b/i);
|
||||
});
|
||||
Reference in new issue
Block a user