Compare commits

...
Author SHA1 Message Date
MervinPraisonandClaude Opus 5.5 91ce6290a0 fix(ci): match reusable-workflow job names in jobCompletedAt
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-26 10:32:45 +00:00
MervinPraisonandClaude Opus 5.5 f34ae8ff34 fix(ci): count gate-loosening label changes and bound the outsider window by the Claude job's end
Addresses review on #6007: removing no-auto-merge or a manual-only label (or adding an
override) is a touch; the flag step passes claude-response's completed_at as the upper bound.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-26 10:31:15 +00:00
MervinPraisonandClaude Opus 5.5 321620fa86 fix(ci): outsider flagging counts only what the run changed; writers can clear a wrong flag
Every automation acts as the App, so 'the App touched it during an outsider run' also caught routine pipeline label syncs on unrelated PRs (e.g. #5901/#5907 flagged by run 39229895860, whose transcript never mentions them), and the sticky rule then cascaded flags through references (#5945/#5947 -> #5949/#5950). touchedByRun now counts only content-changing App events (comments, reviews, pushes, title/body edits, creating the item, adding the claude label) inside the run window; replaying the real run flags only its own PR. A person with write access can clear a wrong flag by removing the label; bots cannot.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-10-26 10:20:02 +00:00
3 changed files with 98 additions and 7 deletions

No files matched your search

@@ -178,13 +178,43 @@ assert('no source issue', o.sourceIssueNumbers({ head: { ref: 'feat/x' }, body:
await o.touchedByRun(runGh([{ event: 'committed', committer: { date: '2026-10-17T10:07:00Z' } }]), 'o', 'r', item(), SINCE, APP));
assert('App editing the body during the run is touched (GraphQL login has no [bot])',
await o.touchedByRun(runGh([], [{ editedAt: '2026-10-17T10:08:00Z', editor: { login: 'praisonai-triage-agent' } }]), 'o', 'r', item(), SINCE, APP));
assert('routine label sync by the App during the run is not touched (shared App identity)',
!(await o.touchedByRun(runGh([{ event: 'unlabeled', actor: { login: 'praisonai-triage-agent[bot]' }, label: { name: 'pipeline/blocked:ci' }, created_at: '2026-10-17T10:06:00Z' }]), 'o', 'r', item(), SINCE, APP)));
assert('the App adding the claude label during the run is touched (it starts a Claude run)',
await o.touchedByRun(runGh([{ event: 'labeled', actor: { login: 'praisonai-triage-agent[bot]' }, label: { name: 'claude' }, created_at: '2026-10-17T10:06:00Z' }]), 'o', 'r', item(), SINCE, APP));
const appLabel = (event, name) => runGh([{ event, actor: { login: 'praisonai-triage-agent[bot]' }, label: { name }, created_at: '2026-10-17T10:06:00Z' }]);
assert('the App removing no-auto-merge during the run is touched (it loosens the gate)',
await o.touchedByRun(appLabel('unlabeled', 'no-auto-merge'), 'o', 'r', item(), SINCE, APP));
assert('the App removing a manual-only label during the run is touched',
await o.touchedByRun(appLabel('unlabeled', 'needs-manual-review'), 'o', 'r', item(), SINCE, APP));
assert('the App adding a maintainer-accept override during the run is touched',
await o.touchedByRun(appLabel('labeled', 'maintainer-accept-core-tools'), 'o', 'r', item(), SINCE, APP));
assert('the App adding no-auto-merge (tightening) is not touched',
!(await o.touchedByRun(appLabel('labeled', 'no-auto-merge'), 'o', 'r', item(), SINCE, APP)));
assert('the gate releasing its own lock label is not touched',
!(await o.touchedByRun(appLabel('unlabeled', 'claude-merge-gate-active'), 'o', 'r', item(), SINCE, APP)));
assert('the App force-pushing during the run is touched',
await o.touchedByRun(runGh([{ event: 'head_ref_force_pushed', actor: { login: 'praisonai-triage-agent[bot]' }, created_at: '2026-10-17T10:06:00Z' }]), 'o', 'r', item(), SINCE, APP));
assert('a maintainer comment during the run is not touched',
!(await o.touchedByRun(runGh([{ event: 'commented', user: { login: 'MervinPraison' }, created_at: '2026-10-17T10:06:00Z' }]), 'o', 'r', item(), SINCE, APP)));
assert('App activity after the run ended is not touched',
!(await o.touchedByRun(runGh([{ event: 'commented', user: { login: 'praisonai-triage-agent[bot]' }, created_at: '2026-10-20T10:00:00Z' }]), 'o', 'r', item(), SINCE, APP, '2026-10-17T10:30:00Z')));
assert('App activity before the run is not touched',
!(await o.touchedByRun(runGh([{ event: 'labeled', actor: { login: 'praisonai-triage-agent[bot]' }, created_at: '2026-10-17T09:00:00Z' }]), 'o', 'r', item(), SINCE, APP)));
const editFail = await (async () => { const g = runGh([]); g.graphql = async () => { throw new Error('rate'); }; return o.touchedByRun(g, 'o', 'r', item(), SINCE, APP).then(() => null, (e) => e); })();
assert('an edit-history lookup failure fails closed (throws)', editFail instanceof Error);
{
const jobs = [{ name: 'issue-triage', completed_at: '2026-10-17T10:00:01Z' }, { name: 'claude-response', completed_at: '2026-10-17T10:20:00Z' }];
const g = { rest: { actions: { listJobsForWorkflowRun: 'jobs' } }, paginate: async () => jobs };
assert('the Claude job end time bounds the run window', (await o.jobCompletedAt(g, 'o', 'r', 1, 'claude-response')) === '2026-10-17T10:20:00Z');
const hub = { rest: g.rest, paginate: async () => [{ name: 'claude / claude-response', completed_at: '2026-10-17T10:21:00Z' }] };
assert('a reusable workflow job (caller / job) is found', (await o.jobCompletedAt(hub, 'o', 'r', 1, 'claude-response')) === '2026-10-17T10:21:00Z');
assert('an unfinished or missing job gives no bound', (await o.jobCompletedAt(g, 'o', 'r', 1, 'nope')) === null);
const bad = { rest: { actions: { listJobsForWorkflowRun: 'jobs' } }, paginate: async () => { throw new Error('403'); } };
assert('a jobs lookup failure gives no bound (caller counts up to now)', (await o.jobCompletedAt(bad, 'o', 'r', 1, 'claude-response')) === null);
}
// A request mentioning another repo's numbers as bare #N (Docs issues cite PraisonAI PRs this way)
// must not make the run outsider: a number that names nothing here carries no content.
r = await o.referencedIssueReasons(fakeGithub({ issueError: 404 }), 'o', 'r', 'see upstream #5987 and #6366');
@@ -211,6 +241,21 @@ assert('no source issue', o.sourceIssueNumbers({ head: { ref: 'feat/x' }, body:
r = await o.outsiderOriginReasons(fakeGithub({ issueError: 404 }), 'o', 'r', internalPr(), []);
assert('a missing SOURCE issue (404) still fails closed', r.some((x) => x.includes('unavailable (HTTP 404)')));
// A person with write access can clear a wrong flag; bots cannot.
const lab = (event, login, t) => ({ event, label: { name: o.OUTSIDER_ORIGIN_LABEL }, actor: { login }, created_at: t });
r = await o.outsiderOriginReasons(fakeGithub({ issue: maintainerIssue, timeline: [
lab('labeled', 'praisonai-triage-agent[bot]', '2026-10-19T09:30:00Z'), lab('unlabeled', 'MervinPraison', '2026-10-26T10:00:00Z')] }), 'o', 'r', internalPr(), []);
assert('a writer removing the flag clears it', !r.some((x) => x.includes('flagged')));
r = await o.outsiderOriginReasons(fakeGithub({ issue: maintainerIssue, timeline: [
lab('labeled', 'praisonai-triage-agent[bot]', '2026-10-19T09:30:00Z'), lab('unlabeled', 'praisonai-triage-agent[bot]', '2026-10-26T10:00:00Z')] }), 'o', 'r', internalPr(), []);
assert('a bot removing the flag does not clear it', r.some((x) => x.includes('flagged')));
r = await o.outsiderOriginReasons(fakeGithub({ issue: maintainerIssue, timeline: [
lab('labeled', 'praisonai-triage-agent[bot]', '2026-10-19T09:30:00Z'), lab('unlabeled', 'stranger', '2026-10-26T10:00:00Z')] }), 'o', 'r', internalPr(), []);
assert('a non-writer removing the flag does not clear it', r.some((x) => x.includes('flagged')));
r = await o.outsiderOriginReasons(fakeGithub({ issue: maintainerIssue, timeline: [
lab('labeled', 'x[bot]', '2026-10-19T09:30:00Z'), lab('unlabeled', 'MervinPraison', '2026-10-20T10:00:00Z'), lab('labeled', 'x[bot]', '2026-10-21T10:00:00Z')] }), 'o', 'r', internalPr(), []);
assert('flagged again after a clear is flagged', r.some((x) => x.includes('flagged')));
r = await o.outsiderOriginReasons(fakeGithub({ issueError: 410 }), 'o', 'r', internalPr(), []);
assert('a deleted or transferred source issue (410) counts as unknown, so outsider', r.some((x) => x.includes('unavailable (HTTP 410)')));
+48 -6
View File
@@ -26,6 +26,15 @@ const TRUSTED_AUTOMATION = new Set(['Copilot']);
const SECURITY_STATUS_CONTEXT = 'security/outsider-review';
const SECURITY_STATUS_CREATOR = 'github-actions[bot]';
const OUTSIDER_ORIGIN_LABEL = 'origin/outsider';
// Timeline events that change content Claude or the merge gate reads.
const CONTENT_EVENTS = new Set(['commented', 'reviewed', 'line-commented', 'commit-commented', 'head_ref_force_pushed', 'renamed']);
// Label changes that loosen what Claude or the merge gate does (see merge-gate.js):
// adding a trigger or an override, or removing a stop or manual-only label.
// Pipeline status labels and the gate's own lock are routine and do not count.
const LOOSENING_LABELS = {
labeled: new Set(['claude', 'merge-gate-ci-only', 'maintainer-accept-example', 'maintainer-accept-core-tools']),
unlabeled: new Set(['no-auto-merge', 'claude-conflict-pending', 'auto-merged-by-gate', 'security', 'breaking-change', 'needs-manual-review', 'release']),
};
// Build, dependency, CI and install-time surface. Code here runs on developer
// machines, in CI with secrets, or at install time, so an outsider-origin
@@ -164,12 +173,24 @@ async function outsiderOriginReasons(github, owner, repo, pr, comments = []) {
return reasons;
}
/**
* Was this ever flagged? Timeline events cannot be edited, so deleting the
* outsider content does not clear it. Only a person with write access can clear
* a wrong flag, by removing the label after it was last added (bots cannot:
* every Claude run acts as the App).
*/
async function wasLabelled(github, owner, repo, number, label) {
try {
const events = await github.paginate(github.rest.issues.listEventsForTimeline, {
const events = (await github.paginate(github.rest.issues.listEventsForTimeline, {
owner, repo, issue_number: number, per_page: 100,
});
return events.some((e) => e.event === 'labeled' && e.label?.name === label);
})).filter((e) => (e.event === 'labeled' || e.event === 'unlabeled') && e.label?.name === label);
const lastAdded = events.map((e) => e.event).lastIndexOf('labeled');
if (lastAdded < 0) return false;
const isWriter = makePermissionLookup(github, owner, repo);
for (const e of events.slice(lastAdded + 1)) {
if (e.event === 'unlabeled' && !isAutomation(e.actor?.login) && (await isWriter(e.actor.login))) return false;
}
return true;
} catch (err) {
if (err.status === 404 || err.status === 410) return false;
throw err;
@@ -233,16 +254,20 @@ async function issueReasons(github, owner, repo, issueNumber, isWriter, label =
* commits to it, or edited its text. Unrelated activity by people does not count,
* so flagging what an outsider run touched does not sweep up other work.
*/
async function touchedByRun(github, owner, repo, item, since, appLogins) {
async function touchedByRun(github, owner, repo, item, since, appLogins, until = new Date().toISOString()) {
const t = new Date(since).getTime();
const after = (d) => !!d && new Date(d).getTime() >= t;
const end = new Date(until).getTime();
const after = (d) => !!d && new Date(d).getTime() >= t && new Date(d).getTime() <= end;
const isApp = (login) => !!login && (appLogins.has(login) || appLogins.has(`${login}[bot]`));
if (isApp(item.user?.login) && after(item.created_at)) return true;
const events = await github.paginate(github.rest.issues.listEventsForTimeline, {
owner, repo, issue_number: item.number, per_page: 100,
});
for (const e of events) {
if (isApp(e.actor?.login || e.user?.login) && after(e.created_at || e.submitted_at || e.updated_at)) return true;
// Only actions that change what Claude or the gate reads. Label syncing by the
// pipeline also runs as the App and must not count as the outsider run.
const content = CONTENT_EVENTS.has(e.event) || !!LOOSENING_LABELS[e.event]?.has(e.label?.name);
if (content && isApp(e.actor?.login || e.user?.login) && after(e.created_at || e.submitted_at || e.updated_at)) return true;
if (e.event === 'committed' && after(e.committer?.date)) return true;
}
let data;
@@ -265,6 +290,22 @@ async function touchedByRun(github, owner, repo, item, since, appLogins) {
return edits.some((n) => after(n.editedAt) && isApp(n.editor?.login));
}
/**
* When a job of this workflow run finished (null if unknown). It bounds what counts
* as that job's activity, so work by later runs is not attributed to it.
*/
async function jobCompletedAt(github, owner, repo, runId, jobName) {
try {
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
owner, repo, run_id: runId, per_page: 100,
});
// A reusable workflow's jobs are named '<caller job> / <job>'.
return jobs.find((j) => j.name === jobName || j.name.endsWith(` / ${jobName}`))?.completed_at || null;
} catch {
return null;
}
}
/**
* Why an issue Claude is asked to work on is outsider-influenced ([] when not):
* the issue itself, anything it was flagged as before, and every issue the
@@ -441,6 +482,7 @@ module.exports = {
referencedIssueNumbers,
referencedIssueReasons,
touchedByRun,
jobCompletedAt,
outsiderSensitivePaths,
riskFlags,
formatRiskFlags,
+5 -1
View File
@@ -558,13 +558,17 @@ jobs:
const o = require(path.join(process.env.GITHUB_WORKSPACE, '.github/scripts/outsider-review.js'));
const { owner, repo } = context.repo;
const since = process.env.STARTED_AT || new Date(0).toISOString();
// Claude's job is over (its token revoked, its processes killed); later
// activity belongs to other runs. Unknown: count everything up to now.
const until = (await o.jobCompletedAt(github, owner, repo, context.runId, 'claude-response')) || new Date().toISOString();
core.info(`Outsider run window: ${since} .. ${until}`);
// Issues and PRs created, edited, commented on or pushed to since then.
const touched = await github.paginate(github.rest.issues.listForRepo, { owner, repo, state: 'all', since, per_page: 100 });
const app = new Set(['praisonai-triage-agent[bot]']);
for (const item of touched) {
let flag = true;
try {
flag = await o.touchedByRun(github, owner, repo, item, since, app);
flag = await o.touchedByRun(github, owner, repo, item, since, app, until);
} catch (err) {
core.warning(`#${item.number}: ${err.message}; flagging it to be safe`);
}