mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-08 17:10:50 -10:00
fix(web): tighten community draft review follow-ups
Review of the first commit on this branch found gaps; this addresses them. - Posting claim: the claim now carries a random token that is read back before the GitHub call, so two concurrent posts of one draft that meet in the same KV location post once. KV has no compare-and-set and is eventually consistent across locations, so this is documented as best effort, not a lock. - GitHub 5xx, 408 and 429 answers keep the claim (the post may exist); only other 4xx answers release it for an immediate retry. - Resolution markers no longer suppress triage / PR-review / stale drafting for 90 days regardless of activity: an item updated more than 10 minutes after the maintainer's decision can be drafted again (our own post's updated_at bump stays covered). Digests, dupes and content-watch findings keep the 90-day suppression; their identity already encodes the content. - /digest publishes only the language the maintainer reviewed (approvedLang), revalidates the page after approval or discard, and a posted draft can no longer be discarded (409), which would have silently unpublished a digest. - The admin client shows the route's warning (state not saved, or digest not published because it was edited or its record is gone). - listDrafts reads at most 500 drafts in parallel batches of 50 and keeps what it read when a later list page fails. - runSemanticDrift and the triage / PR-review / stale runs count a draft only when saveDraft wrote it. Tests: web vitest 57 files / 526 passed; targeted community-agent-review-state, content-watch, community-agent-security, public-api-security 4 files / 45 passed; 14 of the new or changed tests fail on the previous source; tsc --noEmit exit 0; eslint on touched files exit 0; web npm run check ran through next build. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_014ZwqatxgVFxHvovngywnks
This commit is contained in:
1 parent
f229b40d59
commit
1d83478bd3
8 files changed
+356
-62
No files matched your search
@@ -37,6 +37,9 @@ export function AdminClient({ drafts, posted, isZh, typeLabels }: Props) {
|
||||
}
|
||||
}
|
||||
setEditing(null);
|
||||
// The post went through but something after it did not (draft state
|
||||
// not saved, or the digest was not published on /digest).
|
||||
if (typeof data.warning === "string") alert(data.warning);
|
||||
} else {
|
||||
alert(`Error: ${data.error}`);
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
isPublishedDigest,
|
||||
type WeeklyDigestRecord as WeeklyDigest,
|
||||
} from "@/lib/community-agent";
|
||||
import { getDigest, pickTextLocale } from "@/lib/i18n/dictionaries";
|
||||
import { getDigest } from "@/lib/i18n/dictionaries";
|
||||
import { getEnv } from "@/lib/kv";
|
||||
import { buildPageMetadata } from "@/lib/page-meta";
|
||||
|
||||
@@ -71,9 +71,9 @@ export default async function DigestArchivePage({ params }: { params: Promise<{
|
||||
}
|
||||
}
|
||||
|
||||
// The records are bilingual; show the reader's language (English for
|
||||
// every locale without a Chinese record).
|
||||
const zh = pickTextLocale(locale) === "zh";
|
||||
// A maintainer reviews a digest in one language (the admin locale), and
|
||||
// only that language is published, so each digest renders in the language
|
||||
// it was reviewed in whatever the reader's locale.
|
||||
|
||||
if (digests.length === 0) {
|
||||
return (
|
||||
@@ -91,7 +91,9 @@ export default async function DigestArchivePage({ params }: { params: Promise<{
|
||||
<PageHeader title={t.title} lede={t.lead} pose="read" />
|
||||
<div className="page-body">
|
||||
<div className="page-body-narrow digest-list">
|
||||
{digests.map((digest: WeeklyDigest) => (
|
||||
{digests.map((digest: WeeklyDigest) => {
|
||||
const zh = digest.approvedLang === "zh";
|
||||
return (
|
||||
<article key={digest.weekId} className="changelog-release" lang={zh ? "zh" : "en"}>
|
||||
<header className="digest-head">
|
||||
<span className="pill tabular">{digest.weekId}</span>
|
||||
@@ -111,7 +113,8 @@ export default async function DigestArchivePage({ params }: { params: Promise<{
|
||||
))}
|
||||
</div>
|
||||
</article>
|
||||
))}
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { revalidatePath } from "next/cache";
|
||||
import { NextResponse } from "next/server";
|
||||
import { BodyReadError, readBoundedBody } from "@/lib/bounded-body";
|
||||
import {
|
||||
approveDigestRecord,
|
||||
claimDraftForPosting,
|
||||
clearDraftResolution,
|
||||
deleteDigestRecord,
|
||||
deleteDraft,
|
||||
@@ -41,6 +43,25 @@ const ALLOWED_ACTIONS = new Set(["post", "discard"]);
|
||||
const ALLOWED_ORIGINS = new Set(["https://codewhale.net", "https://www.codewhale.net"]);
|
||||
const MAX_BODY_BYTES = 65_536;
|
||||
|
||||
/** Refresh the ISR copy of /digest after its published set changes. */
|
||||
function revalidateDigestPage() {
|
||||
try {
|
||||
revalidatePath("/[locale]/digest", "page");
|
||||
} catch (e) {
|
||||
// The page still refreshes on its hourly revalidate.
|
||||
console.error("digest revalidation failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A GitHub 4xx other than 408/429 means the post was not created, so the
|
||||
* claim can be released. A 5xx, 408 or 429 can come back after GitHub already
|
||||
* created the comment or issue, so its outcome is unknown.
|
||||
*/
|
||||
function githubDefinitelyRejected(status: number): boolean {
|
||||
return status >= 400 && status < 500 && status !== 408 && status !== 429;
|
||||
}
|
||||
|
||||
export async function POST(req: Request) {
|
||||
const env = await getAgentEnv();
|
||||
|
||||
@@ -105,6 +126,12 @@ export async function POST(req: Request) {
|
||||
}
|
||||
|
||||
if (action === "discard") {
|
||||
// A posted draft is already public on GitHub (and, for a digest, on
|
||||
// /digest); discarding it would silently unpublish or relabel it.
|
||||
const resolution = await getDraftResolution(env.CURATED_KV, parsedKey.type, parsedKey.id);
|
||||
if (draft.posted || resolution?.state === "posted" || resolution?.state === "posting") {
|
||||
return NextResponse.json({ error: "draft already posted" }, { status: 409 });
|
||||
}
|
||||
try {
|
||||
// The marker stops the next cron run from regenerating this draft.
|
||||
await markDraftResolved(env.CURATED_KV, parsedKey.type, parsedKey.id, "discarded");
|
||||
@@ -115,6 +142,7 @@ export async function POST(req: Request) {
|
||||
} catch (e) {
|
||||
return NextResponse.json({ error: `discard failed: ${String(e)}` }, { status: 500 });
|
||||
}
|
||||
if (draft.type === "digest") revalidateDigestPage();
|
||||
return NextResponse.json({ ok: true, action: "discarded" });
|
||||
}
|
||||
|
||||
@@ -137,7 +165,9 @@ export async function POST(req: Request) {
|
||||
return NextResponse.json({ error: `draft already ${resolution.state}` }, { status: 409 });
|
||||
}
|
||||
try {
|
||||
await markDraftResolved(env.CURATED_KV, parsedKey.type, parsedKey.id, "posting");
|
||||
if (!(await claimDraftForPosting(env.CURATED_KV, parsedKey.type, parsedKey.id))) {
|
||||
return NextResponse.json({ error: "draft already posting" }, { status: 409 });
|
||||
}
|
||||
} catch (e) {
|
||||
return NextResponse.json({ error: `could not claim draft: ${String(e)}` }, { status: 500 });
|
||||
}
|
||||
@@ -153,7 +183,7 @@ export async function POST(req: Request) {
|
||||
await env.CURATED_KV?.put(draftKey, JSON.stringify(draft), { expirationTtl: 60 * 60 * 24 * 7 });
|
||||
return undefined;
|
||||
} catch (e) {
|
||||
return `posted, but saving draft state failed: ${String(e)}`;
|
||||
return `Posted to GitHub, but saving the draft state failed (${String(e)}). Do not post it again; it may reappear as pending.`;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -175,17 +205,20 @@ export async function POST(req: Request) {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const unknownOutcome = () =>
|
||||
const unknownOutcome = (detail: string) =>
|
||||
NextResponse.json(
|
||||
{ error: "GitHub request failed; check GitHub before retrying (retry unlocks in 15 minutes)" },
|
||||
{ error: `${detail}; check GitHub before retrying (retry unlocks in 15 minutes)` },
|
||||
{ status: 502 }
|
||||
);
|
||||
const githubFailed = async (res: Response) => {
|
||||
// GitHub definitively rejected the post, so release the claim.
|
||||
const text = await res.text().catch(() => "");
|
||||
if (!githubDefinitelyRejected(res.status)) {
|
||||
// Keep the claim: GitHub may have created the post anyway.
|
||||
return unknownOutcome(`GitHub ${res.status}: ${text}`);
|
||||
}
|
||||
try {
|
||||
await clearDraftResolution(env.CURATED_KV, parsedKey.type, parsedKey.id);
|
||||
} catch { /* the claim expires on its own */ }
|
||||
const text = await res.text();
|
||||
return NextResponse.json({ error: `GitHub ${res.status}: ${text}` }, { status: 502 });
|
||||
};
|
||||
|
||||
@@ -198,7 +231,7 @@ export async function POST(req: Request) {
|
||||
const issuesUrl = `https://api.github.com/repos/${digestRepo}/issues`;
|
||||
|
||||
const digestRes = await postToGitHub(issuesUrl, { title, body: digestBody, labels: ["digest"] }, "token");
|
||||
if (!digestRes) return unknownOutcome();
|
||||
if (!digestRes) return unknownOutcome("GitHub request failed");
|
||||
if (!digestRes.ok) return githubFailed(digestRes);
|
||||
|
||||
const issue = await digestRes.json().catch(() => ({})) as { number?: number; html_url?: string };
|
||||
@@ -208,16 +241,20 @@ export async function POST(req: Request) {
|
||||
if (typeof issue.html_url === "string") draft.targetUrl = issue.html_url;
|
||||
let warning = await recordPosted();
|
||||
|
||||
// Publishing to /digest is the approval. The structured record holds
|
||||
// the unedited model text, so an edited digest is posted to GitHub but
|
||||
// not published there.
|
||||
// Publishing to /digest is the approval, for the language shown to the
|
||||
// maintainer only. The structured record holds the unedited model
|
||||
// text, so an edited digest is posted to GitHub but not published there.
|
||||
let published = false;
|
||||
if (editedBody === undefined || editedBody === originalBody) {
|
||||
try {
|
||||
published = await approveDigestRecord(env.CURATED_KV, draft.id);
|
||||
published = await approveDigestRecord(env.CURATED_KV, draft.id, lang === "zh" ? "zh" : "en");
|
||||
if (published) revalidateDigestPage();
|
||||
} catch (e) {
|
||||
warning ??= `posted, but publishing the digest page failed: ${String(e)}`;
|
||||
warning ??= `Posted to GitHub, but publishing the digest page failed (${String(e)}).`;
|
||||
}
|
||||
if (!published) warning ??= "Posted to GitHub, but the weekly record is gone, so /digest does not show it.";
|
||||
} else {
|
||||
warning ??= "Posted to GitHub. The text was edited, so /digest does not show this week.";
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
@@ -234,7 +271,7 @@ export async function POST(req: Request) {
|
||||
const commentUrl = `https://api.github.com/repos/${repo}/issues/${draft.targetNumber}/comments`;
|
||||
|
||||
const ghRes = await postToGitHub(commentUrl, { body: commentBody }, "Bearer");
|
||||
if (!ghRes) return unknownOutcome();
|
||||
if (!ghRes) return unknownOutcome("GitHub request failed");
|
||||
if (!ghRes.ok) return githubFailed(ghRes);
|
||||
|
||||
// Mark as posted
|
||||
|
||||
@@ -23,26 +23,43 @@ vi.mock("@/lib/github", async (importOriginal) => {
|
||||
});
|
||||
|
||||
import { POST as adminPost } from "../app/api/admin/post/route";
|
||||
import { isPublishedDigest, listDrafts, saveDraft, type AgentDraft } from "./community-agent";
|
||||
import {
|
||||
isPublishedDigest,
|
||||
listDrafts,
|
||||
MAX_LISTED_DRAFTS,
|
||||
saveDraft,
|
||||
type AgentDraft,
|
||||
} from "./community-agent";
|
||||
import { runDigest, runDupes, runTriage } from "./community-agent-tasks";
|
||||
|
||||
/** In-memory KV that pages like Cloudflare KV (max 1000 keys per list call). */
|
||||
class FakeKv {
|
||||
readonly values = new Map<string, string>();
|
||||
failPutsMatching: RegExp | null = null;
|
||||
/** Yield to the event loop inside every call, like a network round trip. */
|
||||
yieldEachOp = false;
|
||||
pageSize = 1000;
|
||||
failListAtCursor: string | null = null;
|
||||
|
||||
private async tick() {
|
||||
if (this.yieldEachOp) await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
async get(key: string): Promise<string | null> {
|
||||
await this.tick();
|
||||
return this.values.get(key) ?? null;
|
||||
}
|
||||
|
||||
async put(key: string, value: string): Promise<void> {
|
||||
await this.tick();
|
||||
if (this.failPutsMatching?.test(key)) throw new Error("kv put failed");
|
||||
this.values.set(key, value);
|
||||
}
|
||||
|
||||
async list(options?: { prefix?: string; limit?: number; cursor?: string }) {
|
||||
if (options?.cursor && options.cursor === this.failListAtCursor) throw new Error("kv list failed");
|
||||
const prefix = options?.prefix ?? "";
|
||||
const limit = Math.min(options?.limit ?? 1000, 1000);
|
||||
const limit = Math.min(options?.limit ?? 1000, this.pageSize);
|
||||
const start = options?.cursor ? Number(options.cursor) : 0;
|
||||
const all = [...this.values.keys()].filter((k) => k.startsWith(prefix)).sort();
|
||||
const page = all.slice(start, start + limit);
|
||||
@@ -182,7 +199,40 @@ describe("weekly digest publication requires maintainer approval", () => {
|
||||
const res = await adminPost(adminRequest(postBody({ action: "post", draftKey, lang: "en" })));
|
||||
await expect(res.json()).resolves.toMatchObject({ ok: true, published: true, number: 900 });
|
||||
expect(posts).toHaveLength(1);
|
||||
expect(isPublishedDigest(JSON.parse(kv.values.get(recordKey)!))).toBe(true);
|
||||
const published = JSON.parse(kv.values.get(recordKey)!);
|
||||
expect(isPublishedDigest(published)).toBe(true);
|
||||
// Only the reviewed language is published.
|
||||
expect(published).toMatchObject({ approvedLang: "en" });
|
||||
});
|
||||
|
||||
it("records the Chinese review as the published language when posted from the zh admin", async () => {
|
||||
const kv = new FakeKv();
|
||||
stubDigestSources();
|
||||
mocks.agentChat.mockResolvedValue({ content: JSON.stringify(DIGEST_MODEL_OUTPUT), usage: { input: 1, output: 1 } });
|
||||
await runDigest({ CURATED_KV: kv, DEEPSEEK_API_KEY: "k" });
|
||||
|
||||
useAdminEnv(kv);
|
||||
stubGitHub();
|
||||
const draftKey = onlyKey(kv, "draft:digest:");
|
||||
const res = await adminPost(adminRequest(postBody({ action: "post", draftKey, lang: "zh" })));
|
||||
await expect(res.json()).resolves.toMatchObject({ ok: true, published: true });
|
||||
expect(JSON.parse(kv.values.get(onlyKey(kv, "digest:weekly-"))!)).toMatchObject({ approvedLang: "zh" });
|
||||
});
|
||||
|
||||
it("refuses to discard a posted digest, which would silently unpublish it", async () => {
|
||||
const kv = new FakeKv();
|
||||
stubDigestSources();
|
||||
mocks.agentChat.mockResolvedValue({ content: JSON.stringify(DIGEST_MODEL_OUTPUT), usage: { input: 1, output: 1 } });
|
||||
await runDigest({ CURATED_KV: kv, DEEPSEEK_API_KEY: "k" });
|
||||
|
||||
useAdminEnv(kv);
|
||||
stubGitHub();
|
||||
const draftKey = onlyKey(kv, "draft:digest:");
|
||||
expect((await adminPost(adminRequest(postBody({ action: "post", draftKey, lang: "en" })))).status).toBe(200);
|
||||
|
||||
const discard = await adminPost(adminRequest(postBody({ action: "discard", draftKey })));
|
||||
expect(discard.status).toBe(409);
|
||||
expect(isPublishedDigest(JSON.parse(kv.values.get(onlyKey(kv, "digest:weekly-"))!))).toBe(true);
|
||||
});
|
||||
|
||||
it("does not publish an edited digest's unedited model text", async () => {
|
||||
@@ -195,7 +245,12 @@ describe("weekly digest publication requires maintainer approval", () => {
|
||||
stubGitHub();
|
||||
const draftKey = onlyKey(kv, "draft:digest:");
|
||||
const res = await adminPost(adminRequest(postBody({ action: "post", draftKey, lang: "en", editedBody: "# Edited" })));
|
||||
await expect(res.json()).resolves.toMatchObject({ ok: true, published: false });
|
||||
// The maintainer is told why the digest is not on /digest.
|
||||
await expect(res.json()).resolves.toMatchObject({
|
||||
ok: true,
|
||||
published: false,
|
||||
warning: expect.stringContaining("/digest"),
|
||||
});
|
||||
expect(isPublishedDigest(JSON.parse(kv.values.get(onlyKey(kv, "digest:weekly-"))!))).toBe(false);
|
||||
});
|
||||
|
||||
@@ -237,8 +292,11 @@ describe("weekly digest publication requires maintainer approval", () => {
|
||||
it("hides legacy records that were never approved", () => {
|
||||
const legacy = { ...DIGEST_MODEL_OUTPUT, weekId: "2026-W01", generatedAt: "2026-01-05T00:00:00.000Z" };
|
||||
expect(isPublishedDigest(legacy)).toBe(false);
|
||||
expect(isPublishedDigest({ ...legacy, approved: true })).toBe(true);
|
||||
expect(isPublishedDigest({ ...legacy, approved: true, sections: [{ heading: "x", items: [1] }] })).toBe(false);
|
||||
expect(isPublishedDigest({ ...legacy, approved: true })).toBe(false);
|
||||
expect(isPublishedDigest({ ...legacy, approved: true, approvedLang: "en" })).toBe(true);
|
||||
expect(
|
||||
isPublishedDigest({ ...legacy, approved: true, approvedLang: "en", sections: [{ heading: "x", items: [1] }] })
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -287,12 +345,30 @@ describe("resolved drafts are not resurrected by the cron", () => {
|
||||
expect(res.status).toBe(200);
|
||||
|
||||
// Our own comment bumps updated_at past the draft's generatedAt.
|
||||
stubIssues("2099-01-01T00:00:00.000Z");
|
||||
stubIssues(new Date().toISOString());
|
||||
await runTriage({ CURATED_KV: kv, DEEPSEEK_API_KEY: "k" });
|
||||
expect(mocks.agentChat).not.toHaveBeenCalled();
|
||||
expect(JSON.parse(kv.values.get("draft:triage:42")!)).toMatchObject({ posted: true });
|
||||
});
|
||||
|
||||
it("drafts again when the item has new activity well after the maintainer's decision", async () => {
|
||||
const kv = new FakeKv();
|
||||
await saveDraft(kv, draft());
|
||||
useAdminEnv(kv);
|
||||
stubGitHub();
|
||||
expect((await adminPost(adminRequest(postBody({ action: "post", draftKey: "draft:triage:42" })))).status).toBe(200);
|
||||
|
||||
// New commits or a reply a day later.
|
||||
stubIssues(new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString());
|
||||
await expect(runTriage({ CURATED_KV: kv, DEEPSEEK_API_KEY: "k" })).resolves.toMatchObject({ processed: 1 });
|
||||
expect(JSON.parse(kv.values.get("draft:triage:42")!)).toMatchObject({ posted: false, bodyEn: "review" });
|
||||
|
||||
// The fresh draft can be posted.
|
||||
const posts = stubGitHub();
|
||||
expect((await adminPost(adminRequest(postBody({ action: "post", draftKey: "draft:triage:42" })))).status).toBe(200);
|
||||
expect(posts).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not let the dupes run overwrite a posted dupes draft", async () => {
|
||||
const kv = new FakeKv();
|
||||
kv.values.set("draft:dupes:7", JSON.stringify(draft({ id: "7", type: "dupes", targetNumber: 7, posted: true })));
|
||||
@@ -341,7 +417,7 @@ describe("admin post action is idempotent and bounded", () => {
|
||||
const kv = new FakeKv();
|
||||
await saveDraft(kv, draft());
|
||||
useAdminEnv(kv);
|
||||
stubGitHub(500);
|
||||
stubGitHub(422);
|
||||
|
||||
const failed = await adminPost(adminRequest(postBody({ action: "post", draftKey: "draft:triage:42" })));
|
||||
expect(failed.status).toBe(502);
|
||||
@@ -352,6 +428,41 @@ describe("admin post action is idempotent and bounded", () => {
|
||||
expect(posts).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([500, 502, 504, 429])(
|
||||
"keeps the claim when GitHub answers %i, since the post may have been created",
|
||||
async (status) => {
|
||||
const kv = new FakeKv();
|
||||
await saveDraft(kv, draft());
|
||||
useAdminEnv(kv);
|
||||
const posts = stubGitHub(status);
|
||||
|
||||
const failed = await adminPost(adminRequest(postBody({ action: "post", draftKey: "draft:triage:42" })));
|
||||
expect(failed.status).toBe(502);
|
||||
await expect(failed.json()).resolves.toMatchObject({ error: expect.stringContaining("check GitHub before retrying") });
|
||||
|
||||
stubGitHub();
|
||||
const retry = await adminPost(adminRequest(postBody({ action: "post", draftKey: "draft:triage:42" })));
|
||||
expect(retry.status).toBe(409);
|
||||
expect(posts).toHaveLength(1);
|
||||
}
|
||||
);
|
||||
|
||||
it("posts once when two requests for the same draft race", async () => {
|
||||
const kv = new FakeKv();
|
||||
await saveDraft(kv, draft());
|
||||
kv.yieldEachOp = true;
|
||||
useAdminEnv(kv);
|
||||
const posts = stubGitHub();
|
||||
|
||||
const results = await Promise.all([
|
||||
adminPost(adminRequest(postBody({ action: "post", draftKey: "draft:triage:42" }))),
|
||||
adminPost(adminRequest(postBody({ action: "post", draftKey: "draft:triage:42" }))),
|
||||
]);
|
||||
|
||||
expect(results.map((r) => r.status).sort()).toEqual([200, 409]);
|
||||
expect(posts).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("answers malformed JSON with a JSON 400", async () => {
|
||||
const kv = new FakeKv();
|
||||
useAdminEnv(kv);
|
||||
@@ -379,13 +490,36 @@ describe("admin post action is idempotent and bounded", () => {
|
||||
});
|
||||
|
||||
describe("admin draft queue", () => {
|
||||
it("lists every draft across KV list pages", async () => {
|
||||
const kv = new FakeKv();
|
||||
for (let i = 1; i <= 1_500; i++) {
|
||||
function fill(kv: FakeKv, count: number) {
|
||||
for (let i = 1; i <= count; i++) {
|
||||
kv.values.set(`draft:triage:${i}`, JSON.stringify(draft({ id: String(i), targetNumber: i })));
|
||||
}
|
||||
}
|
||||
|
||||
const drafts = await listDrafts(kv);
|
||||
expect(drafts).toHaveLength(1_500);
|
||||
it("follows the KV list cursor past the first page", async () => {
|
||||
const kv = new FakeKv();
|
||||
kv.pageSize = 100;
|
||||
fill(kv, 250);
|
||||
|
||||
await expect(listDrafts(kv)).resolves.toHaveLength(250);
|
||||
});
|
||||
|
||||
it("reads at most MAX_LISTED_DRAFTS drafts in one request", async () => {
|
||||
const kv = new FakeKv();
|
||||
fill(kv, MAX_LISTED_DRAFTS + 1_000);
|
||||
const get = vi.spyOn(kv, "get");
|
||||
|
||||
await expect(listDrafts(kv)).resolves.toHaveLength(MAX_LISTED_DRAFTS);
|
||||
expect(get).toHaveBeenCalledTimes(MAX_LISTED_DRAFTS);
|
||||
});
|
||||
|
||||
it("keeps the drafts already read when a later list page fails", async () => {
|
||||
const kv = new FakeKv();
|
||||
kv.pageSize = 100;
|
||||
kv.failListAtCursor = "200";
|
||||
fill(kv, 250);
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
await expect(listDrafts(kv)).resolves.toHaveLength(200);
|
||||
});
|
||||
});
|
||||
@@ -132,9 +132,9 @@ export async function runTriage(env: AgentEnv): Promise<Record<string, unknown>>
|
||||
generatedAt: new Date().toISOString(),
|
||||
posted: false,
|
||||
};
|
||||
await saveDraft(env.CURATED_KV, draft);
|
||||
await logUsage(env.CURATED_KV, usage.input, usage.output);
|
||||
processed++;
|
||||
if (await saveDraft(env.CURATED_KV, draft, issue.updated_at)) processed++;
|
||||
else skipped++;
|
||||
} catch {
|
||||
skipped++;
|
||||
}
|
||||
@@ -215,9 +215,9 @@ export async function runPrReview(env: AgentEnv): Promise<Record<string, unknown
|
||||
generatedAt: new Date().toISOString(),
|
||||
posted: false,
|
||||
};
|
||||
await saveDraft(env.CURATED_KV, draft);
|
||||
await logUsage(env.CURATED_KV, usage.input, usage.output);
|
||||
processed++;
|
||||
if (await saveDraft(env.CURATED_KV, draft, pr.updated_at)) processed++;
|
||||
else skipped++;
|
||||
} catch {
|
||||
skipped++;
|
||||
}
|
||||
@@ -282,9 +282,9 @@ export async function runStale(env: AgentEnv): Promise<Record<string, unknown>>
|
||||
generatedAt: new Date().toISOString(),
|
||||
posted: false,
|
||||
};
|
||||
await saveDraft(env.CURATED_KV, draft);
|
||||
await logUsage(env.CURATED_KV, usage.input, usage.output);
|
||||
processed++;
|
||||
if (await saveDraft(env.CURATED_KV, draft, issue.updated_at)) processed++;
|
||||
else skipped++;
|
||||
} catch {
|
||||
skipped++;
|
||||
}
|
||||
|
||||
+125
-21
@@ -278,13 +278,30 @@ export async function getAgentEnv(): Promise<CommunityAgentEnv> {
|
||||
* Persist a generated draft for maintainer review. Returns false without
|
||||
* writing when the maintainer already posted or discarded this draft
|
||||
* identity, so a cron run can never resurrect a resolved draft as pending.
|
||||
*
|
||||
* `sourceUpdatedAt` is the GitHub item's `updated_at` for drafts about an
|
||||
* issue or PR. Activity well after the maintainer's decision (new commits, a
|
||||
* reply) reopens the identity; see `resolutionCovers`.
|
||||
*/
|
||||
export async function saveDraft(kv: KVNamespace | undefined, draft: AgentDraft): Promise<boolean> {
|
||||
export async function saveDraft(
|
||||
kv: KVNamespace | undefined,
|
||||
draft: AgentDraft,
|
||||
sourceUpdatedAt?: string
|
||||
): Promise<boolean> {
|
||||
if (!kv) return false;
|
||||
const key = draftKey(draft.type, draft.id);
|
||||
if (await getDraftResolution(kv, draft.type, draft.id)) return false;
|
||||
const existing = await getDraft(kv, key);
|
||||
if (existing?.posted) return false;
|
||||
const resolution = await getDraftResolution(kv, draft.type, draft.id);
|
||||
if (resolution) {
|
||||
if (resolutionCovers(resolution, sourceUpdatedAt)) return false;
|
||||
// Reopened by new activity. Clear the marker first: if the put below
|
||||
// then fails, the old posted draft still blocks a redraft, which is the
|
||||
// safe side.
|
||||
await clearDraftResolution(kv, draft.type, draft.id);
|
||||
} else {
|
||||
// A posted draft without a marker predates the markers; keep it.
|
||||
const existing = await getDraft(kv, key);
|
||||
if (existing?.posted) return false;
|
||||
}
|
||||
await kv.put(key, JSON.stringify(draft), { expirationTtl: 60 * 60 * 24 * 30 }); // 30 days
|
||||
return true;
|
||||
}
|
||||
@@ -300,6 +317,8 @@ export type DraftResolutionState = "posting" | "posted" | "discarded";
|
||||
export interface DraftResolution {
|
||||
state: DraftResolutionState;
|
||||
at: string;
|
||||
/** Random token of the request holding a "posting" claim. */
|
||||
claim?: string;
|
||||
}
|
||||
|
||||
const RESOLUTION_PREFIX = "draft-resolved:";
|
||||
@@ -308,6 +327,25 @@ const RESOLUTION_TTL_SEC = 60 * 60 * 24 * 90; // 90 days
|
||||
// outcome (network error mid-request) blocks immediate retries without
|
||||
// locking the draft forever.
|
||||
const POSTING_CLAIM_TTL_SEC = 60 * 15;
|
||||
// Our own post bumps the GitHub item's updated_at moments before the marker
|
||||
// is written; only activity later than this after the decision reopens it.
|
||||
const RESOLUTION_REOPEN_GRACE_MS = 10 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* True while a maintainer decision still applies to the item as it is now.
|
||||
* A decision covers everything up to shortly after it was made; an item
|
||||
* updated later (new commits, a reply) can be drafted again. An in-flight
|
||||
* post, or an item without an update time (digests, dupes, content-watch
|
||||
* findings, whose identity already encodes their content), stays covered
|
||||
* for the marker's lifetime.
|
||||
*/
|
||||
export function resolutionCovers(resolution: DraftResolution, sourceUpdatedAt?: string): boolean {
|
||||
if (resolution.state === "posting" || !sourceUpdatedAt) return true;
|
||||
const decidedAt = Date.parse(resolution.at);
|
||||
const updatedAt = Date.parse(sourceUpdatedAt);
|
||||
if (!Number.isFinite(decidedAt) || !Number.isFinite(updatedAt)) return true;
|
||||
return updatedAt <= decidedAt + RESOLUTION_REOPEN_GRACE_MS;
|
||||
}
|
||||
|
||||
function resolutionKey(type: AgentDraftType, id: string): string {
|
||||
return RESOLUTION_PREFIX + draftKey(type, id).slice("draft:".length);
|
||||
@@ -324,7 +362,11 @@ export async function getDraftResolution(
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as Partial<DraftResolution>;
|
||||
if (parsed.state === "posting" || parsed.state === "posted" || parsed.state === "discarded") {
|
||||
return { state: parsed.state, at: typeof parsed.at === "string" ? parsed.at : "" };
|
||||
return {
|
||||
state: parsed.state,
|
||||
at: typeof parsed.at === "string" ? parsed.at : "",
|
||||
...(typeof parsed.claim === "string" ? { claim: parsed.claim } : {}),
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
/* fall through: treat an unreadable marker as present */
|
||||
@@ -345,6 +387,29 @@ export async function markDraftResolved(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Claim a draft before posting it to GitHub. Returns false when another
|
||||
* request holds or wins the claim.
|
||||
*
|
||||
* KV has no compare-and-set, so this is best effort, not a lock: it writes a
|
||||
* random token and reads it back, which stops concurrent posts that meet in
|
||||
* one location (two tabs, a double submit) but cannot fully order writers in
|
||||
* different edge locations, because KV is eventually consistent across them.
|
||||
*/
|
||||
export async function claimDraftForPosting(
|
||||
kv: KVNamespace | undefined,
|
||||
type: AgentDraftType,
|
||||
id: string
|
||||
): Promise<boolean> {
|
||||
if (!kv) return true;
|
||||
const key = resolutionKey(type, id);
|
||||
const claim = crypto.randomUUID();
|
||||
const value: DraftResolution = { state: "posting", at: new Date().toISOString(), claim };
|
||||
await kv.put(key, JSON.stringify(value), { expirationTtl: POSTING_CLAIM_TTL_SEC });
|
||||
const current = await getDraftResolution(kv, type, id);
|
||||
return current?.state === "posting" && current.claim === claim;
|
||||
}
|
||||
|
||||
export async function clearDraftResolution(
|
||||
kv: KVNamespace | undefined,
|
||||
type: AgentDraftType,
|
||||
@@ -358,7 +423,7 @@ export async function clearDraftResolution(
|
||||
//
|
||||
// The cron writes the structured digest unapproved; only the maintainer's
|
||||
// post action flips `approved`, and the public /digest page renders only
|
||||
// approved records.
|
||||
// approved records, in the one language the maintainer reviewed.
|
||||
|
||||
export const DIGEST_RECORD_PREFIX = "digest:weekly-";
|
||||
const DIGEST_RECORD_TTL_SEC = 60 * 60 * 24 * 90;
|
||||
@@ -373,6 +438,8 @@ export interface WeeklyDigestRecord {
|
||||
generatedAt: string;
|
||||
approved?: boolean;
|
||||
approvedAt?: string;
|
||||
/** The language the maintainer reviewed; the only one /digest shows. */
|
||||
approvedLang?: "en" | "zh";
|
||||
}
|
||||
|
||||
export function digestRecordKey(weekId: string): string {
|
||||
@@ -386,6 +453,7 @@ export function isPublishedDigest(value: unknown): value is WeeklyDigestRecord {
|
||||
const d = value as Record<string, unknown>;
|
||||
return (
|
||||
d.approved === true &&
|
||||
(d.approvedLang === "en" || d.approvedLang === "zh") &&
|
||||
typeof d.weekId === "string" &&
|
||||
typeof d.titleEn === "string" &&
|
||||
typeof d.titleZh === "string" &&
|
||||
@@ -403,14 +471,26 @@ export function isPublishedDigest(value: unknown): value is WeeklyDigestRecord {
|
||||
);
|
||||
}
|
||||
|
||||
/** Mark the stored digest for `weekId` approved. Returns false if it is gone. */
|
||||
export async function approveDigestRecord(kv: KVNamespace | undefined, weekId: string): Promise<boolean> {
|
||||
/**
|
||||
* Mark the stored digest for `weekId` approved in the language the maintainer
|
||||
* reviewed. Returns false if it is gone.
|
||||
*/
|
||||
export async function approveDigestRecord(
|
||||
kv: KVNamespace | undefined,
|
||||
weekId: string,
|
||||
lang: "en" | "zh"
|
||||
): Promise<boolean> {
|
||||
if (!kv) return false;
|
||||
const key = digestRecordKey(weekId);
|
||||
const raw = await kv.get(key);
|
||||
if (!raw) return false;
|
||||
const record = JSON.parse(raw) as WeeklyDigestRecord;
|
||||
const approved: WeeklyDigestRecord = { ...record, approved: true, approvedAt: new Date().toISOString() };
|
||||
const approved: WeeklyDigestRecord = {
|
||||
...record,
|
||||
approved: true,
|
||||
approvedAt: new Date().toISOString(),
|
||||
approvedLang: lang,
|
||||
};
|
||||
if (!isPublishedDigest(approved)) return false;
|
||||
await kv.put(key, JSON.stringify(approved), { expirationTtl: DIGEST_RECORD_TTL_SEC });
|
||||
return true;
|
||||
@@ -446,20 +526,42 @@ export async function getDraft(kv: KVNamespace | undefined, key: string): Promis
|
||||
}
|
||||
}
|
||||
|
||||
// One draft read is one KV operation, and a Worker invocation gets a bounded
|
||||
// number of them, so the admin queue reads at most this many drafts.
|
||||
export const MAX_LISTED_DRAFTS = 500;
|
||||
const DRAFT_READ_BATCH = 50;
|
||||
|
||||
/**
|
||||
* Read up to MAX_LISTED_DRAFTS drafts, following the KV list cursor and
|
||||
* reading drafts in parallel batches. A failure after the first list call
|
||||
* returns what was read so far instead of discarding it.
|
||||
*/
|
||||
export async function listDrafts(kv: KVNamespace | undefined, prefix = "draft:"): Promise<AgentDraft[]> {
|
||||
if (!kv) return [];
|
||||
const drafts: AgentDraft[] = [];
|
||||
let seen = 0;
|
||||
let cursor: string | undefined;
|
||||
// KV returns at most 1000 keys per call; follow the cursor so the admin
|
||||
// queue is never silently truncated. The page cap only bounds a runaway.
|
||||
for (let page = 0; page < 20; page++) {
|
||||
const listed = await kv.list({ prefix, limit: 1000, ...(cursor ? { cursor } : {}) });
|
||||
for (const k of listed.keys) {
|
||||
const draft = await getDraft(kv, k.name);
|
||||
if (draft) drafts.push(draft);
|
||||
try {
|
||||
while (seen < MAX_LISTED_DRAFTS) {
|
||||
const listed = await kv.list({
|
||||
prefix,
|
||||
limit: Math.min(1000, MAX_LISTED_DRAFTS - seen),
|
||||
...(cursor ? { cursor } : {}),
|
||||
});
|
||||
const names = listed.keys.map((k) => k.name).slice(0, MAX_LISTED_DRAFTS - seen);
|
||||
seen += names.length;
|
||||
for (let i = 0; i < names.length; i += DRAFT_READ_BATCH) {
|
||||
const batch = await Promise.all(
|
||||
names.slice(i, i + DRAFT_READ_BATCH).map((name) => getDraft(kv, name).catch(() => null))
|
||||
);
|
||||
for (const draft of batch) if (draft) drafts.push(draft);
|
||||
}
|
||||
if (names.length === 0 || listed.list_complete !== false || !listed.cursor) break;
|
||||
cursor = listed.cursor;
|
||||
}
|
||||
if (listed.list_complete !== false || !listed.cursor) break;
|
||||
cursor = listed.cursor;
|
||||
} catch (e) {
|
||||
if (seen === 0) throw e;
|
||||
console.error("listDrafts: returning a partial queue", e);
|
||||
}
|
||||
return drafts;
|
||||
}
|
||||
@@ -533,8 +635,8 @@ export async function logUsage(
|
||||
|
||||
/**
|
||||
* True when a generator should not spend a model call drafting this item:
|
||||
* the maintainer already posted or discarded it, or the stored draft is newer
|
||||
* than the item's last update.
|
||||
* a maintainer decision still covers it (see `resolutionCovers`), or the
|
||||
* stored draft is newer than the item's last update.
|
||||
*/
|
||||
export async function hasFreshDraft(
|
||||
kv: KVNamespace | undefined,
|
||||
@@ -544,9 +646,11 @@ export async function hasFreshDraft(
|
||||
): Promise<boolean> {
|
||||
if (!kv) return false;
|
||||
if (!AGENT_DRAFT_TYPE_SET.has(type)) return false;
|
||||
if (await getDraftResolution(kv, type as AgentDraftType, id)) return true;
|
||||
const resolution = await getDraftResolution(kv, type as AgentDraftType, id);
|
||||
if (resolution) return resolutionCovers(resolution, updatedAt);
|
||||
const existing = await getDraft(kv, draftKey(type as AgentDraftType, id));
|
||||
if (!existing) return false;
|
||||
// A posted draft without a marker predates the markers.
|
||||
if (existing.posted) return true;
|
||||
return new Date(existing.generatedAt) > new Date(updatedAt);
|
||||
}
|
||||
@@ -6,7 +6,7 @@ vi.mock("./community-agent", async (importOriginal) => {
|
||||
});
|
||||
|
||||
import { runLinkCheck, runSemanticDrift, watchDraftId } from "./content-watch";
|
||||
import { agentChat, draftStorageKey, type AgentDraft } from "./community-agent";
|
||||
import { agentChat, deleteDraft, draftStorageKey, markDraftResolved, type AgentDraft } from "./community-agent";
|
||||
|
||||
const agentChatMock = agentChat as Mock;
|
||||
|
||||
@@ -157,6 +157,19 @@ describe("runSemanticDrift draft identity", () => {
|
||||
expect(draftEntries(kv)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not count a finding the maintainer already discarded as drafted", async () => {
|
||||
mockDrifts([finding]);
|
||||
await runSemanticDrift(env());
|
||||
const [[key, draft]] = draftEntries(kv);
|
||||
await markDraftResolved(kv, "semantic-drift", draft.id, "discarded");
|
||||
await deleteDraft(kv, key);
|
||||
|
||||
const second = await runSemanticDrift(env());
|
||||
|
||||
expect(second).toEqual({ ok: true, drafted: 0 });
|
||||
expect(draftEntries(kv)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("creates a new draft when the finding's evidence changes", async () => {
|
||||
mockDrifts([finding]);
|
||||
await runSemanticDrift(env());
|
||||
|
||||
@@ -363,8 +363,8 @@ ${docsText}`;
|
||||
const existing = await getDraft(env.CURATED_KV, draftStorageKey(draft));
|
||||
if (existing) continue;
|
||||
|
||||
await saveDraft(env.CURATED_KV, draft);
|
||||
drafted++;
|
||||
// saveDraft refuses findings the maintainer already discarded.
|
||||
if (await saveDraft(env.CURATED_KV, draft)) drafted++;
|
||||
}
|
||||
|
||||
return { ok: true, drafted };
|
||||
|
||||
Reference in new issue
Block a user