seo: date sitemap entries by body changes, not frontmatter sweeps (#118)

Follow-up to #117. That PR got `<lastmod>` publishing at all; this one
makes the dates worth resubmitting.

## The problem

**190 of 193 pages read `2026-08-25`** — the date of #115, which added
`seo_title`/`seo_description` to all 192 pages. Here is everything it
changed in a representative file:

```diff
+---
+seo_title: "Key Concepts: Companies, Agents, Tasks"
+seo_description: The seven ideas behind Paperclip — company, agent, task...
+---
+
 # Key Concepts
```

Not a word of prose. `lastmod` is meant to carry the date a page's
**content** last changed, so resubmitting this hands Search Console the
"whole site changed at once" signal that `dropUniformLastmod` and the
shallow-checkout guard both exist to prevent. It cleared
`dropUniformLastmod` only because 3 pages happened to differ.

## The fix

Walk each file's history and report the newest commit that changed the
**rendered body** — comparing `parseFrontmatter().body` across
consecutive commits, so frontmatter-only commits are skipped. Reusing
the build's own parser means the body compared is exactly the body the
site renders.

| | Before | After |
|---|---|---|
| Distinct dates | **2** | **31** |
| Largest single-date cluster | 190 pages | 54 pages |
| Range | 2 days | 2026-04-17 → 2026-08-27 |

Top of the new distribution:

| Date | Pages |
|---|---|
| 2026-08-18 | 54 |
| 2026-06-04 | 21 |
| 2026-08-25 | 17 |
| 2026-07-08 | 16 |
| 2026-07-20 | 15 |
| 2026-04-22 | 12 |

Spot-checked both directions:

- `key-concepts.md` — last touched by the frontmatter sweep, now dates
to **2026-07-08**, the commit that changed "Claude Local" to "Claude
Code" in the body.
- `day-to-day/issues.md` — **keeps 2026-08-25**, because release commit
`6c4519a` that day added a real "Who may resolve a card" section. Its
frontmatter-only `paperclip_version` bump in the same release was
correctly skipped.

## Two supporting fixes

**Concurrency.** ~600 git round-trips in series cost 40s. Resolving them
in parallel: **11s**.

**A race the concurrency exposed** — and the reason this is worth
reading closely. `hasTrustworthyGitHistory()` memoised its *resolved
value*, so once all 192 pages asked at once, every one of them passed
the guard before the first resolved:

```
Could not deepen the shallow checkout: fatal: Unable to create '.git/shallow.lock': File exists.
Still shallow: omitting sitemap <lastmod> rather than dating every page alike.
   × 192
```

192 concurrent `git fetch --unshallow` calls fighting over the lock, and
**a sitemap with no dates at all**. This only reproduces on a shallow
clone — which is to say only on Cloudflare Pages, never locally or in
`docs-test`. Fixed by memoising the promise (single-flight).

## Verification

| Scenario | Result |
|---|---|
| Full clone | 193 dates, 31 distinct, 11s |
| `git clone --depth 1` (reproduces CF) | one unshallow, **193 dates, 31
distinct** |
| Shallow + unreachable remote | no dates, exit 0 |
| `npm run docs:test` (6 suites) | pass |
| `npm run sync:test` | 39 pass, 0 fail (+1 new) |

New unit test covers the `--follow --name-status` rename parsing — 55
docs files have been renamed, and reading the wrong field silently drops
their pre-rename history.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 5 (1M context) <[email protected]>

scripts/sync/test.mjs

@@ -10,7 +10,7 @@ import { tmpdir } from "node:os";
 import { join, dirname, resolve } from "node:path";
 import { spawnSync } from "node:child_process";
 import { fileURLToPath } from "node:url";
-import { isPathInside, parseFrontmatter } from "../../site/build-release.mjs";
+import { isPathInside, parseFollowLog, parseFrontmatter } from "../../site/build-release.mjs";
 import { instanceEnv } from "../screenshots/config.mjs";
 
 const SELF_DIR = dirname(fileURLToPath(import.meta.url));
@@ -1110,6 +1110,36 @@ test("frontmatter: malformed (missing closing fence) falls back to full body", (
   assert(Object.keys(frontmatter).length === 0, `frontmatter should be empty on malformed input, got: ${JSON.stringify(frontmatter)}`);
 });
 
+test("build-release: follow-log parser reads the path as of each commit across renames", () => {
+  // `git log --follow --name-status --format='C %H %cs'`: a rename lists both
+  // paths, and the one to hand `git show` is the path at that commit (the new
+  // one). Getting this wrong silently drops pre-rename history for the 55 docs
+  // files that have been moved.
+  const stdout = [
+    "C 325c9eb819b32c292a1453ffcc1ee7cb98848710 2026-08-25",
+    "",
+    "M\tdocs/guides/welcome/key-concepts.md",
+    "C 56b8d7d52de2c0410b16dd0a50e6a5f2990df1aa 2026-04-22",
+    "",
+    "R095\tdocs/user-guides/guides/key-concepts.md\tdocs/guides/welcome/key-concepts.md",
+    "C 3a852ec7e939280e614260a1954f24146001e0dc 2026-04-17",
+    "",
+    "A\tdocs/user-guides/guides/key-concepts.md",
+    "",
+  ].join("\n");
+  const commits = parseFollowLog(stdout);
+  assert(commits.length === 3, `expected 3 commits, got ${commits.length}`);
+  assert(commits[0].date === "2026-08-25", `first date wrong: ${commits[0].date}`);
+  assert(
+    commits[1].path === "docs/guides/welcome/key-concepts.md",
+    `rename commit must use the post-rename path, got: ${commits[1].path}`,
+  );
+  assert(
+    commits[2].path === "docs/user-guides/guides/key-concepts.md",
+    `pre-rename commit must keep the old path, got: ${commits[2].path}`,
+  );
+});
+
 test("build-release: path containment rejects sibling docs-prefixed directories", () => {
   const parent = resolve("/tmp/repo/docs");
   assert(isPathInside(parent, resolve("/tmp/repo/docs/guide/page.md")), "normal child path should be inside");

site/build-release.mjs

@@ -621,10 +621,21 @@ function injectSeo(html, metadata, { baseHref = null } = {}) {
 }
 
 async function pageMetadataForNav(nav, outDir, siteUrl, basePath) {
+  const navPages = [...flattenNavPages(nav)];
+  // Each page's date costs a `git log` plus a few `git show` calls, so resolving
+  // them one page at a time leaves the build waiting on ~600 round-trips in
+  // series. They are independent — start them all, then read them off below.
+  const lastmods = new Map(
+    await Promise.all(
+      navPages.map(async ({ page }) => [
+        page.file,
+        await gitLastModified(path.join(docsRoot, page.file)),
+      ]),
+    ),
+  );
   const pages = [];
-  for (const { page, section } of flattenNavPages(nav)) {
+  for (const { page, section } of navPages) {
     const releaseMarkdownPath = path.join(outDir, page.file);
-    const sourceMarkdownPath = path.join(docsRoot, page.file);
     const markdown = await fs.readFile(releaseMarkdownPath, "utf8");
     const h1 = markdown.match(/^#\s+(.+)$/m)?.[1]?.trim();
     // Authored `seo_title` / `seo_description` frontmatter wins. The sidebar
@@ -639,7 +650,7 @@ async function pageMetadataForNav(nav, outDir, siteUrl, basePath) {
       title: `${pageTitle} | Paperclip Docs`,
       description: authoredDescription || markdownDescription(markdown),
       url: routeUrlForPage(siteUrl, basePath, page),
-      lastmod: await gitLastModified(sourceMarkdownPath),
+      lastmod: lastmods.get(page.file),
       siteUrl,
       basePath,
     });
@@ -684,8 +695,16 @@ async function deepenCheckout() {
   }
 }
 
-async function hasTrustworthyGitHistory() {
-  if (trustworthyGitHistory !== null) return trustworthyGitHistory;
+// Memoise the *promise*, not the resolved value. Every page asks this question
+// at once, and caching only the answer lets all of them past the guard before
+// the first has finished — which means 192 concurrent `git fetch --unshallow`
+// calls fighting over .git/shallow.lock, and no dates at all.
+function hasTrustworthyGitHistory() {
+  trustworthyGitHistory ??= resolveTrustworthyGitHistory();
+  return trustworthyGitHistory;
+}
+
+async function resolveTrustworthyGitHistory() {
   try {
     let usable = !(await isShallowRepository());
     if (!usable) {
@@ -697,11 +716,55 @@ async function hasTrustworthyGitHistory() {
           : "Still shallow: omitting sitemap <lastmod> rather than dating every page alike.",
       );
     }
-    trustworthyGitHistory = usable;
+    return usable;
   } catch {
-    trustworthyGitHistory = false;
+    return false;
   }
-  return trustworthyGitHistory;
+}
+
+// `lastmod` is meant to carry the date the page's *content* last changed, and
+// a metadata sweep is not that: #115 rewrote seo_title/seo_description on all
+// 192 pages in one commit without touching a word of prose. Dating every page
+// by that commit hands Search Console the "whole site changed at once" signal
+// the rest of this file works to avoid — so walk the file's history and report
+// the newest commit that actually changed the rendered body.
+const GIT_MAX_BUFFER = 64 * 1024 * 1024;
+
+function renderedBody(source) {
+  return parseFrontmatter(source).body.trim();
+}
+
+async function fileAtCommit(sha, repoRelativePath) {
+  try {
+    const { stdout } = await execFileAsync("git", ["show", `${sha}:${repoRelativePath}`], {
+      cwd: repoRoot,
+      maxBuffer: GIT_MAX_BUFFER,
+    });
+    return stdout;
+  } catch {
+    return null;
+  }
+}
+
+// `--follow` reports each commit under the path the file carried *at that*
+// commit, so renames have to be read back out of the status line to know what
+// to hand `git show`: `M<TAB>path` for an edit, `R095<TAB>old<TAB>new` for a
+// rename. The path as of the commit is the last field either way. 55 of the
+// docs files have a rename in their history, so this is not a rare branch.
+export function parseFollowLog(stdout) {
+  const commits = [];
+  let pending = null;
+  for (const line of stdout.split("\n")) {
+    const header = /^C ([0-9a-f]{40}) (\d{4}-\d{2}-\d{2})$/.exec(line);
+    if (header) {
+      pending = { sha: header[1], date: header[2], path: null };
+      commits.push(pending);
+      continue;
+    }
+    if (!pending || pending.path || !line.includes("\t")) continue;
+    pending.path = line.split("\t").at(-1).trim() || null;
+  }
+  return commits.filter((commit) => commit.path);
 }
 
 async function gitLastModified(sourcePath) {
@@ -711,11 +774,23 @@ async function gitLastModified(sourcePath) {
   try {
     const { stdout } = await execFileAsync(
       "git",
-      ["log", "-1", "--follow", "--format=%cs", "--", repoRelativePath],
-      { cwd: repoRoot },
+      ["log", "--follow", "--name-status", "--format=C %H %cs", "--", repoRelativePath],
+      { cwd: repoRoot, maxBuffer: GIT_MAX_BUFFER },
     );
-    const value = stdout.trim();
-    return /^\d{4}-\d{2}-\d{2}$/.test(value) ? value : undefined;
+    const commits = parseFollowLog(stdout);
+    if (!commits.length) return undefined;
+    let newer = await fileAtCommit(commits[0].sha, commits[0].path);
+    if (newer === null) return commits[0].date;
+    for (let index = 0; index < commits.length - 1; index += 1) {
+      const older = await fileAtCommit(commits[index + 1].sha, commits[index + 1].path);
+      // A commit we cannot read across is a boundary, not a body change we can
+      // rule out — treat it as the point this content came into existence.
+      if (older === null) return commits[index].date;
+      if (renderedBody(older) !== renderedBody(newer)) return commits[index].date;
+      newer = older;
+    }
+    // Every commit since creation only ever touched frontmatter.
+    return commits.at(-1).date;
   } catch {
     // An omitted lastmod is safer than publishing the build timestamp as if
     // every document changed at once.