Stop publishing the deploy date as every document's sitemap lastmod

Cloudflare Pages builds from a shallow clone. A shallow checkout does not make
`git log -1 -- <file>` fail, it makes every file report the single available
commit, so the existing try/catch fail-safe never fired and all 192 sitemap URLs
shipped the same false <lastmod>. Detect a shallow repository up front, try to
recover real history, and omit lastmod entirely when it cannot be established.

Co-Authored-By: Paperclip <[email protected]>

package.json

@@ -7,6 +7,7 @@
     "docs:build:github-subpath": "node site/build-release.mjs --base-path /paperclip-docs/ --out-dir .site",
     "docs:build:auto": "node site/build-release.mjs --base-path auto --out-dir .site",
     "docs:test:static-routes": "node scripts/verify-static-routes.mjs",
+    "docs:test:sitemap-lastmod-history": "node scripts/verify-sitemap-lastmod-history.mjs",
     "docs:test:asset-fingerprints": "node scripts/verify-asset-fingerprints.mjs",
     "docs:test:hierarchical-skills-nav": "node scripts/verify-hierarchical-skills-nav.mjs",
     "docs:test:skill-source-blocks": "node scripts/verify-skill-source-blocks.mjs",

scripts/verify-sitemap-lastmod-history.mjs

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+// Proves the sitemap never publishes a uniform build date as a per-document
+// lastmod. A shallow checkout does not make `git log` fail, so the only way to
+// catch the regression is to build from a real shallow clone.
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, rmSync, symlinkSync, existsSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
+
+let failures = 0;
+function assert(condition, message) {
+  if (condition) {
+    console.log(`  ok  ${message}`);
+    return;
+  }
+  failures += 1;
+  console.error(`FAIL  ${message}`);
+}
+
+function git(args, cwd) {
+  return execFileSync("git", args, { cwd, encoding: "utf8" }).trim();
+}
+
+function locs(sitemap) {
+  return [...sitemap.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
+}
+
+function lastmods(sitemap) {
+  return [...sitemap.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map((m) => m[1]);
+}
+
+function buildShallowClone({ label, reachableOrigin }) {
+  const workDir = mkdtempSync(join(tmpdir(), "docs-lastmod-"));
+  const clone = join(workDir, "clone");
+  try {
+    const branch = git(["rev-parse", "--abbrev-ref", "HEAD"], repoRoot);
+    git(["clone", "--quiet", "--depth", "1", "--no-local", "--branch", branch, `file://${repoRoot}`, clone], workDir);
+    if (!reachableOrigin) {
+      git(["remote", "set-url", "origin", "file:///nonexistent/unreachable.git"], clone);
+    }
+
+    assert(git(["rev-parse", "--is-shallow-repository"], clone) === "true", `${label}: fixture clone is shallow`);
+    symlinkSync(join(repoRoot, "node_modules"), join(clone, "node_modules"));
+
+    execFileSync("node", ["site/build-release.mjs", "--base-path", "/", "--out-dir", ".site"], {
+      cwd: clone,
+      stdio: "pipe",
+    });
+
+    const sitemapPath = join(clone, ".site", "sitemap.xml");
+    assert(existsSync(sitemapPath), `${label}: sitemap.xml was generated`);
+    return readFileSync(sitemapPath, "utf8");
+  } finally {
+    rmSync(workDir, { recursive: true, force: true });
+  }
+}
+
+console.log("Case 1 — full history build publishes real per-document dates");
+{
+  const outDir = join(repoRoot, ".site");
+  assert(
+    existsSync(join(outDir, "sitemap.xml")),
+    "run `npm run docs:build` before this check so .site/sitemap.xml exists",
+  );
+  if (existsSync(join(outDir, "sitemap.xml"))) {
+    const sitemap = readFileSync(join(outDir, "sitemap.xml"), "utf8");
+    const urls = locs(sitemap);
+    const dates = lastmods(sitemap);
+    assert(urls.length > 0, `sitemap lists routes (${urls.length})`);
+    assert(dates.length === urls.length, `every route carries a lastmod (${dates.length}/${urls.length})`);
+    assert(
+      new Set(dates).size > 1,
+      `lastmod values vary across documents (${new Set(dates).size} distinct) rather than one build date`,
+    );
+  }
+}
+
+console.log("Case 2 — shallow clone that can reach origin recovers real history");
+{
+  const sitemap = buildShallowClone({ label: "reachable", reachableOrigin: true });
+  const urls = locs(sitemap);
+  const dates = lastmods(sitemap);
+  assert(dates.length === urls.length, `unshallowed build dates every route (${dates.length}/${urls.length})`);
+  assert(new Set(dates).size > 1, `unshallowed build produces varied dates (${new Set(dates).size} distinct)`);
+}
+
+console.log("Case 3 — shallow clone with unreachable origin omits dates instead of lying");
+{
+  const sitemap = buildShallowClone({ label: "unreachable", reachableOrigin: false });
+  const urls = locs(sitemap);
+  const dates = lastmods(sitemap);
+  assert(urls.length > 0, `routes are still published (${urls.length})`);
+  assert(dates.length === 0, `no lastmod is published when history is incomplete (found ${dates.length})`);
+  assert(!sitemap.includes("<lastmod>"), "sitemap contains no lastmod element at all");
+  assert(/^<\?xml/.test(sitemap.trim()) && sitemap.includes("</urlset>"), "sitemap remains well-formed XML");
+}
+
+if (failures > 0) {
+  console.error(`\nSitemap lastmod history verification failed (${failures} assertion(s)).`);
+  process.exit(1);
+}
+console.log("\nSitemap lastmod history verification passed.");

scripts/verify-static-routes.mjs

@@ -144,7 +144,12 @@ try {
   assert(sitemap.includes("https://docs.paperclip.ing/reference/skills"), "sitemap is missing reference/skills");
   assert(!sitemap.includes("<changefreq>"), "sitemap should not publish ignored changefreq values");
   assert(!sitemap.includes("<priority>"), "sitemap should not publish ignored priority values");
+  const sitemapUrls = [...sitemap.matchAll(/<loc>([^<]+)<\/loc>/g)].map((match) => match[1]);
   const sitemapLastmods = [...sitemap.matchAll(/<lastmod>([^<]+)<\/lastmod>/g)].map((match) => match[1]);
+  assert(
+    sitemapLastmods.length === sitemapUrls.length,
+    "a complete Git checkout should publish one content-history lastmod per sitemap URL",
+  );
   assert(new Set(sitemapLastmods).size > 1, "sitemap lastmod values should reflect document history, not one build date");
   assert(
     existsSync(join(outDir, "reference/skills/bundled/index.html")),

site/build-release.mjs

@@ -621,6 +621,7 @@ function injectSeo(html, metadata, { baseHref = null } = {}) {
 }
 
 async function pageMetadataForNav(nav, outDir, siteUrl, basePath) {
+  const hasCompleteGitHistory = await ensureCompleteGitHistory();
   const pages = [];
   for (const { page, section } of flattenNavPages(nav)) {
     const releaseMarkdownPath = path.join(outDir, page.file);
@@ -634,7 +635,7 @@ async function pageMetadataForNav(nav, outDir, siteUrl, basePath) {
       title: `${pageTitle} | Paperclip Docs`,
       description: markdownDescription(markdown),
       url: routeUrlForPage(siteUrl, basePath, page),
-      lastmod: await gitLastModified(sourceMarkdownPath),
+      lastmod: await gitLastModified(sourceMarkdownPath, { hasCompleteGitHistory }),
       siteUrl,
       basePath,
     });
@@ -642,7 +643,32 @@ async function pageMetadataForNav(nav, outDir, siteUrl, basePath) {
   return pages;
 }
 
-async function gitLastModified(sourcePath) {
+// A shallow checkout does not make `git log` fail; it makes every file report the
+// single available commit. Detecting that up front is the only way to avoid
+// publishing the deploy date as if every document changed at once.
+async function ensureCompleteGitHistory() {
+  try {
+    const { stdout } = await execFileAsync("git", ["rev-parse", "--is-shallow-repository"], { cwd: repoRoot });
+    if (stdout.trim() !== "true") return true;
+
+    await execFileAsync("git", ["fetch", "--quiet", "--unshallow", "--no-tags", "origin"], {
+      cwd: repoRoot,
+      timeout: 30_000,
+    });
+    const { stdout: updatedState } = await execFileAsync("git", ["rev-parse", "--is-shallow-repository"], {
+      cwd: repoRoot,
+    });
+    if (updatedState.trim() !== "true") return true;
+  } catch (error) {
+    console.warn(`Could not load complete Git history for sitemap lastmod values: ${error.message}`);
+  }
+
+  console.warn("Omitting sitemap lastmod values because Git history is incomplete.");
+  return false;
+}
+
+async function gitLastModified(sourcePath, { hasCompleteGitHistory }) {
+  if (!hasCompleteGitHistory) return undefined;
   if (!isPathInside(repoRoot, sourcePath)) return undefined;
   const repoRelativePath = toPosixPath(path.relative(repoRoot, sourcePath));
   try {