From ce1381617993ec60aa70ee8a9c16bac2d8ccb162 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 6 Jul 2026 20:13:40 -0500 Subject: [PATCH] [codex] Fix docs SEO crawl hygiene (#46) ## Summary Fixes the docs-side causes behind the Google Search Console duplicate/canonical SEO issues by tightening generated static routing behavior. ## What changed - Removed the Cloudflare Pages wildcard `/* /index.html 200` SPA fallback from the generated docs release so deleted or unknown asset URLs can return 404 instead of serving docs HTML. - Removed the global `X-Robots-Tag: index, follow` header and replaced it with noindex coverage for non-page static assets, `sitemap.xml`, `robots.txt`, and the generated `404.html`. - Added legacy route handling for removed docs URLs and extended static route verification for redirects, removed assets, noindex headers, and 404 behavior. ## Root cause The generated Cloudflare Pages release treated too many non-page/static URLs as indexable page responses. That allowed removed assets or auxiliary files to be crawled as duplicate indexable documents instead of clear 404/noindex responses. ## Validation - `npm run docs:test:static-routes` - `npm run docs:build` PAP-12839 --------- Co-authored-by: DevRel Gemini Co-authored-by: Paperclip --- scripts/verify-static-routes.mjs | 44 +++++++++++++++- site/build-release.mjs | 89 ++++++++++++++++++++++++++++++-- site/redirects.json | 20 ++++++- 3 files changed, 145 insertions(+), 8 deletions(-) diff --git a/scripts/verify-static-routes.mjs b/scripts/verify-static-routes.mjs index c5307b5..ecfb4d4 100644 --- a/scripts/verify-static-routes.mjs +++ b/scripts/verify-static-routes.mjs @@ -111,11 +111,51 @@ try { "Cloudflare redirects are missing the no-slash canonical redirect for the quickstart path", ); assert( - redirects.indexOf(canonicalRedirect) < redirects.indexOf("/* /index.html 200"), - "canonical route redirects must appear before the SPA fallback rewrite", + !redirects.includes("/* /index.html 200"), + "Cloudflare redirects must not rewrite unknown URLs and removed assets to the docs shell", + ); + + const legacyRedirects = JSON.parse(readFileSync(join(root, "site/redirects.json"), "utf8")); + for (const [sourceRoute, destinationRoute] of Object.entries(legacyRedirects)) { + const destinationIndexPath = join(outDir, destinationRoute, "index.html"); + assert( + existsSync(destinationIndexPath), + `legacy redirect target is not a generated page: ${sourceRoute} -> ${destinationRoute}`, + ); + const noSlashRedirect = `/${sourceRoute} /${destinationRoute}/ 301`; + const slashRedirect = `/${sourceRoute}/ /${destinationRoute}/ 301`; + assert(redirects.includes(noSlashRedirect), `missing legacy no-slash redirect: ${noSlashRedirect}`); + assert(redirects.includes(slashRedirect), `missing legacy slash redirect: ${slashRedirect}`); + } + assert( + redirects.includes("/guides/agent-developer/how-agents-work /guides/org/agents/ 301"), + "Search Console legacy agent-developer URL should redirect to the current agents guide", + ); + assert( + redirects.includes("/start/quickstart/ /guides/getting-started/five-minute-path/ 301"), + "Search Console legacy quickstart URL should redirect to the current quickstart", ); const headers = read("_headers"); + assert(!headers.includes("X-Robots-Tag: index, follow"), "headers must not globally mark every static file indexable"); + assert( + headers.includes("/*.css\n X-Robots-Tag: noindex, nofollow"), + "CSS assets should be marked noindex in Cloudflare headers", + ); + assert( + headers.includes("/*.js\n X-Robots-Tag: noindex, nofollow"), + "JS assets should be marked noindex in Cloudflare headers", + ); + assert( + headers.includes("/*.md\n X-Robots-Tag: noindex, nofollow"), + "copied markdown source files should be marked noindex in Cloudflare headers", + ); + assert( + headers.includes("/sitemap.xml\n Content-Type: application/xml; charset=utf-8\n X-Robots-Tag: noindex, nofollow"), + "sitemap.xml should be served as XML and excluded from search results", + ); + const notFoundHtml = read("404.html"); + assert(notFoundHtml.includes(''), "404 page must be noindex"); assert( headers.includes("Strict-Transport-Security: max-age=31536000; includeSubDomains; preload"), "Cloudflare headers are missing HSTS", diff --git a/site/build-release.mjs b/site/build-release.mjs index 38316d2..61aeb73 100644 --- a/site/build-release.mjs +++ b/site/build-release.mjs @@ -13,6 +13,7 @@ const sourceIndexPath = path.join(__dirname, "index.html"); const sourceStylesPath = path.join(__dirname, "styles.css"); const sourceAppJsPath = path.join(__dirname, "app.js"); const sourceNavPath = path.join(__dirname, "content.json"); +const sourceRedirectsPath = path.join(__dirname, "redirects.json"); const sourceVendorDir = path.join(__dirname, "vendor"); const screenshotsSourceDir = path.join(docsRoot, "user-guides", "screenshots"); const defaultSiteUrl = "https://docs.paperclip.ing"; @@ -655,7 +656,18 @@ function cloudflarePathForRoute(basePath, routePath, { trailingSlash = false } = return `/${key}${trailingSlash ? "/" : ""}`; } -function buildCloudflareRedirects({ basePath, pages }) { +function cloudflareRedirectLine(basePath, sourceRoute, destinationRoute) { + const sourcePath = cloudflarePathForRoute(basePath, sourceRoute); + const sourceSlashPath = cloudflarePathForRoute(basePath, sourceRoute, { trailingSlash: true }); + const destinationPath = cloudflarePathForRoute(basePath, destinationRoute, { trailingSlash: true }); + if (sourceSlashPath === destinationPath) return []; + return [ + `${sourcePath} ${destinationPath} 301`, + `${sourceSlashPath} ${destinationPath} 301`, + ]; +} + +function buildCloudflareRedirects({ basePath, pages, legacyRedirects = {} }) { const routeRedirects = pages .map(({ page }) => { const sourcePath = cloudflarePathForRoute(basePath, page.slug); @@ -663,19 +675,24 @@ function buildCloudflareRedirects({ basePath, pages }) { return `${sourcePath} ${destinationPath} 301`; }) .join("\n"); + const legacyRouteRedirects = Object.entries(legacyRedirects) + .flatMap(([sourceRoute, destinationRoute]) => + cloudflareRedirectLine(basePath, sourceRoute, destinationRoute) + ) + .join("\n"); return `# Canonical docs URLs include trailing slashes. Keep no-slash requests # on a normal one-hop 301 instead of Cloudflare Pages' implicit directory 308. ${routeRedirects} -# Serve generated files first; fall back to the SPA shell for client routes. -/* /index.html 200 +# Legacy docs URLs moved during the information architecture cleanup. Redirect +# them before unknown URLs fall through to 404 so crawlers see one canonical URL per page. +${legacyRouteRedirects} `; } function buildCloudflareHeaders() { return `/* - X-Robots-Tag: index, follow Referrer-Policy: strict-origin-when-cross-origin X-Content-Type-Options: nosniff Strict-Transport-Security: max-age=31536000; includeSubDomains; preload @@ -685,9 +702,69 @@ function buildCloudflareHeaders() { /sitemap.xml Content-Type: application/xml; charset=utf-8 + X-Robots-Tag: noindex, nofollow /robots.txt Content-Type: text/plain; charset=utf-8 + X-Robots-Tag: noindex, nofollow + +/*.css + X-Robots-Tag: noindex, nofollow + +/*.js + X-Robots-Tag: noindex, nofollow + +/*.json + X-Robots-Tag: noindex, nofollow + +/*.md + X-Robots-Tag: noindex, nofollow + +/*.png + X-Robots-Tag: noindex, nofollow + +/*.jpg + X-Robots-Tag: noindex, nofollow + +/*.jpeg + X-Robots-Tag: noindex, nofollow + +/*.webp + X-Robots-Tag: noindex, nofollow + +/*.svg + X-Robots-Tag: noindex, nofollow + +/*.txt + X-Robots-Tag: noindex, nofollow +`; +} + +function buildNotFoundPage(siteUrl, basePath) { + const metadata = { + title: "Not found | Paperclip Docs", + description: "This Paperclip Docs URL does not exist.", + url: siteUrlForPath(siteUrl, basePath, "404.html"), + siteUrl, + basePath, + }; + return ` + + + + + ${escapeHtml(metadata.title)} + + + + +
+

Not found

+

This Paperclip Docs URL does not exist.

+

Open the docs home page

+
+ + `; } @@ -909,6 +986,7 @@ ${basePathGuidance} - Serve the bundle root at \`${deploymentBasePath}\` - Keep all copied files together so requests for \`content.json\`, markdown files, images, fonts, and JS resolve normally - Serve generated files such as \`sitemap.xml\`, \`robots.txt\`, and nested route directories unchanged +- Do not add a wildcard SPA rewrite such as \`/* /index.html 200\`; unknown URLs and removed assets must return 404 so crawlers do not treat them as duplicate docs pages If \`content.json\` or linked markdown files are missing from the uploaded bundle, the docs app will fail to load content. @@ -948,6 +1026,7 @@ async function minifyJs(source) { async function main() { const options = parseArgs(process.argv.slice(2)); const sourceNav = JSON.parse(await fs.readFile(sourceNavPath, "utf8")); + const sourceRedirects = JSON.parse(await fs.readFile(sourceRedirectsPath, "utf8")); const releaseNav = attachSlugs(rewriteNav(sourceNav)); const { markdownFiles, warnings } = await collectReleaseFiles(sourceNav); @@ -968,6 +1047,7 @@ async function main() { await fs.writeFile(path.join(options.outDir, "nginx.conf.example"), buildNginxConfig(options.basePath)); await fs.writeFile(path.join(options.outDir, "DEPLOY.md"), buildDeployGuide(options.basePath)); await fs.writeFile(path.join(options.outDir, "_headers"), buildCloudflareHeaders()); + await fs.writeFile(path.join(options.outDir, "404.html"), buildNotFoundPage(options.siteUrl, options.basePath)); // Copy markdown files, stripping YAML frontmatter, and collect per-file // frontmatter to surface via content.json (keyed by repo-relative path). @@ -997,6 +1077,7 @@ async function main() { await fs.writeFile(path.join(options.outDir, "_redirects"), buildCloudflareRedirects({ basePath: options.basePath, pages: pageMetadata, + legacyRedirects: sourceRedirects, })); const rootMetadata = { title: "Paperclip Docs", diff --git a/site/redirects.json b/site/redirects.json index 50c66c2..6e77004 100644 --- a/site/redirects.json +++ b/site/redirects.json @@ -14,7 +14,16 @@ "guides/day-to-day/costs-and-budgets": "guides/day-to-day/costs", "user-guides/guides/activity-log": "guides/day-to-day/activity-log", "user-guides/guides/feedback-voting": "guides/day-to-day/feedback-voting", - "guides/agent-developer/how-agents-work": "guides/org/how-agents-work", + "guides/board-operator/dashboard": "guides/day-to-day/dashboard", + "guides/board-operator/managing-tasks": "guides/day-to-day/issues", + "guides/board-operator/managing-agents": "guides/org/agents", + "guides/board-operator/org-structure": "guides/org/org-structure", + "guides/board-operator/creating-a-company": "guides/getting-started/your-first-company", + "guides/agent-developer/how-agents-work": "guides/org/agents", + "guides/agent-developer/comments-and-communication": "guides/day-to-day/issues", + "guides/agent-developer/task-workflow": "guides/day-to-day/issues", + "guides/agent-developer/handling-approvals": "guides/day-to-day/approvals", + "guides/agent-developer/cost-reporting": "guides/day-to-day/costs", "user-guides/guides/skills": "guides/org/skills", "user-guides/guides/org-structure": "guides/org/org-structure", "user-guides/guides/delegation": "guides/org/delegation", @@ -29,6 +38,13 @@ "user-guides/guides/administration/company": "administration/company", "user-guides/guides/administration/settings": "administration/settings", "user-guides/guides/administration/plugins": "administration/plugins", + "start/quickstart": "guides/getting-started/five-minute-path", + "start/core-concepts": "guides/welcome/key-concepts", + "tutorials": "guides/getting-started/five-minute-path", + "how-to": "how-to/enable-multi-user-login", + "glossary": "guides/welcome/glossary", + "reference": "reference/api/overview", + "explanation": "guides/welcome/what-is-paperclip", "api/overview": "reference/api/overview", "api/authentication": "reference/api/authentication", "api/companies": "reference/api/companies", @@ -43,7 +59,7 @@ "api/routines": "reference/api/routines", "cli/overview": "reference/cli/overview", "cli/setup-commands": "reference/cli/setup-commands", - "cli/control-plane-commands": "reference/cli/control-plane-commands", + "cli/control-plane-commands": "reference/cli/overview", "adapters/overview": "reference/adapters/overview", "adapters/claude-local": "reference/adapters/claude-local", "adapters/codex-local": "reference/adapters/codex-local",