mirror of
https://github.com/paperclipai/paperclip-docs.git
synced 2026-11-07 06:16:07 -10:00
[codex] Fix docs SEO crawl hygiene (#46)
## Summary Fixes the docs-side causes behind the Google Search Console duplicate/canonical SEO issues by tightening generated static routing behavior. ## What changed - Removed the Cloudflare Pages wildcard `/* /index.html 200` SPA fallback from the generated docs release so deleted or unknown asset URLs can return 404 instead of serving docs HTML. - Removed the global `X-Robots-Tag: index, follow` header and replaced it with noindex coverage for non-page static assets, `sitemap.xml`, `robots.txt`, and the generated `404.html`. - Added legacy route handling for removed docs URLs and extended static route verification for redirects, removed assets, noindex headers, and 404 behavior. ## Root cause The generated Cloudflare Pages release treated too many non-page/static URLs as indexable page responses. That allowed removed assets or auxiliary files to be crawled as duplicate indexable documents instead of clear 404/noindex responses. ## Validation - `npm run docs:test:static-routes` - `npm run docs:build` PAP-12839 --------- Co-authored-by: DevRel Gemini <[email protected]> Co-authored-by: Paperclip <[email protected]>
This commit is contained in:
3 files changed
+145
-8
No files matched your search
@@ -111,11 +111,51 @@ try {
|
||||
"Cloudflare redirects are missing the no-slash canonical redirect for the quickstart path",
|
||||
);
|
||||
assert(
|
||||
redirects.indexOf(canonicalRedirect) < redirects.indexOf("/* /index.html 200"),
|
||||
"canonical route redirects must appear before the SPA fallback rewrite",
|
||||
!redirects.includes("/* /index.html 200"),
|
||||
"Cloudflare redirects must not rewrite unknown URLs and removed assets to the docs shell",
|
||||
);
|
||||
|
||||
const legacyRedirects = JSON.parse(readFileSync(join(root, "site/redirects.json"), "utf8"));
|
||||
for (const [sourceRoute, destinationRoute] of Object.entries(legacyRedirects)) {
|
||||
const destinationIndexPath = join(outDir, destinationRoute, "index.html");
|
||||
assert(
|
||||
existsSync(destinationIndexPath),
|
||||
`legacy redirect target is not a generated page: ${sourceRoute} -> ${destinationRoute}`,
|
||||
);
|
||||
const noSlashRedirect = `/${sourceRoute} /${destinationRoute}/ 301`;
|
||||
const slashRedirect = `/${sourceRoute}/ /${destinationRoute}/ 301`;
|
||||
assert(redirects.includes(noSlashRedirect), `missing legacy no-slash redirect: ${noSlashRedirect}`);
|
||||
assert(redirects.includes(slashRedirect), `missing legacy slash redirect: ${slashRedirect}`);
|
||||
}
|
||||
assert(
|
||||
redirects.includes("/guides/agent-developer/how-agents-work /guides/org/agents/ 301"),
|
||||
"Search Console legacy agent-developer URL should redirect to the current agents guide",
|
||||
);
|
||||
assert(
|
||||
redirects.includes("/start/quickstart/ /guides/getting-started/five-minute-path/ 301"),
|
||||
"Search Console legacy quickstart URL should redirect to the current quickstart",
|
||||
);
|
||||
|
||||
const headers = read("_headers");
|
||||
assert(!headers.includes("X-Robots-Tag: index, follow"), "headers must not globally mark every static file indexable");
|
||||
assert(
|
||||
headers.includes("/*.css\n X-Robots-Tag: noindex, nofollow"),
|
||||
"CSS assets should be marked noindex in Cloudflare headers",
|
||||
);
|
||||
assert(
|
||||
headers.includes("/*.js\n X-Robots-Tag: noindex, nofollow"),
|
||||
"JS assets should be marked noindex in Cloudflare headers",
|
||||
);
|
||||
assert(
|
||||
headers.includes("/*.md\n X-Robots-Tag: noindex, nofollow"),
|
||||
"copied markdown source files should be marked noindex in Cloudflare headers",
|
||||
);
|
||||
assert(
|
||||
headers.includes("/sitemap.xml\n Content-Type: application/xml; charset=utf-8\n X-Robots-Tag: noindex, nofollow"),
|
||||
"sitemap.xml should be served as XML and excluded from search results",
|
||||
);
|
||||
const notFoundHtml = read("404.html");
|
||||
assert(notFoundHtml.includes('<meta name="robots" content="noindex, nofollow" />'), "404 page must be noindex");
|
||||
assert(
|
||||
headers.includes("Strict-Transport-Security: max-age=31536000; includeSubDomains; preload"),
|
||||
"Cloudflare headers are missing HSTS",
|
||||
|
||||
+85
-4
@@ -13,6 +13,7 @@ const sourceIndexPath = path.join(__dirname, "index.html");
|
||||
const sourceStylesPath = path.join(__dirname, "styles.css");
|
||||
const sourceAppJsPath = path.join(__dirname, "app.js");
|
||||
const sourceNavPath = path.join(__dirname, "content.json");
|
||||
const sourceRedirectsPath = path.join(__dirname, "redirects.json");
|
||||
const sourceVendorDir = path.join(__dirname, "vendor");
|
||||
const screenshotsSourceDir = path.join(docsRoot, "user-guides", "screenshots");
|
||||
const defaultSiteUrl = "https://docs.paperclip.ing";
|
||||
@@ -655,7 +656,18 @@ function cloudflarePathForRoute(basePath, routePath, { trailingSlash = false } =
|
||||
return `/${key}${trailingSlash ? "/" : ""}`;
|
||||
}
|
||||
|
||||
function buildCloudflareRedirects({ basePath, pages }) {
|
||||
function cloudflareRedirectLine(basePath, sourceRoute, destinationRoute) {
|
||||
const sourcePath = cloudflarePathForRoute(basePath, sourceRoute);
|
||||
const sourceSlashPath = cloudflarePathForRoute(basePath, sourceRoute, { trailingSlash: true });
|
||||
const destinationPath = cloudflarePathForRoute(basePath, destinationRoute, { trailingSlash: true });
|
||||
if (sourceSlashPath === destinationPath) return [];
|
||||
return [
|
||||
`${sourcePath} ${destinationPath} 301`,
|
||||
`${sourceSlashPath} ${destinationPath} 301`,
|
||||
];
|
||||
}
|
||||
|
||||
function buildCloudflareRedirects({ basePath, pages, legacyRedirects = {} }) {
|
||||
const routeRedirects = pages
|
||||
.map(({ page }) => {
|
||||
const sourcePath = cloudflarePathForRoute(basePath, page.slug);
|
||||
@@ -663,19 +675,24 @@ function buildCloudflareRedirects({ basePath, pages }) {
|
||||
return `${sourcePath} ${destinationPath} 301`;
|
||||
})
|
||||
.join("\n");
|
||||
const legacyRouteRedirects = Object.entries(legacyRedirects)
|
||||
.flatMap(([sourceRoute, destinationRoute]) =>
|
||||
cloudflareRedirectLine(basePath, sourceRoute, destinationRoute)
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
return `# Canonical docs URLs include trailing slashes. Keep no-slash requests
|
||||
# on a normal one-hop 301 instead of Cloudflare Pages' implicit directory 308.
|
||||
${routeRedirects}
|
||||
|
||||
# Serve generated files first; fall back to the SPA shell for client routes.
|
||||
/* /index.html 200
|
||||
# Legacy docs URLs moved during the information architecture cleanup. Redirect
|
||||
# them before unknown URLs fall through to 404 so crawlers see one canonical URL per page.
|
||||
${legacyRouteRedirects}
|
||||
`;
|
||||
}
|
||||
|
||||
function buildCloudflareHeaders() {
|
||||
return `/*
|
||||
X-Robots-Tag: index, follow
|
||||
Referrer-Policy: strict-origin-when-cross-origin
|
||||
X-Content-Type-Options: nosniff
|
||||
Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
|
||||
@@ -685,9 +702,69 @@ function buildCloudflareHeaders() {
|
||||
|
||||
/sitemap.xml
|
||||
Content-Type: application/xml; charset=utf-8
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/robots.txt
|
||||
Content-Type: text/plain; charset=utf-8
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.css
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.js
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.json
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.md
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.png
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.jpg
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.jpeg
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.webp
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.svg
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
|
||||
/*.txt
|
||||
X-Robots-Tag: noindex, nofollow
|
||||
`;
|
||||
}
|
||||
|
||||
function buildNotFoundPage(siteUrl, basePath) {
|
||||
const metadata = {
|
||||
title: "Not found | Paperclip Docs",
|
||||
description: "This Paperclip Docs URL does not exist.",
|
||||
url: siteUrlForPath(siteUrl, basePath, "404.html"),
|
||||
siteUrl,
|
||||
basePath,
|
||||
};
|
||||
return `<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>${escapeHtml(metadata.title)}</title>
|
||||
<meta name="robots" content="noindex, nofollow" />
|
||||
<link rel="canonical" href="${escapeHtml(metadata.url)}" />
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Not found</h1>
|
||||
<p>This Paperclip Docs URL does not exist.</p>
|
||||
<p><a href="${escapeAttr(siteUrlForPath(siteUrl, basePath))}">Open the docs home page</a></p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
}
|
||||
|
||||
@@ -909,6 +986,7 @@ ${basePathGuidance}
|
||||
- Serve the bundle root at \`${deploymentBasePath}\`
|
||||
- Keep all copied files together so requests for \`content.json\`, markdown files, images, fonts, and JS resolve normally
|
||||
- Serve generated files such as \`sitemap.xml\`, \`robots.txt\`, and nested route directories unchanged
|
||||
- Do not add a wildcard SPA rewrite such as \`/* /index.html 200\`; unknown URLs and removed assets must return 404 so crawlers do not treat them as duplicate docs pages
|
||||
|
||||
If \`content.json\` or linked markdown files are missing from the uploaded bundle, the docs app will fail to load content.
|
||||
|
||||
@@ -948,6 +1026,7 @@ async function minifyJs(source) {
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const sourceNav = JSON.parse(await fs.readFile(sourceNavPath, "utf8"));
|
||||
const sourceRedirects = JSON.parse(await fs.readFile(sourceRedirectsPath, "utf8"));
|
||||
const releaseNav = attachSlugs(rewriteNav(sourceNav));
|
||||
const { markdownFiles, warnings } = await collectReleaseFiles(sourceNav);
|
||||
|
||||
@@ -968,6 +1047,7 @@ async function main() {
|
||||
await fs.writeFile(path.join(options.outDir, "nginx.conf.example"), buildNginxConfig(options.basePath));
|
||||
await fs.writeFile(path.join(options.outDir, "DEPLOY.md"), buildDeployGuide(options.basePath));
|
||||
await fs.writeFile(path.join(options.outDir, "_headers"), buildCloudflareHeaders());
|
||||
await fs.writeFile(path.join(options.outDir, "404.html"), buildNotFoundPage(options.siteUrl, options.basePath));
|
||||
|
||||
// Copy markdown files, stripping YAML frontmatter, and collect per-file
|
||||
// frontmatter to surface via content.json (keyed by repo-relative path).
|
||||
@@ -997,6 +1077,7 @@ async function main() {
|
||||
await fs.writeFile(path.join(options.outDir, "_redirects"), buildCloudflareRedirects({
|
||||
basePath: options.basePath,
|
||||
pages: pageMetadata,
|
||||
legacyRedirects: sourceRedirects,
|
||||
}));
|
||||
const rootMetadata = {
|
||||
title: "Paperclip Docs",
|
||||
|
||||
+18
-2
@@ -14,7 +14,16 @@
|
||||
"guides/day-to-day/costs-and-budgets": "guides/day-to-day/costs",
|
||||
"user-guides/guides/activity-log": "guides/day-to-day/activity-log",
|
||||
"user-guides/guides/feedback-voting": "guides/day-to-day/feedback-voting",
|
||||
"guides/agent-developer/how-agents-work": "guides/org/how-agents-work",
|
||||
"guides/board-operator/dashboard": "guides/day-to-day/dashboard",
|
||||
"guides/board-operator/managing-tasks": "guides/day-to-day/issues",
|
||||
"guides/board-operator/managing-agents": "guides/org/agents",
|
||||
"guides/board-operator/org-structure": "guides/org/org-structure",
|
||||
"guides/board-operator/creating-a-company": "guides/getting-started/your-first-company",
|
||||
"guides/agent-developer/how-agents-work": "guides/org/agents",
|
||||
"guides/agent-developer/comments-and-communication": "guides/day-to-day/issues",
|
||||
"guides/agent-developer/task-workflow": "guides/day-to-day/issues",
|
||||
"guides/agent-developer/handling-approvals": "guides/day-to-day/approvals",
|
||||
"guides/agent-developer/cost-reporting": "guides/day-to-day/costs",
|
||||
"user-guides/guides/skills": "guides/org/skills",
|
||||
"user-guides/guides/org-structure": "guides/org/org-structure",
|
||||
"user-guides/guides/delegation": "guides/org/delegation",
|
||||
@@ -29,6 +38,13 @@
|
||||
"user-guides/guides/administration/company": "administration/company",
|
||||
"user-guides/guides/administration/settings": "administration/settings",
|
||||
"user-guides/guides/administration/plugins": "administration/plugins",
|
||||
"start/quickstart": "guides/getting-started/five-minute-path",
|
||||
"start/core-concepts": "guides/welcome/key-concepts",
|
||||
"tutorials": "guides/getting-started/five-minute-path",
|
||||
"how-to": "how-to/enable-multi-user-login",
|
||||
"glossary": "guides/welcome/glossary",
|
||||
"reference": "reference/api/overview",
|
||||
"explanation": "guides/welcome/what-is-paperclip",
|
||||
"api/overview": "reference/api/overview",
|
||||
"api/authentication": "reference/api/authentication",
|
||||
"api/companies": "reference/api/companies",
|
||||
@@ -43,7 +59,7 @@
|
||||
"api/routines": "reference/api/routines",
|
||||
"cli/overview": "reference/cli/overview",
|
||||
"cli/setup-commands": "reference/cli/setup-commands",
|
||||
"cli/control-plane-commands": "reference/cli/control-plane-commands",
|
||||
"cli/control-plane-commands": "reference/cli/overview",
|
||||
"adapters/overview": "reference/adapters/overview",
|
||||
"adapters/claude-local": "reference/adapters/claude-local",
|
||||
"adapters/codex-local": "reference/adapters/codex-local",
|
||||
|
||||
Reference in new issue
Block a user