fix(stats): fix data page canonicals, 404s, and sitemap coverage (#52525)

packages/stats/app/src/component/model-compare-detail.tsx

@@ -48,6 +48,7 @@ import {
 } from "../lib/comparison-pages"
 import { baseUrl } from "../lib/language"
 import { runStatsEffect } from "../stats-runtime"
+import { NotFoundMeta } from "./not-found-meta"
 
 const compareHeaderLinks: readonly HeaderLink[] = [
   { href: `${import.meta.env.BASE_URL}#top-models`, label: "Top Models" },
@@ -235,20 +236,28 @@ export default function ModelCompareDetailPage(props: ModelCompareDetailPageProp
 
   return (
     <main data-page="stats" data-layout="compare-detail" data-theme={themePreference()}>
-      <Show when={catalog() !== undefined}>
+      {/* Server-rendered head tags are never removed, so render them once all data has loaded. */}
+      <Show when={catalog() !== undefined && stats() !== undefined}>
         <Title>{title()}</Title>
         <Meta name="description" content={description()} />
-        <Meta name="robots" content={models().length > 2 ? "noindex,follow" : "index,follow"} />
-        <Link rel="canonical" href={canonicalUrl()} />
-        <Meta property="og:type" content="website" />
-        <Meta property="og:site_name" content="OpenCode" />
-        <Meta property="og:title" content={title()} />
-        <Meta property="og:description" content={description()} />
-        <Meta property="og:url" content={canonicalUrl()} />
-        <Meta name="twitter:card" content="summary" />
-        <Meta name="twitter:title" content={title()} />
-        <Meta name="twitter:description" content={description()} />
-        <script type="application/ld+json">{structuredData()}</script>
+        <Show
+          when={models()
+            .slice(0, 2)
+            .every((model) => model.catalog || model.stats)}
+          fallback={<NotFoundMeta unavailable={catalog()?.models.length === 0} />}
+        >
+          <Meta name="robots" content={models().length > 2 ? "noindex,follow" : "index,follow"} />
+          <Link rel="canonical" href={canonicalUrl()} />
+          <Meta property="og:type" content="website" />
+          <Meta property="og:site_name" content="OpenCode" />
+          <Meta property="og:title" content={title()} />
+          <Meta property="og:description" content={description()} />
+          <Meta property="og:url" content={canonicalUrl()} />
+          <Meta name="twitter:card" content="summary" />
+          <Meta name="twitter:title" content={title()} />
+          <Meta name="twitter:description" content={description()} />
+          <script type="application/ld+json">{structuredData()}</script>
+        </Show>
       </Show>
       <Header
         githubStars={githubStars() ?? githubLink.fallbackStars}

packages/stats/app/src/component/not-found-meta.tsx

@@ -0,0 +1,12 @@
+import { Meta } from "@solidjs/meta"
+import { HttpStatusCode } from "@solidjs/start"
+
+// Use 503 when upstream data failed to load so crawlers retry instead of dropping the URL.
+export function NotFoundMeta(props: { unavailable?: boolean }) {
+  return (
+    <>
+      <HttpStatusCode code={props.unavailable ? 503 : 404} />
+      <Meta name="robots" content="noindex,follow" />
+    </>
+  )
+}

packages/stats/app/src/routes/[...404].tsx

@@ -0,0 +1,18 @@
+import { Title } from "@solidjs/meta"
+import { NotFoundMeta } from "../component/not-found-meta"
+import { useLanguage } from "../context/language"
+
+export default function NotFound() {
+  const language = useLanguage()
+  return (
+    <main data-page="stats">
+      <Title>Page not found</Title>
+      <NotFoundMeta />
+      <div data-component="empty-state">
+        <strong>Page not found</strong>
+        <p>This data page doesn't exist.</p>
+        <a href={language.route(import.meta.env.BASE_URL)}>Model data</a>
+      </div>
+    </main>
+  )
+}

packages/stats/app/src/routes/[lab]/[model].tsx

@@ -14,10 +14,12 @@ import { createAsync, query, useParams } from "@solidjs/router"
 import { createMemo, createSignal, createUniqueId, For, onMount, Show, type JSX } from "solid-js"
 import { getRequestEvent } from "solid-js/web"
 import { LocaleLinks } from "../../component/locale-links"
+import { NotFoundMeta } from "../../component/not-found-meta"
 import { useI18n } from "../../context/i18n"
 import { useLanguage } from "../../context/language"
 import { localizedUrl } from "../../lib/language"
 import {
+  catalogModelPath,
   findModelCatalogEntry,
   formatCatalogLabName,
   isKnownCatalogLab,
@@ -47,7 +49,7 @@ import {
 } from "../stats-shell"
 
 const statsUnfurlPath = "banner.png"
-const glmFlashCatalogId = "zhipuai/glm-5.3-flash"
+const glmFlashPath = "/data/zhipuai/glm-5-3-flash"
 const glmFlashModel = "glm-5.3-flash"
 const shortMonths = ["JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"] as const
 
@@ -110,10 +112,10 @@ export default function StatsModel() {
   const modelTitle = createMemo(() => i18n.t("model.title", { model: searchModelName() }))
   const modelDescription = createMemo(() => i18n.t("model.description", { model: searchModelName() }))
   const modelPath = createMemo(() => {
-    const fallback = formerName()
-      ? glmFlashCatalogId
-      : [labParam(), stats()?.slug ?? canonicalModel()].filter((part) => part.length > 0).join("/")
-    return `/data/${catalogEntry()?.id ?? fallback}`
+    const entry = catalogEntry()
+    if (entry) return catalogModelPath(entry)
+    if (formerName()) return glmFlashPath
+    return `/data/${providerSlug(labParam())}/${stats()?.slug ?? providerSlug(canonicalModel())}`
   })
   const modelUrl = createMemo(() => localizedUrl(language.locale(), modelPath()))
   const statsUnfurlUrl = new URL(statsUnfurlPath, localizedUrl("en", "/data/")).toString()
@@ -152,24 +154,32 @@ export default function StatsModel() {
 
   return (
     <main data-page="stats" data-theme={themePreference()}>
-      <Title>{modelTitle()}</Title>
-      <Meta name="description" content={modelDescription()} />
-      <LocaleLinks path={modelPath()} />
-      <Meta property="og:type" content="website" />
-      <Meta property="og:site_name" content="OpenCode" />
-      <Meta property="og:title" content={modelTitle()} />
-      <Meta property="og:description" content={modelDescription()} />
-      <Meta property="og:url" content={modelUrl()} />
-      <Meta property="og:image" content={statsUnfurlUrl} />
-      <Meta property="og:image:type" content="image/png" />
-      <Meta property="og:image:width" content="1200" />
-      <Meta property="og:image:height" content="630" />
-      <Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
-      <Meta name="twitter:card" content="summary_large_image" />
-      <Meta name="twitter:title" content={modelTitle()} />
-      <Meta name="twitter:description" content={modelDescription()} />
-      <Meta name="twitter:image" content={statsUnfurlUrl} />
-      <Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
+      {/* Server-rendered head tags are never removed, so render them once data has loaded. */}
+      <Show when={page()}>
+        <Title>{modelTitle()}</Title>
+        <Meta name="description" content={modelDescription()} />
+        <Show
+          when={catalogEntry() || stats()}
+          fallback={<NotFoundMeta unavailable={page()?.catalog.labs.length === 0} />}
+        >
+          <LocaleLinks path={modelPath()} />
+          <Meta property="og:type" content="website" />
+          <Meta property="og:site_name" content="OpenCode" />
+          <Meta property="og:title" content={modelTitle()} />
+          <Meta property="og:description" content={modelDescription()} />
+          <Meta property="og:url" content={modelUrl()} />
+          <Meta property="og:image" content={statsUnfurlUrl} />
+          <Meta property="og:image:type" content="image/png" />
+          <Meta property="og:image:width" content="1200" />
+          <Meta property="og:image:height" content="630" />
+          <Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
+          <Meta name="twitter:card" content="summary_large_image" />
+          <Meta name="twitter:title" content={modelTitle()} />
+          <Meta name="twitter:description" content={modelDescription()} />
+          <Meta name="twitter:image" content={statsUnfurlUrl} />
+          <Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
+        </Show>
+      </Show>
       <Header
         githubStars={githubStars() ?? githubLink.fallbackStars}
         links={modelHeaderLinks()}
@@ -325,7 +335,7 @@ function ModelHero(props: {
             current
             label={modelName()}
             options={labModels().map((model) => ({
-              href: language.route(`${import.meta.env.BASE_URL}${model.id}`),
+              href: language.route(catalogModelPath(model)),
               label: model.name,
               value: model.id,
             }))}

packages/stats/app/src/routes/[lab]/index.tsx

@@ -12,10 +12,12 @@ import { createAsync, query, useParams } from "@solidjs/router"
 import { createMemo, createSignal, createUniqueId, For, onMount, Show, type JSX } from "solid-js"
 import { getRequestEvent } from "solid-js/web"
 import { LocaleLinks } from "../../component/locale-links"
+import { NotFoundMeta } from "../../component/not-found-meta"
 import { useI18n } from "../../context/i18n"
 import { useLanguage } from "../../context/language"
 import { localizedUrl } from "../../lib/language"
 import {
+  catalogModelPath,
   catalogSlug,
   findModelCatalogLab,
   formatCatalogLabName,
@@ -124,24 +126,29 @@ export default function StatsLab() {
 
   return (
     <main data-page="stats" data-theme={themePreference()}>
-      <Title>{labTitle()}</Title>
-      <Meta name="description" content={labDescription()} />
-      <LocaleLinks path={labPath()} />
-      <Meta property="og:type" content="website" />
-      <Meta property="og:site_name" content="OpenCode" />
-      <Meta property="og:title" content={labTitle()} />
-      <Meta property="og:description" content={labDescription()} />
-      <Meta property="og:url" content={labUrl()} />
-      <Meta property="og:image" content={statsUnfurlUrl} />
-      <Meta property="og:image:type" content="image/png" />
-      <Meta property="og:image:width" content="1200" />
-      <Meta property="og:image:height" content="630" />
-      <Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
-      <Meta name="twitter:card" content="summary_large_image" />
-      <Meta name="twitter:title" content={labTitle()} />
-      <Meta name="twitter:description" content={labDescription()} />
-      <Meta name="twitter:image" content={statsUnfurlUrl} />
-      <Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
+      {/* Server-rendered head tags are never removed, so render them once data has loaded. */}
+      <Show when={page()}>
+        <Title>{labTitle()}</Title>
+        <Meta name="description" content={labDescription()} />
+        <Show when={lab()} fallback={<NotFoundMeta unavailable={page()?.labs.length === 0} />}>
+          <LocaleLinks path={labPath()} />
+          <Meta property="og:type" content="website" />
+          <Meta property="og:site_name" content="OpenCode" />
+          <Meta property="og:title" content={labTitle()} />
+          <Meta property="og:description" content={labDescription()} />
+          <Meta property="og:url" content={labUrl()} />
+          <Meta property="og:image" content={statsUnfurlUrl} />
+          <Meta property="og:image:type" content="image/png" />
+          <Meta property="og:image:width" content="1200" />
+          <Meta property="og:image:height" content="630" />
+          <Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
+          <Meta name="twitter:card" content="summary_large_image" />
+          <Meta name="twitter:title" content={labTitle()} />
+          <Meta name="twitter:description" content={labDescription()} />
+          <Meta name="twitter:image" content={statsUnfurlUrl} />
+          <Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
+        </Show>
+      </Show>
       <Header
         githubStars={githubStars() ?? githubLink.fallbackStars}
         links={labHeaderLinks()}
@@ -595,7 +602,7 @@ function LabModelRow(props: {
   return (
     <a
       data-component="lab-model-row"
-      href={language.route(`${import.meta.env.BASE_URL}${props.model.id}`)}
+      href={language.route(catalogModelPath(props.model))}
       role="row"
       aria-label={props.model.name}
       onBlur={() => props.onTooltipChange(undefined)}

packages/stats/app/src/routes/compare/[firstFamily]/[secondFamily].tsx

@@ -1,7 +1,8 @@
-import { Meta, Title } from "@solidjs/meta"
+import { Title } from "@solidjs/meta"
 import { createAsync, useParams } from "@solidjs/router"
 import { createMemo, Show } from "solid-js"
 import ModelCompareDetailPage from "../../../component/model-compare-detail"
+import { NotFoundMeta } from "../../../component/not-found-meta"
 import { resolveComparisonFamily } from "../../../lib/comparison-pages"
 import { getModelCatalog } from "../../model-catalog"
 
@@ -23,7 +24,7 @@ export default function ModelCompareFamily() {
       fallback={
         <Show when={comparison() === null}>
           <Title>Model comparison not found</Title>
-          <Meta name="robots" content="noindex,follow" />
+          <NotFoundMeta unavailable={catalog()?.models.length === 0} />
           <main data-page="stats">
             <div data-component="empty-state">
               <strong>Comparison not found</strong>

packages/stats/app/src/routes/compare/index.tsx

@@ -108,24 +108,27 @@ export default function ModelCompareIndex() {
 
   return (
     <main data-page="stats" data-theme={themePreference()}>
-      <Title>{compareTitle}</Title>
-      <Meta name="description" content={compareDescription} />
-      <LocaleLinks path={comparePath} />
-      <Meta property="og:type" content="website" />
-      <Meta property="og:site_name" content="OpenCode" />
-      <Meta property="og:title" content={compareTitle} />
-      <Meta property="og:description" content={compareDescription} />
-      <Meta property="og:url" content={compareUrl()} />
-      <Meta property="og:image" content={statsUnfurlUrl} />
-      <Meta property="og:image:type" content="image/png" />
-      <Meta property="og:image:width" content="1200" />
-      <Meta property="og:image:height" content="630" />
-      <Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
-      <Meta name="twitter:card" content="summary_large_image" />
-      <Meta name="twitter:title" content={compareTitle} />
-      <Meta name="twitter:description" content={compareDescription} />
-      <Meta name="twitter:image" content={statsUnfurlUrl} />
-      <Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
+      {/* Server-rendered head tags are never removed, so render them once data has loaded. */}
+      <Show when={catalog()}>
+        <Title>{compareTitle}</Title>
+        <Meta name="description" content={compareDescription} />
+        <LocaleLinks path={comparePath} />
+        <Meta property="og:type" content="website" />
+        <Meta property="og:site_name" content="OpenCode" />
+        <Meta property="og:title" content={compareTitle} />
+        <Meta property="og:description" content={compareDescription} />
+        <Meta property="og:url" content={compareUrl()} />
+        <Meta property="og:image" content={statsUnfurlUrl} />
+        <Meta property="og:image:type" content="image/png" />
+        <Meta property="og:image:width" content="1200" />
+        <Meta property="og:image:height" content="630" />
+        <Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
+        <Meta name="twitter:card" content="summary_large_image" />
+        <Meta name="twitter:title" content={compareTitle} />
+        <Meta name="twitter:description" content={compareDescription} />
+        <Meta name="twitter:image" content={statsUnfurlUrl} />
+        <Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
+      </Show>
       <Header
         githubStars={githubStars() ?? githubLink.fallbackStars}
         links={compareHeaderLinks()}

packages/stats/app/src/routes/index.tsx

@@ -124,24 +124,27 @@ export default function StatsHome() {
 
   return (
     <main data-page="stats" data-theme={themePreference()}>
-      <Title>{i18n.t("app.title")}</Title>
-      <Meta name="description" content={i18n.t("app.description")} />
-      <LocaleLinks path="/data/" />
-      <Meta property="og:type" content="website" />
-      <Meta property="og:site_name" content="OpenCode" />
-      <Meta property="og:title" content={i18n.t("app.title")} />
-      <Meta property="og:description" content={i18n.t("app.description")} />
-      <Meta property="og:url" content={statsHomeUrl} />
-      <Meta property="og:image" content={statsUnfurlUrl} />
-      <Meta property="og:image:type" content="image/jpeg" />
-      <Meta property="og:image:width" content="1200" />
-      <Meta property="og:image:height" content="630" />
-      <Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
-      <Meta name="twitter:card" content="summary_large_image" />
-      <Meta name="twitter:title" content={i18n.t("app.title")} />
-      <Meta name="twitter:description" content={i18n.t("app.description")} />
-      <Meta name="twitter:image" content={statsUnfurlUrl} />
-      <Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
+      {/* Server-rendered head tags are never removed, so render them once data has loaded. */}
+      <Show when={data()}>
+        <Title>{i18n.t("app.title")}</Title>
+        <Meta name="description" content={i18n.t("app.description")} />
+        <LocaleLinks path="/data/" />
+        <Meta property="og:type" content="website" />
+        <Meta property="og:site_name" content="OpenCode" />
+        <Meta property="og:title" content={i18n.t("app.title")} />
+        <Meta property="og:description" content={i18n.t("app.description")} />
+        <Meta property="og:url" content={statsHomeUrl} />
+        <Meta property="og:image" content={statsUnfurlUrl} />
+        <Meta property="og:image:type" content="image/jpeg" />
+        <Meta property="og:image:width" content="1200" />
+        <Meta property="og:image:height" content="630" />
+        <Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
+        <Meta name="twitter:card" content="summary_large_image" />
+        <Meta name="twitter:title" content={i18n.t("app.title")} />
+        <Meta name="twitter:description" content={i18n.t("app.description")} />
+        <Meta name="twitter:image" content={statsUnfurlUrl} />
+        <Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
+      </Show>
       <Header githubStars={githubStars() ?? githubLink.fallbackStars} />
       <div data-component="container">
         <div data-component="content">

packages/stats/app/src/routes/model-catalog.ts

@@ -91,6 +91,10 @@ export function findModelCatalogLab(catalog: ModelCatalog, lab: string) {
   return catalog.labs.find((entry) => entry.id === id)
 }
 
+export function catalogModelPath(entry: Pick<ModelCatalogEntry, "lab" | "slug">) {
+  return `/data/${entry.lab}/${entry.slug}`
+}
+
 export function formatCatalogLabName(lab: string) {
   const known: Record<string, string> = {
     alibaba: "Alibaba",

packages/stats/app/src/routes/sitemap.xml.ts

@@ -9,7 +9,7 @@ import {
   resolveComparisonFamily,
 } from "../lib/comparison-pages"
 import { baseUrl } from "../lib/language"
-import { loadModelCatalog } from "./model-catalog"
+import { catalogModelPath, findModelCatalogEntry, loadModelCatalog } from "./model-catalog"
 
 type SitemapEntry = {
   path: string
@@ -55,7 +55,25 @@ export async function GET() {
       ]
     }),
   )
-  const entries = uniqueSitemapEntries([{ path: "/data/compare", lastmod }, ...familyComparisons, ...modelComparisons])
+  const labPages = catalog.labs.map((lab) => ({
+    path: `/data/${lab.id}`,
+    lastmod: sitemapDate(stats?.updatedAt, ...lab.models.map((model) => model.lastUpdated ?? model.releaseDate)),
+  }))
+  // Skip entries whose URL resolves to another model, such as stats aliases.
+  const modelPages = [...catalog.models, ...(catalog.aliases ?? [])]
+    .filter((model) => findModelCatalogEntry(catalog, model.slug, model.lab)?.id === model.id)
+    .map((model) => ({
+      path: catalogModelPath(model),
+      lastmod: sitemapDate(stats?.updatedAt, model.lastUpdated ?? model.releaseDate),
+    }))
+  const entries = uniqueSitemapEntries([
+    { path: "/data/", lastmod: sitemapDate(stats?.updatedAt) },
+    { path: "/data/compare", lastmod },
+    ...labPages,
+    ...modelPages,
+    ...familyComparisons,
+    ...modelComparisons,
+  ])
 
   return new Response(sitemapXml(entries), {
     headers: {