fix(stats): hide models listed in a secret (#52834)

infra/stats.ts

@@ -87,6 +87,10 @@ const statsSyncConfig = new sst.Linkable("StatsSyncConfig", {
   },
 })
 
+// Comma-separated model IDs kept off the public stats site. A secret, so excluding
+// an unreleased model does not publish its name in git.
+const statsHiddenModels = new sst.Secret("StatsHiddenModels", "")
+
 const r2SqlAuthToken = new sst.Secret("R2SqlAuthToken")
 const r2Sql = new sst.Linkable("R2Sql", {
   properties: {
@@ -109,7 +113,7 @@ export const statSync = new sst.aws.Service("StatsSyncService", {
     dockerfile: "packages/stats/server/Dockerfile",
   },
   command: ["bun", "src/stat-sync.ts"],
-  link: [database, r2Sql, r2SqlAuthToken, statsSyncConfig],
+  link: [database, r2Sql, r2SqlAuthToken, statsHiddenModels, statsSyncConfig],
   scaling: {
     min: 1,
     max: 1,

packages/stats/core/src/domain/geo.ts

@@ -55,7 +55,10 @@ export declare namespace GeoStatRepo {
       readonly model?: string
     }) => Effect.Effect<GeoStatRow[], DatabaseError>
     readonly upsert: (rows: GeoStatRow[]) => Effect.Effect<void, DatabaseError>
-    readonly deleteRetiredDimensions: (rows: GeoStatRow[]) => Effect.Effect<void, DatabaseError>
+    readonly deleteRetiredDimensions: (
+      rows: GeoStatRow[],
+      hiddenModels: readonly string[],
+    ) => Effect.Effect<void, DatabaseError>
     readonly deleteUnknownDimensions: (rows: GeoStatRow[]) => Effect.Effect<void, DatabaseError>
   }
 }
@@ -193,7 +196,10 @@ export class GeoStatRepo extends Context.Service<GeoStatRepo, GeoStatRepo.Servic
           })
       }
 
-      const deleteRetiredDimensions = Effect.fn("GeoStatRepo.deleteRetiredDimensions")(function* (rows: GeoStatRow[]) {
+      const deleteRetiredDimensions = Effect.fn("GeoStatRepo.deleteRetiredDimensions")(function* (
+        rows: GeoStatRow[],
+        hiddenModels: readonly string[],
+      ) {
         const scope = statRowScope(rows)
         if (!scope) return
 
@@ -210,7 +216,7 @@ export class GeoStatRepo extends Context.Service<GeoStatRepo, GeoStatRepo.Servic
                   inArray(geoStat.source, scope.sources),
                   or(
                     inArray(geoStat.provider, RETIRED_STAT_PROVIDERS),
-                    inArray(geoStat.model, RETIRED_STAT_MODELS),
+                    inArray(geoStat.model, [...RETIRED_STAT_MODELS, ...hiddenModels]),
                     and(eq(geoStat.provider, "unknown"), eq(geoStat.model, "hy4-preview")),
                   ),
                 ),

packages/stats/core/src/domain/inference.test.ts

@@ -245,6 +245,19 @@ describe("inference stat normalization", () => {
     ).toMatchObject([{ period_key: "2026-W20" }])
   })
 
+  test("excludes hidden models from stats and retention queries", () => {
+    const source = { namespace: "inference", table: "generation", dataset: "zen", hiddenModels: ["hidden-model"] }
+    const queries = [
+      ...buildStatsQueries(new Date("2026-08-10T00:00:00.000Z"), new Date("2026-08-11T00:00:00.000Z"), source),
+      ...buildRetentionQueries(new Date("2026-08-10T00:00:00.000Z"), new Date("2026-08-24T00:00:00.000Z"), source).map(
+        (item) => item.query,
+      ),
+    ]
+
+    expect(queries.length).toBeGreaterThan(0)
+    queries.forEach((query) => expect(query).toContain("lower(model) NOT IN ('alpha-gpt-next', 'hidden-model')"))
+  })
+
   test("builds bounded R2 SQL queries for each day and week", () => {
     const queries = buildStatsQueries(new Date("2026-08-10T00:00:00.000Z"), new Date("2026-08-12T12:00:00.000Z"), {
       namespace: "inference",

packages/stats/core/src/domain/inference.ts

@@ -26,7 +26,12 @@ import {
 } from "./stat"
 
 export type StatDimension = "model" | "provider" | "geo" | "geo_model"
-export type StatsQuerySource = { namespace: string; table: string; dataset: string }
+export type StatsQuerySource = {
+  namespace: string
+  table: string
+  dataset: string
+  hiddenModels?: readonly string[]
+}
 export type RetentionQuery = { cohortDates: string[]; query: string }
 type StatsQueryFamily = "usage" | "geo"
 
@@ -50,6 +55,7 @@ export function buildStatsQueries(
     namespace: Resource.R2Sql.namespace,
     table: Resource.R2Sql.table,
     dataset: Resource.StatsSyncConfig.dataset,
+    hiddenModels: hiddenStatModels(),
   }
   return [...statPeriods("week", periodStart, periodEnd), ...statPeriods("day", periodStart, periodEnd)].flatMap(
     (period) => [buildStatsQuery(period, source, "usage", catalog), buildStatsQuery(period, source, "geo", catalog)],
@@ -66,6 +72,7 @@ export function buildRetentionQueries(
     namespace: Resource.R2Sql.namespace,
     table: Resource.R2Sql.table,
     dataset: Resource.StatsSyncConfig.dataset,
+    hiddenModels: hiddenStatModels(),
   }
   const periods = retentionPeriods(periodStart, periodEnd)
   // Bound the user-level joins to one activity week and its return week.
@@ -144,7 +151,7 @@ WITH normalized AS (
   FROM normalized
   WHERE activity_week IS NOT NULL
     AND user_key <> ''
-    AND lower(model) NOT IN (${[...EXCLUDED_MODELS].map(sqlString).join(", ")})
+    AND lower(model) NOT IN (${excludedModels(source).map(sqlString).join(", ")})
 ), model_usage AS (
   SELECT
     activity_week AS cohort_date,
@@ -329,7 +336,7 @@ WITH normalized AS (
     cost_output_microcents,
     cost_total_microcents
   FROM normalized
-  WHERE lower(model) NOT IN (${[...EXCLUDED_MODELS].map(sqlString).join(", ")})
+  WHERE lower(model) NOT IN (${excludedModels(source).map(sqlString).join(", ")})
 )
 SELECT
   ${sqlString(period.grain)} AS grain,
@@ -448,6 +455,17 @@ function sqlIdentifier(value: string) {
   return `"${value.replace(/"/g, '""')}"`
 }
 
+export function hiddenStatModels() {
+  return Resource.StatsHiddenModels.value
+    .split(",")
+    .map((model) => model.trim().toLowerCase())
+    .filter(Boolean)
+}
+
+function excludedModels(source: StatsQuerySource) {
+  return [...EXCLUDED_MODELS, ...(source.hiddenModels ?? [])]
+}
+
 function sqlString(value: string) {
   return `'${value.replace(/'/g, "''")}'`
 }

packages/stats/core/src/domain/model.ts

@@ -46,7 +46,10 @@ export declare namespace ModelStatRepo {
     readonly listDaily: () => Effect.Effect<ModelStatMetric[], DatabaseError>
     readonly lastSyncedAt: () => Effect.Effect<Date | null, DatabaseError>
     readonly upsert: (rows: ModelStatRow[]) => Effect.Effect<void, DatabaseError>
-    readonly deleteRetiredDimensions: (rows: ModelStatRow[]) => Effect.Effect<void, DatabaseError>
+    readonly deleteRetiredDimensions: (
+      rows: ModelStatRow[],
+      hiddenModels: readonly string[],
+    ) => Effect.Effect<void, DatabaseError>
     readonly deleteUnknownDimensions: (rows: ModelStatRow[]) => Effect.Effect<void, DatabaseError>
   }
 }
@@ -178,6 +181,7 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
 
       const deleteRetiredDimensions = Effect.fn("ModelStatRepo.deleteRetiredDimensions")(function* (
         rows: ModelStatRow[],
+        hiddenModels: readonly string[],
       ) {
         const scope = statRowScope(rows)
         if (!scope) return
@@ -195,7 +199,7 @@ export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.
                   inArray(modelStat.source, scope.sources),
                   or(
                     inArray(modelStat.provider, RETIRED_STAT_PROVIDERS),
-                    inArray(modelStat.model, RETIRED_STAT_MODELS),
+                    inArray(modelStat.model, [...RETIRED_STAT_MODELS, ...hiddenModels]),
                     and(eq(modelStat.provider, "unknown"), eq(modelStat.model, "hy4-preview")),
                   ),
                 ),

packages/stats/core/src/resource.d.ts

@@ -13,6 +13,10 @@ declare module "sst/resource" {
       type: "sst.sst.Secret"
       value: string
     }
+    StatsHiddenModels: {
+      type: "sst.sst.Secret"
+      value: string
+    }
     StatsSyncConfig: {
       dataset: string
       type: "sst.sst.Linkable"

packages/stats/core/src/stat-sync.ts

@@ -6,6 +6,7 @@ import { GeoStatRepo, rowsFromAggregates as geoRowsFromAggregates } from "./doma
 import {
   buildRetentionQueries,
   buildStatsQueries,
+  hiddenStatModels,
   toGeoAggregate,
   toModelAggregate,
   toProviderAggregate,
@@ -148,9 +149,9 @@ export const syncStats: (options?: {
     )
     yield* Effect.all(
       [
-        modelStats.deleteRetiredDimensions(modelRows),
+        modelStats.deleteRetiredDimensions(modelRows, hiddenStatModels()),
         providerStats.deleteRetiredDimensions(providerRows),
-        geoStats.deleteRetiredDimensions(geoRows),
+        geoStats.deleteRetiredDimensions(geoRows, hiddenStatModels()),
       ],
       { concurrency: "unbounded", discard: true },
     )