feat(console): send Honeycomb alerts to Slack

infra/console.ts

@@ -233,6 +233,7 @@ const bucket = new sst.cloudflare.Bucket("ZenData")
 const bucketNew = new sst.cloudflare.Bucket("ZenDataNew")
 
 const DISCORD_INCIDENT_WEBHOOK_URL = new sst.Secret("DISCORD_INCIDENT_WEBHOOK_URL")
+const SLACK_INCIDENT_WEBHOOK_URL = new sst.Secret("SLACK_INCIDENT_WEBHOOK_URL")
 const AWS_SES_ACCESS_KEY_ID = new sst.Secret("AWS_SES_ACCESS_KEY_ID")
 const AWS_SES_SECRET_ACCESS_KEY = new sst.Secret("AWS_SES_SECRET_ACCESS_KEY")
 
@@ -258,6 +259,7 @@ new sst.cloudflare.x.SolidStart("Console", {
     STRIPE_WEBHOOK_SECRET,
     SECRET.SupportApiKey,
     DISCORD_INCIDENT_WEBHOOK_URL,
+    SLACK_INCIDENT_WEBHOOK_URL,
     SECRET.HoneycombWebhookSecret,
     STRIPE_SECRET_KEY,
     EMAILOCTOPUS_API_KEY,

packages/console/app/src/routes/honeycomb/webhook.ts

@@ -37,31 +37,29 @@ const honeycombWebhookPayload = z.discriminatedUnion("type", [
   }),
 ])
 
-const postDiscordMessage = async (payload: z.infer<typeof honeycombWebhookPayload>) => {
-  const names =
-    payload.type === "custom"
-      ? []
-      : payload.groups.flatMap((item) =>
-          item.group.map((g) => {
-            const result = item.result == null ? undefined : Number(item.result)
-            return `- ${g.value}${
-              result !== undefined && Number.isFinite(result)
-                ? payload.type === "model_low_tps"
-                  ? ` (${Math.round(result)} TPS)`
-                  : ` (${Math.round(result * 100)}% errors)`
-                : ""
-            }`
-          }),
-        )
+const alertDetails = (payload: z.infer<typeof honeycombWebhookPayload>) =>
+  payload.type === "custom"
+    ? []
+    : payload.groups.flatMap((item) =>
+        item.group.map((group) => {
+          const result = item.result == null ? undefined : Number(item.result)
+          return `- ${group.value}${
+            result !== undefined && Number.isFinite(result)
+              ? payload.type === "model_low_tps"
+                ? ` (${Math.round(result)} TPS)`
+                : ` (${Math.round(result * 100)}% errors)`
+              : ""
+          }`
+        }),
+      )
 
+const postDiscordMessage = async (payload: z.infer<typeof honeycombWebhookPayload>) => {
   const content = [
     `[**${payload.isTest ? "[TEST] " : ""}${payload.name ?? "Honeycomb alert"}**](${payload.url})`,
-    ...names,
+    ...alertDetails(payload),
     "",
     `<@&${DISCORD_ALERT_ROLE_ID}>`,
-  ]
-    .filter((line) => line !== undefined)
-    .join("\n")
+  ].join("\n")
 
   return fetch(Resource.DISCORD_INCIDENT_WEBHOOK_URL.value, {
     method: "POST",
@@ -74,6 +72,21 @@ const postDiscordMessage = async (payload: z.infer<typeof honeycombWebhookPayloa
   })
 }
 
+const postSlackMessage = async (payload: z.infer<typeof honeycombWebhookPayload>) => {
+  const text = [
+    `<${payload.url}|*${payload.isTest ? "[TEST] " : ""}${payload.name ?? "Honeycomb alert"}*>`,
+    ...alertDetails(payload),
+    "",
+    "<!channel>",
+  ].join("\n")
+
+  return fetch(Resource.SLACK_INCIDENT_WEBHOOK_URL.value, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ text, unfurl_links: false }),
+  })
+}
+
 export async function POST(input: APIEvent) {
   const token = input.request.headers.get("X-Honeycomb-Webhook-Token")
   if (!safeEqual(token ?? "", Resource.HoneycombWebhookSecret.value)) {
@@ -96,9 +109,10 @@ export async function POST(input: APIEvent) {
     return Response.json({ message: "ignored" }, { status: 200 })
   }
 
-  const response = await postDiscordMessage(parsed.data)
-  if (!response.ok) {
-    return Response.json({ message: "discord webhook failed" }, { status: 502 })
+  const [discord, slack] = await Promise.all([postDiscordMessage(parsed.data), postSlackMessage(parsed.data)])
+  if (!discord.ok || !slack.ok) {
+    console.error("Honeycomb alert delivery failed", { discord: discord.status, slack: slack.status })
+    return Response.json({ message: "alert webhook failed" }, { status: 502 })
   }
 
   return Response.json({ message: "sent" }, { status: 200 })

sst-env.d.ts

@@ -120,6 +120,10 @@ declare module "sst" {
       "type": "sst.sst.Secret"
       "value": string
     }
+    "SLACK_INCIDENT_WEBHOOK_URL": {
+      "type": "sst.sst.Secret"
+      "value": string
+    }
     "STRIPE_PUBLISHABLE_KEY": {
       "type": "sst.sst.Secret"
       "value": string
@@ -309,4 +313,4 @@ declare module "sst" {
 }
 
 import "sst"
-export {}
+export {}