feat: materialization configuration on Endpoint frontend (#40650)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

frontend/src/types.ts

@@ -2156,10 +2156,21 @@ export interface EndpointType extends WithAccessControl {
     updated_at: string
     created_by: UserBasicType | null
     cache_age_seconds: number
+    is_materialized: boolean
     /** Purely local value to determine whether the query endpoint should be highlighted, e.g. as a fresh duplicate. */
     _highlight?: boolean
     /** Last execution time from ClickHouse query_log table */
     last_executed_at?: string
+    materialization?: EndpointMaterializationType
+}
+
+export interface EndpointMaterializationType {
+    can_materialize: boolean
+    reason?: string
+    status?: string
+    error?: string
+    last_materialized_at?: string
+    sync_frequency?: DataWarehouseSyncInterval
 }
 
 export interface DashboardBasicType extends WithAccessControl {

products/endpoints/frontend/EndpointCodeExamples.tsx

@@ -5,6 +5,7 @@ import { LemonSelect } from '@posthog/lemon-ui'
 import { CodeSnippet, Language } from 'lib/components/CodeSnippet'
 
 import { SceneSection } from '~/layout/scenes/components/SceneSection'
+import { isInsightQueryNode } from '~/queries/utils'
 
 import { CodeExampleTab, endpointLogic } from './endpointLogic'
 
@@ -19,10 +20,10 @@ function generateVariablesJson(variables: Record<string, any>): string {
     }
 
     return entries
-        .map(([key, value], index) => {
+        .map(([_, value], index) => {
             const isLast = index === entries.length - 1
             const comma = isLast ? '' : ','
-            return `      "${key}": ${JSON.stringify(value.value)}${comma}`
+            return `      "${value.code_name}": ${JSON.stringify(value.value)}${comma}`
         })
         .join('\n')
 }
@@ -93,11 +94,11 @@ export function EndpointCodeExamples({ tabId }: EndpointCodeExamplesProps): JSX.
     const { setActiveCodeExampleTab } = useActions(endpointLogic({ tabId }))
     const { activeCodeExampleTab, endpoint } = useValues(endpointLogic({ tabId }))
 
-    if (!endpoint) {
+    if (!endpoint || isInsightQueryNode(endpoint.query)) {
         return <></>
     }
 
-    const variables = endpoint.parameters || {}
+    const variables = endpoint.query.variables || {}
 
     const getCodeExample = (tab: CodeExampleTab): string => {
         switch (tab) {

products/endpoints/frontend/EndpointConfiguration.tsx

@@ -1,10 +1,12 @@
 import { useActions, useValues } from 'kea'
 
-import { LemonDivider, LemonSelect } from '@posthog/lemon-ui'
+import { IconDatabase, IconRefresh } from '@posthog/icons'
+import { LemonBanner, LemonDivider, LemonSelect, LemonSwitch } from '@posthog/lemon-ui'
 
 import { LemonField } from 'lib/lemon-ui/LemonField'
 
 import { SceneSection } from '~/layout/scenes/components/SceneSection'
+import { DataWarehouseSyncInterval } from '~/types'
 
 import { endpointLogic } from './endpointLogic'
 
@@ -13,7 +15,6 @@ interface EndpointConfigurationProps {
 }
 
 type CacheAgeOption = number | null
-
 const CACHE_AGE_OPTIONS: { value: CacheAgeOption; label: string }[] = [
     { value: null, label: 'Default caching behavior' },
     { value: 300, label: '5 minutes' },
@@ -25,14 +26,35 @@ const CACHE_AGE_OPTIONS: { value: CacheAgeOption; label: string }[] = [
     { value: 259200, label: '3 days' },
 ]
 
+const SYNC_FREQUENCY_OPTIONS: { value: DataWarehouseSyncInterval; label: string }[] = [
+    { value: '1hour', label: 'Every hour' },
+    { value: '6hour', label: 'Every 6 hours' },
+    { value: '24hour', label: 'Once a day' },
+    { value: '7day', label: 'Once a week' },
+]
+
 export function EndpointConfiguration({ tabId }: EndpointConfigurationProps): JSX.Element {
-    const { endpoint, cacheAge } = useValues(endpointLogic({ tabId }))
-    const { setCacheAge } = useActions(endpointLogic({ tabId }))
+    const { setCacheAge, setSyncFrequency, setIsMaterialized } = useActions(endpointLogic({ tabId }))
+    const {
+        endpoint,
+        cacheAge,
+        syncFrequency,
+        isMaterialized: localIsMaterialized,
+    } = useValues(endpointLogic({ tabId }))
 
     if (!endpoint) {
         return <></>
     }
 
+    const canMaterialize = endpoint.materialization?.can_materialize ?? false
+    const isMaterialized = localIsMaterialized !== null ? localIsMaterialized : endpoint.is_materialized
+    const materializationStatus = endpoint.materialization?.status
+    const lastMaterializedAt = endpoint.materialization?.last_materialized_at
+
+    const handleToggleMaterialization = (): void => {
+        setIsMaterialized(!isMaterialized)
+    }
+
     return (
         <SceneSection title="Configure this endpoint">
             <div className="flex flex-col gap-4 max-w-2xl">
@@ -43,6 +65,59 @@ export function EndpointConfiguration({ tabId }: EndpointConfigurationProps): JS
                 >
                     <LemonSelect value={cacheAge} onChange={setCacheAge} options={CACHE_AGE_OPTIONS} />
                 </LemonField.Pure>
+                <LemonField.Pure
+                    label="Materialization"
+                    info="Pre-compute and store query results in S3 for better query performance."
+                >
+                    <LemonSwitch
+                        label="Enable materialization"
+                        checked={isMaterialized}
+                        onChange={handleToggleMaterialization}
+                        disabled={!canMaterialize}
+                        disabledReason={!canMaterialize ? endpoint.materialization?.reason : undefined}
+                        bordered
+                    />
+                </LemonField.Pure>
+
+                <div className="space-y-4">
+                    {isMaterialized && (
+                        <div className="space-y-3 p-4 bg-accent-3000 border border-border rounded">
+                            <div className="flex items-center justify-between">
+                                <div className="flex items-center gap-2">
+                                    <IconDatabase className="text-lg" />
+                                    <span className="font-medium">Materialization status</span>
+                                </div>
+                                <span className="text-xs px-2 py-1 bg-success-highlight text-success rounded">
+                                    {materializationStatus || 'Pending'}
+                                </span>
+                            </div>
+
+                            {lastMaterializedAt && (
+                                <div className="flex items-center gap-2 text-xs text-secondary">
+                                    <IconRefresh className="text-base" />
+                                    <span>Last materialized: {new Date(lastMaterializedAt).toLocaleString()}</span>
+                                </div>
+                            )}
+
+                            {endpoint.materialization?.error && (
+                                <LemonBanner type="error" className="mt-2">
+                                    {endpoint.materialization.error}
+                                </LemonBanner>
+                            )}
+                        </div>
+                    )}
+
+                    {isMaterialized && (
+                        <LemonField.Pure label="Sync frequency" info="How often the materialization is refreshed">
+                            <LemonSelect
+                                value={syncFrequency || '24hour'}
+                                onChange={setSyncFrequency}
+                                options={SYNC_FREQUENCY_OPTIONS}
+                                disabledReason={!isMaterialized ? 'Requires materializing the endpoint.' : undefined}
+                            />
+                        </LemonField.Pure>
+                    )}
+                </div>
             </div>
             <LemonDivider />
         </SceneSection>

products/endpoints/frontend/EndpointHeader.tsx

@@ -3,6 +3,7 @@ import { useActions, useValues } from 'kea'
 import { LemonButton } from '@posthog/lemon-ui'
 
 import { SceneTitleSection } from '~/layout/scenes/components/SceneTitleSection'
+import { EndpointRequest } from '~/queries/schema/schema-general'
 import { isInsightVizNode } from '~/queries/utils'
 
 import { endpointLogic } from './endpointLogic'
@@ -14,8 +15,11 @@ export interface EndpointSceneHeaderProps {
 
 export const EndpointSceneHeader = ({ tabId }: EndpointSceneHeaderProps): JSX.Element => {
     const { endpoint, endpointLoading, localQuery } = useValues(endpointSceneLogic({ tabId }))
-    const { endpointName, endpointDescription, cacheAge } = useValues(endpointLogic({ tabId }))
-    const { setEndpointDescription, updateEndpoint, createEndpoint, setCacheAge } = useActions(endpointLogic({ tabId }))
+    const { endpointName, endpointDescription, cacheAge, syncFrequency, isMaterialized } = useValues(
+        endpointLogic({ tabId })
+    )
+    const { setEndpointDescription, updateEndpoint, createEndpoint, setCacheAge, setSyncFrequency, setIsMaterialized } =
+        useActions(endpointLogic({ tabId }))
     const { setLocalQuery } = useActions(endpointSceneLogic({ tabId }))
 
     const isNewEndpoint = !endpoint?.name || endpoint.name === 'new-endpoint'
@@ -24,7 +28,15 @@ export const EndpointSceneHeader = ({ tabId }: EndpointSceneHeaderProps): JSX.El
     const hasDescriptionChange = endpointDescription !== null && endpointDescription !== endpoint?.description
     const hasQueryChange = localQuery !== null
     const hasCacheAgeChange = cacheAge !== (endpoint?.cache_age_seconds ?? null)
-    const hasChanges = hasNameChange || hasDescriptionChange || hasQueryChange || hasCacheAgeChange
+    const hasSyncFrequencyChange = syncFrequency !== (endpoint?.materialization?.sync_frequency ?? null)
+    const hasIsMaterializedChange = isMaterialized !== null && isMaterialized !== endpoint?.is_materialized
+    const hasChanges =
+        hasNameChange ||
+        hasDescriptionChange ||
+        hasQueryChange ||
+        hasCacheAgeChange ||
+        hasSyncFrequencyChange ||
+        hasIsMaterializedChange
 
     const handleSave = (): void => {
         let queryToSave = (localQuery || endpoint?.query) as any
@@ -35,17 +47,20 @@ export const EndpointSceneHeader = ({ tabId }: EndpointSceneHeaderProps): JSX.El
 
         if (isNewEndpoint) {
             createEndpoint({
-                name: endpointName || endpoint?.name || '',
-                description: endpointDescription || endpoint?.description,
+                name: endpointName || '',
+                description: endpointDescription || undefined,
                 query: queryToSave,
             })
         } else {
-            updateEndpoint(endpoint.name, {
-                name: endpointName || endpoint?.name,
-                description: endpointDescription || endpoint?.description,
-                cache_age_seconds: cacheAge ?? undefined,
-                query: queryToSave,
-            })
+            const updatePayload: Partial<EndpointRequest> = {
+                description: hasDescriptionChange ? endpointDescription : undefined,
+                cache_age_seconds: hasCacheAgeChange ? (cacheAge ?? undefined) : undefined,
+                query: hasQueryChange ? queryToSave : undefined,
+                is_materialized: hasIsMaterializedChange ? isMaterialized : undefined,
+                sync_frequency: hasSyncFrequencyChange ? (syncFrequency ?? undefined) : undefined,
+            }
+
+            updateEndpoint(endpoint.name, updatePayload)
         }
     }
 
@@ -53,6 +68,8 @@ export const EndpointSceneHeader = ({ tabId }: EndpointSceneHeaderProps): JSX.El
         if (endpoint) {
             setEndpointDescription(endpoint.description || '')
             setCacheAge(endpoint.cache_age_seconds ?? null)
+            setSyncFrequency(endpoint.materialization?.sync_frequency ?? null)
+            setIsMaterialized(null)
         }
         setLocalQuery(null)
     }

products/endpoints/frontend/endpointLogic.tsx

@@ -4,12 +4,12 @@ import { router } from 'kea-router'
 
 import api from 'lib/api'
 import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
-import { slugify } from 'lib/utils'
+import { debounce, slugify } from 'lib/utils'
 import { permanentlyMount } from 'lib/utils/kea-logic-builders'
 import { urls } from 'scenes/urls'
 
 import { EndpointRequest, HogQLQuery, InsightQueryNode, NodeKind } from '~/queries/schema/schema-general'
-import { EndpointType } from '~/types'
+import { DataWarehouseSyncInterval, EndpointType } from '~/types'
 
 import type { endpointLogicType } from './endpointLogicType'
 import { endpointsLogic } from './endpointsLogic'
@@ -43,18 +43,17 @@ export const endpointLogic = kea<endpointLogicType>([
         setIsUpdateMode: (isUpdateMode: boolean) => ({ isUpdateMode }),
         setSelectedEndpointName: (selectedEndpointName: string | null) => ({ selectedEndpointName }),
         setCacheAge: (cacheAge: number | null) => ({ cacheAge }),
+        setSyncFrequency: (syncFrequency: DataWarehouseSyncInterval | null) => ({ syncFrequency }),
+        setIsMaterialized: (isMaterialized: boolean | null) => ({ isMaterialized }),
         createEndpoint: (request: EndpointRequest) => ({ request }),
         createEndpointSuccess: (response: any) => ({ response }),
-        createEndpointFailure: (error: any) => ({ error }),
+        createEndpointFailure: () => ({}),
         updateEndpoint: (name: string, request: Partial<EndpointRequest>) => ({ name, request }),
         updateEndpointSuccess: (response: any) => ({ response }),
-        updateEndpointFailure: (error: any) => ({ error }),
+        updateEndpointFailure: () => ({}),
         deleteEndpoint: (name: string) => ({ name }),
         deleteEndpointSuccess: (response: any) => ({ response }),
-        deleteEndpointFailure: (error: any) => ({ error }),
-        deactivateEndpoint: (name: string) => ({ name }),
-        deactivateEndpointSuccess: (response: any) => ({ response }),
-        deactivateEndpointFailure: (error: any) => ({ error }),
+        deleteEndpointFailure: () => ({}),
     }),
     reducers({
         endpointName: [null as string | null, { setEndpointName: (_, { endpointName }) => endpointName }],
@@ -81,6 +80,18 @@ export const endpointLogic = kea<endpointLogicType>([
                 setCacheAge: (_, { cacheAge }) => cacheAge,
             },
         ],
+        syncFrequency: [
+            '24hour' as DataWarehouseSyncInterval | null,
+            {
+                setSyncFrequency: (_, { syncFrequency }) => syncFrequency,
+            },
+        ],
+        isMaterialized: [
+            null as boolean | null,
+            {
+                setIsMaterialized: (_, { isMaterialized }) => isMaterialized,
+            },
+        ],
     }),
     loaders(({ actions }) => ({
         endpoint: [
@@ -104,99 +115,77 @@ export const endpointLogic = kea<endpointLogicType>([
 
                     // TODO: This does not belong here. Refactor to the endpointSceneLogic?
                     actions.setCacheAge(endpoint.cache_age_seconds ?? null)
+                    actions.setSyncFrequency(endpoint.materialization?.sync_frequency ?? null)
+                    actions.setIsMaterialized(endpoint.is_materialized ?? null)
 
                     return endpoint
                 },
             },
         ],
     })),
-    listeners(({ actions }) => ({
-        createEndpoint: async ({ request }) => {
-            try {
-                if (request.name) {
-                    request.name = slugify(request.name)
+    listeners(({ actions }) => {
+        const reloadEndpoint = debounce((name: string): void => {
+            actions.loadEndpoint(name)
+        }, 2000)
+        return {
+            createEndpoint: async ({ request }) => {
+                try {
+                    if (request.name) {
+                        request.name = slugify(request.name)
+                    }
+                    const response = await api.endpoint.create(request)
+                    actions.createEndpointSuccess(response)
+                } catch (error) {
+                    console.error('Failed to create endpoint:', error)
+                    actions.createEndpointFailure()
                 }
-                const response = await api.endpoint.create(request)
-                actions.createEndpointSuccess(response)
-            } catch (error) {
-                console.error('Failed to create endpoint:', error)
-                actions.createEndpointFailure(error)
-            }
-        },
-        createEndpointSuccess: ({ response }) => {
-            actions.setEndpointName('')
-            actions.setEndpointDescription('')
-            lemonToast.success(
-                <>
-                    Endpoint created successfully!
-                    <br />
-                    You will be redirected to the endpoint page.
-                </>,
-                {
-                    onClose: () => {
-                        router.actions.push(urls.endpoint(response.name))
+            },
+            createEndpointSuccess: ({ response }) => {
+                actions.setEndpointName('')
+                actions.setEndpointDescription('')
+                lemonToast.success(<>Endpoint created</>, {
+                    button: {
+                        label: 'View',
+                        action: () => router.actions.push(urls.endpoint(response.name)),
                     },
-                }
-            )
-        },
-        createEndpointFailure: ({ error }) => {
-            console.error('Failed to create endpoint:', error)
-            lemonToast.error('Failed to create endpoint')
-        },
-        updateEndpoint: async ({ name, request }) => {
-            try {
-                const response = await api.endpoint.update(name, request)
-                actions.updateEndpointSuccess(response)
-            } catch (error) {
-                console.error('Failed to update endpoint:', error)
-                actions.updateEndpointFailure(error)
-            }
-        },
-        updateEndpointSuccess: ({ response }) => {
-            lemonToast.success('Endpoint updated successfully')
-            actions.loadEndpoint(response.name)
-        },
-        updateEndpointFailure: ({ error }) => {
-            console.error('Failed to update endpoint:', error)
-            lemonToast.error('Failed to update endpoint')
-        },
-        deleteEndpoint: async ({ name }) => {
-            try {
-                // TODO: Add confirmation dialog
-                await api.endpoint.delete(name)
-                actions.deleteEndpointSuccess(name)
-            } catch (error) {
-                console.error('Failed to delete endpoint:', error)
-                actions.deleteEndpointFailure(error)
-            }
-        },
-        deleteEndpointSuccess: () => {
-            lemonToast.success('Endpoint deleted successfully')
-            actions.loadEndpoints()
-        },
-        deleteEndpointFailure: ({ error }) => {
-            console.error('Failed to delete endpoint:', error)
-            lemonToast.error('Failed to delete endpoint')
-        },
-        deactivateEndpoint: async ({ name }) => {
-            try {
-                await api.endpoint.update(name, {
-                    is_active: false,
                 })
-                actions.deactivateEndpointSuccess({})
-            } catch (error) {
-                console.error('Failed to deactivate endpoint:', error)
-                actions.deactivateEndpointFailure(error)
-            }
-        },
-        deactivateEndpointSuccess: () => {
-            lemonToast.success('Endpoint deactivated successfully')
-            actions.loadEndpoints()
-        },
-        deactivateEndpointFailure: ({ error }) => {
-            console.error('Failed to deactivate endpoint:', error)
-            lemonToast.error('Failed to deactivate endpoint')
-        },
-    })),
+            },
+            createEndpointFailure: () => {
+                lemonToast.error('Failed to create endpoint')
+            },
+            updateEndpoint: async ({ name, request }) => {
+                try {
+                    const response = await api.endpoint.update(name, request)
+                    actions.updateEndpointSuccess(response)
+                } catch (error) {
+                    console.error('Failed to update endpoint:', error)
+                    actions.updateEndpointFailure()
+                }
+            },
+            updateEndpointSuccess: ({ response }) => {
+                lemonToast.success('Endpoint updated')
+                reloadEndpoint(response.name)
+            },
+            updateEndpointFailure: () => {
+                lemonToast.error('Failed to update endpoint')
+            },
+            deleteEndpoint: async ({ name }) => {
+                try {
+                    await api.endpoint.delete(name)
+                    actions.deleteEndpointSuccess(name)
+                } catch (error) {
+                    console.error('Failed to delete endpoint:', error)
+                    actions.deleteEndpointFailure()
+                }
+            },
+            deleteEndpointSuccess: () => {
+                lemonToast.success('Endpoint deleted')
+                actions.loadEndpoints()
+            },
+            deleteEndpointFailure: () => {
+                lemonToast.error('Failed to delete endpoint')
+            },
+        }
+    }),
     permanentlyMount(),
 ])