mirror of
https://github.com/BillyOutlast/posthog.git
synced 2026-02-04 03:01:23 +01:00
feat(data-warehouse-scene): view tab (#41313)
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -83,7 +83,9 @@ import {
|
|||||||
DataWarehouseJobStatsRequestPayload,
|
DataWarehouseJobStatsRequestPayload,
|
||||||
DataWarehouseManagedViewsetSavedQuery,
|
DataWarehouseManagedViewsetSavedQuery,
|
||||||
DataWarehouseSavedQuery,
|
DataWarehouseSavedQuery,
|
||||||
|
DataWarehouseSavedQueryDependencies,
|
||||||
DataWarehouseSavedQueryDraft,
|
DataWarehouseSavedQueryDraft,
|
||||||
|
DataWarehouseSavedQueryRunHistory,
|
||||||
DataWarehouseSourceRowCount,
|
DataWarehouseSourceRowCount,
|
||||||
DataWarehouseTable,
|
DataWarehouseTable,
|
||||||
DataWarehouseViewLink,
|
DataWarehouseViewLink,
|
||||||
@@ -3753,6 +3755,14 @@ const api = {
|
|||||||
.withAction('descendants')
|
.withAction('descendants')
|
||||||
.create({ data: { level } })
|
.create({ data: { level } })
|
||||||
},
|
},
|
||||||
|
async dependencies(viewId: DataWarehouseSavedQuery['id']): Promise<DataWarehouseSavedQueryDependencies> {
|
||||||
|
return await new ApiRequest().dataWarehouseSavedQuery(viewId).withAction('dependencies').get()
|
||||||
|
},
|
||||||
|
async runHistory(
|
||||||
|
viewId: DataWarehouseSavedQuery['id']
|
||||||
|
): Promise<{ run_history: DataWarehouseSavedQueryRunHistory[] }> {
|
||||||
|
return await new ApiRequest().dataWarehouseSavedQuery(viewId).withAction('run_history').get()
|
||||||
|
},
|
||||||
dataWarehouseDataModelingJobs: {
|
dataWarehouseDataModelingJobs: {
|
||||||
async list(
|
async list(
|
||||||
savedQueryId: DataWarehouseSavedQuery['id'],
|
savedQueryId: DataWarehouseSavedQuery['id'],
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import { SceneTitleSection } from '~/layout/scenes/components/SceneTitleSection'
|
|||||||
import { DataWarehouseTab, dataWarehouseSceneLogic } from './dataWarehouseSceneLogic'
|
import { DataWarehouseTab, dataWarehouseSceneLogic } from './dataWarehouseSceneLogic'
|
||||||
import { OverviewTab } from './scene/OverviewTab'
|
import { OverviewTab } from './scene/OverviewTab'
|
||||||
import { SourcesTab } from './scene/SourcesTab'
|
import { SourcesTab } from './scene/SourcesTab'
|
||||||
|
import { ViewsTab } from './scene/ViewsTab'
|
||||||
|
|
||||||
export const scene: SceneExport = { component: DataWarehouseScene, logic: dataWarehouseSceneLogic }
|
export const scene: SceneExport = { component: DataWarehouseScene, logic: dataWarehouseSceneLogic }
|
||||||
|
|
||||||
@@ -68,6 +69,11 @@ export function DataWarehouseScene(): JSX.Element {
|
|||||||
label: 'Sources',
|
label: 'Sources',
|
||||||
content: <SourcesTab />,
|
content: <SourcesTab />,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
key: DataWarehouseTab.VIEWS,
|
||||||
|
label: 'Views',
|
||||||
|
content: <ViewsTab />,
|
||||||
|
},
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
</SceneContent>
|
</SceneContent>
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ const REFRESH_INTERVAL = 10000
|
|||||||
export enum DataWarehouseTab {
|
export enum DataWarehouseTab {
|
||||||
OVERVIEW = 'overview',
|
OVERVIEW = 'overview',
|
||||||
SOURCES = 'sources',
|
SOURCES = 'sources',
|
||||||
|
VIEWS = 'views',
|
||||||
}
|
}
|
||||||
|
|
||||||
export const dataWarehouseSceneLogic = kea<dataWarehouseSceneLogicType>([
|
export const dataWarehouseSceneLogic = kea<dataWarehouseSceneLogicType>([
|
||||||
|
|||||||
313
frontend/src/scenes/data-warehouse/scene/ViewsTab.tsx
Normal file
313
frontend/src/scenes/data-warehouse/scene/ViewsTab.tsx
Normal file
@@ -0,0 +1,313 @@
|
|||||||
|
import { useActions, useValues } from 'kea'
|
||||||
|
|
||||||
|
import { LemonButton, LemonInput, LemonTable, LemonTag, LemonTagType, Spinner, Tooltip } from '@posthog/lemon-ui'
|
||||||
|
|
||||||
|
import { TZLabel } from 'lib/components/TZLabel'
|
||||||
|
import { More } from 'lib/lemon-ui/LemonButton/More'
|
||||||
|
import { LemonTableLink } from 'lib/lemon-ui/LemonTable/LemonTableLink'
|
||||||
|
import { humanFriendlyDetailedTime } from 'lib/utils'
|
||||||
|
import { urls } from 'scenes/urls'
|
||||||
|
|
||||||
|
import { DataWarehouseSavedQuery, DataWarehouseSavedQueryRunHistory } from '~/types'
|
||||||
|
|
||||||
|
import { viewsTabLogic } from './viewsTabLogic'
|
||||||
|
|
||||||
|
const STATUS_TAG_SETTINGS: Record<string, LemonTagType> = {
|
||||||
|
Running: 'primary',
|
||||||
|
Completed: 'success',
|
||||||
|
Failed: 'danger',
|
||||||
|
Cancelled: 'muted',
|
||||||
|
Modified: 'warning',
|
||||||
|
}
|
||||||
|
|
||||||
|
function RunHistoryDisplay({
|
||||||
|
runHistory,
|
||||||
|
loading,
|
||||||
|
}: {
|
||||||
|
runHistory?: DataWarehouseSavedQueryRunHistory[]
|
||||||
|
loading?: boolean
|
||||||
|
}): JSX.Element {
|
||||||
|
if (loading) {
|
||||||
|
return <Spinner className="text-sm" />
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!runHistory || runHistory.length === 0) {
|
||||||
|
return <span className="text-muted">-</span>
|
||||||
|
}
|
||||||
|
|
||||||
|
// Show up to 5 most recent runs
|
||||||
|
const displayRuns = runHistory.slice(0, 5)
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex gap-1">
|
||||||
|
{displayRuns.map((run, index) => {
|
||||||
|
const friendlyTime = run.timestamp ? humanFriendlyDetailedTime(run.timestamp) : ''
|
||||||
|
return (
|
||||||
|
<Tooltip
|
||||||
|
key={index}
|
||||||
|
title={`${run.status}${friendlyTime ? ` - ${friendlyTime}` : ''}`}
|
||||||
|
placement="top"
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className={`w-4 h-4 rounded-sm ${run.status === 'Completed' ? 'bg-success' : 'bg-danger'}`}
|
||||||
|
/>
|
||||||
|
</Tooltip>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function DependencyCount({ count, loading }: { count?: number; loading?: boolean }): JSX.Element {
|
||||||
|
if (loading || count === undefined) {
|
||||||
|
return <Spinner className="text-sm" />
|
||||||
|
}
|
||||||
|
return <span>{count}</span>
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ViewsTab(): JSX.Element {
|
||||||
|
const {
|
||||||
|
filteredViews,
|
||||||
|
filteredMaterializedViews,
|
||||||
|
viewsLoading,
|
||||||
|
searchTerm,
|
||||||
|
dependenciesMapLoading,
|
||||||
|
runHistoryMapLoading,
|
||||||
|
materializedViewsCurrentPage,
|
||||||
|
viewsCurrentPage,
|
||||||
|
} = useValues(viewsTabLogic)
|
||||||
|
const { setSearchTerm, deleteView, runMaterialization, setMaterializedViewsPage, setViewsPage } =
|
||||||
|
useActions(viewsTabLogic)
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
{(filteredViews.length > 0 || filteredMaterializedViews.length > 0 || searchTerm) && (
|
||||||
|
<div className="flex gap-2 justify-between items-center">
|
||||||
|
<LemonInput
|
||||||
|
type="search"
|
||||||
|
placeholder="Search views..."
|
||||||
|
onChange={setSearchTerm}
|
||||||
|
value={searchTerm}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Materialized Views Section */}
|
||||||
|
{filteredMaterializedViews.length > 0 && (
|
||||||
|
<div>
|
||||||
|
<h3 className="text-lg font-semibold mb-2">Materialized views</h3>
|
||||||
|
<p className="text-muted mb-2">
|
||||||
|
Materialized views are refreshed on a schedule and stored as tables for faster query
|
||||||
|
performance.
|
||||||
|
</p>
|
||||||
|
<LemonTable
|
||||||
|
dataSource={filteredMaterializedViews}
|
||||||
|
loading={viewsLoading}
|
||||||
|
columns={[
|
||||||
|
{
|
||||||
|
title: 'Name',
|
||||||
|
key: 'name',
|
||||||
|
render: (_, view: DataWarehouseSavedQuery) => (
|
||||||
|
<LemonTableLink
|
||||||
|
to={urls.sqlEditor(undefined, view.id)}
|
||||||
|
title={view.name}
|
||||||
|
description="Materialized view"
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Last run',
|
||||||
|
key: 'last_run_at',
|
||||||
|
render: (_, view) => {
|
||||||
|
return view.last_run_at ? (
|
||||||
|
<TZLabel time={view.last_run_at} formatDate="MMM DD, YYYY" formatTime="HH:mm" />
|
||||||
|
) : (
|
||||||
|
'Never'
|
||||||
|
)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Status',
|
||||||
|
key: 'status',
|
||||||
|
render: (_, view) => {
|
||||||
|
if (!view.status) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
const tagContent = (
|
||||||
|
<LemonTag type={STATUS_TAG_SETTINGS[view.status] || 'default'}>
|
||||||
|
{view.status}
|
||||||
|
</LemonTag>
|
||||||
|
)
|
||||||
|
return view.latest_error && view.status === 'Failed' ? (
|
||||||
|
<Tooltip title={view.latest_error}>{tagContent}</Tooltip>
|
||||||
|
) : (
|
||||||
|
tagContent
|
||||||
|
)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Run history',
|
||||||
|
key: 'run_history',
|
||||||
|
tooltip: 'Recent run status (up to 5 most recent)',
|
||||||
|
render: (_, view) => (
|
||||||
|
<RunHistoryDisplay runHistory={view.run_history} loading={runHistoryMapLoading} />
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Upstream',
|
||||||
|
key: 'upstream_count',
|
||||||
|
tooltip: 'Number of immediate upstream dependencies',
|
||||||
|
render: (_, view) => (
|
||||||
|
<DependencyCount
|
||||||
|
count={view.upstream_dependency_count}
|
||||||
|
loading={dependenciesMapLoading}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Downstream',
|
||||||
|
key: 'downstream_count',
|
||||||
|
tooltip: 'Number of immediate downstream dependencies',
|
||||||
|
render: (_, view) => (
|
||||||
|
<DependencyCount
|
||||||
|
count={view.downstream_dependency_count}
|
||||||
|
loading={dependenciesMapLoading}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'actions',
|
||||||
|
width: 0,
|
||||||
|
render: (_, view) => (
|
||||||
|
<More
|
||||||
|
overlay={
|
||||||
|
<>
|
||||||
|
<LemonButton onClick={() => runMaterialization(view.id)}>
|
||||||
|
Run now
|
||||||
|
</LemonButton>
|
||||||
|
<LemonButton status="danger" onClick={() => deleteView(view.id)}>
|
||||||
|
Delete
|
||||||
|
</LemonButton>
|
||||||
|
</>
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
pagination={{
|
||||||
|
pageSize: 10,
|
||||||
|
currentPage: materializedViewsCurrentPage,
|
||||||
|
onForward: () => {
|
||||||
|
setMaterializedViewsPage(materializedViewsCurrentPage + 1)
|
||||||
|
},
|
||||||
|
onBackward: () => {
|
||||||
|
setMaterializedViewsPage(materializedViewsCurrentPage - 1)
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Regular Views Section */}
|
||||||
|
{filteredViews.length > 0 && (
|
||||||
|
<div>
|
||||||
|
<h3 className="text-lg font-semibold mb-2">Views</h3>
|
||||||
|
<p className="text-muted mb-2">
|
||||||
|
Views are virtual tables created from SQL queries. They are computed on-the-fly when queried.
|
||||||
|
</p>
|
||||||
|
<LemonTable
|
||||||
|
dataSource={filteredViews}
|
||||||
|
loading={viewsLoading}
|
||||||
|
columns={[
|
||||||
|
{
|
||||||
|
title: 'Name',
|
||||||
|
key: 'name',
|
||||||
|
render: (_, view: DataWarehouseSavedQuery) => (
|
||||||
|
<LemonTableLink
|
||||||
|
to={urls.sqlEditor(undefined, view.id)}
|
||||||
|
title={view.name}
|
||||||
|
description="View"
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Created',
|
||||||
|
key: 'created_at',
|
||||||
|
render: (_, view) =>
|
||||||
|
view.created_at ? (
|
||||||
|
<TZLabel time={view.created_at} formatDate="MMM DD, YYYY" formatTime="HH:mm" />
|
||||||
|
) : (
|
||||||
|
'-'
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Upstream',
|
||||||
|
key: 'upstream_count',
|
||||||
|
tooltip: 'Number of immediate upstream dependencies',
|
||||||
|
render: (_, view) => (
|
||||||
|
<DependencyCount
|
||||||
|
count={view.upstream_dependency_count}
|
||||||
|
loading={dependenciesMapLoading}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
title: 'Downstream',
|
||||||
|
key: 'downstream_count',
|
||||||
|
tooltip: 'Number of immediate downstream dependencies',
|
||||||
|
render: (_, view) => (
|
||||||
|
<DependencyCount
|
||||||
|
count={view.downstream_dependency_count}
|
||||||
|
loading={dependenciesMapLoading}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: 'actions',
|
||||||
|
width: 0,
|
||||||
|
render: (_, view) => (
|
||||||
|
<More
|
||||||
|
overlay={
|
||||||
|
<>
|
||||||
|
<LemonButton status="danger" onClick={() => deleteView(view.id)}>
|
||||||
|
Delete
|
||||||
|
</LemonButton>
|
||||||
|
</>
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
pagination={{
|
||||||
|
pageSize: 10,
|
||||||
|
currentPage: viewsCurrentPage,
|
||||||
|
onForward: () => {
|
||||||
|
setViewsPage(viewsCurrentPage + 1)
|
||||||
|
},
|
||||||
|
onBackward: () => {
|
||||||
|
setViewsPage(viewsCurrentPage - 1)
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Empty State */}
|
||||||
|
{!viewsLoading && filteredViews.length === 0 && filteredMaterializedViews.length === 0 && (
|
||||||
|
<div className="text-center py-12">
|
||||||
|
<h3 className="text-xl font-semibold mb-2">No views found</h3>
|
||||||
|
{searchTerm ? (
|
||||||
|
<p className="text-muted">No views match your search. Try adjusting your search term.</p>
|
||||||
|
) : (
|
||||||
|
<p className="text-muted">
|
||||||
|
Create your first view to transform and organize your data warehouse tables.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
<LemonButton type="primary" to={urls.sqlEditor()} className="inline-block">
|
||||||
|
Create view
|
||||||
|
</LemonButton>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
232
frontend/src/scenes/data-warehouse/scene/viewsTabLogic.ts
Normal file
232
frontend/src/scenes/data-warehouse/scene/viewsTabLogic.ts
Normal file
@@ -0,0 +1,232 @@
|
|||||||
|
import { actions, afterMount, connect, kea, listeners, path, reducers, selectors } from 'kea'
|
||||||
|
import { loaders } from 'kea-loaders'
|
||||||
|
|
||||||
|
import { LemonDialog } from '@posthog/lemon-ui'
|
||||||
|
|
||||||
|
import api from 'lib/api'
|
||||||
|
import { lemonToast } from 'lib/lemon-ui/LemonToast'
|
||||||
|
|
||||||
|
import {
|
||||||
|
DataWarehouseSavedQuery,
|
||||||
|
DataWarehouseSavedQueryDependencies,
|
||||||
|
DataWarehouseSavedQueryRunHistory,
|
||||||
|
} from '~/types'
|
||||||
|
|
||||||
|
import { dataWarehouseViewsLogic } from '../saved_queries/dataWarehouseViewsLogic'
|
||||||
|
import type { viewsTabLogicType } from './viewsTabLogicType'
|
||||||
|
|
||||||
|
const PAGE_SIZE = 10
|
||||||
|
|
||||||
|
export const viewsTabLogic = kea<viewsTabLogicType>([
|
||||||
|
path(['scenes', 'data-warehouse', 'scene', 'viewsTabLogic']),
|
||||||
|
connect(() => ({
|
||||||
|
values: [dataWarehouseViewsLogic, ['dataWarehouseSavedQueries', 'dataWarehouseSavedQueriesLoading']],
|
||||||
|
actions: [dataWarehouseViewsLogic, ['deleteDataWarehouseSavedQuery', 'runDataWarehouseSavedQuery']],
|
||||||
|
})),
|
||||||
|
actions({
|
||||||
|
setSearchTerm: (searchTerm: string) => ({ searchTerm }),
|
||||||
|
deleteView: (viewId: string) => ({ viewId }),
|
||||||
|
runMaterialization: (viewId: string) => ({ viewId }),
|
||||||
|
loadDependenciesForViews: (viewIds: string[]) => ({ viewIds }),
|
||||||
|
loadRunHistoryForViews: (viewIds: string[]) => ({ viewIds }),
|
||||||
|
setMaterializedViewsPage: (page: number) => ({ page }),
|
||||||
|
setViewsPage: (page: number) => ({ page }),
|
||||||
|
loadVisibleData: true,
|
||||||
|
}),
|
||||||
|
reducers({
|
||||||
|
searchTerm: [
|
||||||
|
'' as string,
|
||||||
|
{
|
||||||
|
setSearchTerm: (_, { searchTerm }) => searchTerm,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
materializedViewsCurrentPage: [
|
||||||
|
1 as number,
|
||||||
|
{
|
||||||
|
setMaterializedViewsPage: (_, { page }) => page,
|
||||||
|
setSearchTerm: () => 1, // Reset to page 1 on search
|
||||||
|
},
|
||||||
|
],
|
||||||
|
viewsCurrentPage: [
|
||||||
|
1 as number,
|
||||||
|
{
|
||||||
|
setViewsPage: (_, { page }) => page,
|
||||||
|
setSearchTerm: () => 1, // Reset to page 1 on search
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
loaders(({ values }) => ({
|
||||||
|
dependenciesMap: [
|
||||||
|
{} as Record<string, DataWarehouseSavedQueryDependencies>,
|
||||||
|
{
|
||||||
|
loadDependenciesForViews: async ({ viewIds }) => {
|
||||||
|
// Filter out views we've already loaded
|
||||||
|
const viewsToLoad = viewIds.filter((id) => !values.dependenciesMap[id])
|
||||||
|
|
||||||
|
if (viewsToLoad.length === 0) {
|
||||||
|
return values.dependenciesMap
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = await Promise.all(
|
||||||
|
viewsToLoad.map(async (viewId) => {
|
||||||
|
try {
|
||||||
|
const data = await api.dataWarehouseSavedQueries.dependencies(viewId)
|
||||||
|
return { viewId, data }
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Failed to load dependencies for view ${viewId}:`, error)
|
||||||
|
return { viewId, data: { upstream_count: 0, downstream_count: 0 } }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
const newMap = { ...values.dependenciesMap }
|
||||||
|
results.forEach(({ viewId, data }) => {
|
||||||
|
newMap[viewId] = data
|
||||||
|
})
|
||||||
|
return newMap
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
runHistoryMap: [
|
||||||
|
{} as Record<string, DataWarehouseSavedQueryRunHistory[]>,
|
||||||
|
{
|
||||||
|
loadRunHistoryForViews: async ({ viewIds }) => {
|
||||||
|
// Filter out views we've already loaded
|
||||||
|
const viewsToLoad = viewIds.filter((id) => !values.runHistoryMap[id])
|
||||||
|
|
||||||
|
if (viewsToLoad.length === 0) {
|
||||||
|
return values.runHistoryMap
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = await Promise.all(
|
||||||
|
viewsToLoad.map(async (viewId) => {
|
||||||
|
try {
|
||||||
|
const data = await api.dataWarehouseSavedQueries.runHistory(viewId)
|
||||||
|
return { viewId, data: data.run_history }
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Failed to load run history for view ${viewId}:`, error)
|
||||||
|
return { viewId, data: [] }
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
const newMap = { ...values.runHistoryMap }
|
||||||
|
results.forEach(({ viewId, data }) => {
|
||||||
|
newMap[viewId] = data
|
||||||
|
})
|
||||||
|
return newMap
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})),
|
||||||
|
selectors({
|
||||||
|
viewsLoading: [(s) => [s.dataWarehouseSavedQueriesLoading], (loading): boolean => loading],
|
||||||
|
enrichedQueries: [
|
||||||
|
(s) => [s.dataWarehouseSavedQueries, s.dependenciesMap, s.runHistoryMap],
|
||||||
|
(queries, dependenciesMap, runHistoryMap): DataWarehouseSavedQuery[] => {
|
||||||
|
return queries.map((query) => ({
|
||||||
|
...query,
|
||||||
|
upstream_dependency_count: dependenciesMap[query.id]?.upstream_count,
|
||||||
|
downstream_dependency_count: dependenciesMap[query.id]?.downstream_count,
|
||||||
|
run_history: runHistoryMap[query.id],
|
||||||
|
}))
|
||||||
|
},
|
||||||
|
],
|
||||||
|
filteredViews: [
|
||||||
|
(s) => [s.enrichedQueries, s.searchTerm],
|
||||||
|
(queries, searchTerm): DataWarehouseSavedQuery[] => {
|
||||||
|
const views = queries.filter((q) => !q.is_materialized)
|
||||||
|
if (!searchTerm) {
|
||||||
|
return views
|
||||||
|
}
|
||||||
|
return views.filter((v) => v.name.toLowerCase().includes(searchTerm.toLowerCase()))
|
||||||
|
},
|
||||||
|
],
|
||||||
|
filteredMaterializedViews: [
|
||||||
|
(s) => [s.enrichedQueries, s.searchTerm],
|
||||||
|
(queries, searchTerm): DataWarehouseSavedQuery[] => {
|
||||||
|
const views = queries.filter((q) => q.is_materialized)
|
||||||
|
if (!searchTerm) {
|
||||||
|
return views
|
||||||
|
}
|
||||||
|
return views.filter((v) => v.name.toLowerCase().includes(searchTerm.toLowerCase()))
|
||||||
|
},
|
||||||
|
],
|
||||||
|
visibleMaterializedViews: [
|
||||||
|
(s) => [s.filteredMaterializedViews, s.materializedViewsCurrentPage],
|
||||||
|
(views, currentPage): DataWarehouseSavedQuery[] => {
|
||||||
|
const startIndex = (currentPage - 1) * PAGE_SIZE
|
||||||
|
const endIndex = startIndex + PAGE_SIZE
|
||||||
|
return views.slice(startIndex, endIndex)
|
||||||
|
},
|
||||||
|
],
|
||||||
|
visibleViews: [
|
||||||
|
(s) => [s.filteredViews, s.viewsCurrentPage],
|
||||||
|
(views, currentPage): DataWarehouseSavedQuery[] => {
|
||||||
|
const startIndex = (currentPage - 1) * PAGE_SIZE
|
||||||
|
const endIndex = startIndex + PAGE_SIZE
|
||||||
|
return views.slice(startIndex, endIndex)
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
listeners(({ actions, values }) => ({
|
||||||
|
deleteView: ({ viewId }) => {
|
||||||
|
LemonDialog.open({
|
||||||
|
title: 'Delete view?',
|
||||||
|
description: 'Are you sure you want to delete this view? This action cannot be undone.',
|
||||||
|
primaryButton: {
|
||||||
|
children: 'Delete',
|
||||||
|
status: 'danger',
|
||||||
|
onClick: () => {
|
||||||
|
actions.deleteDataWarehouseSavedQuery(viewId)
|
||||||
|
lemonToast.success('View deleted successfully')
|
||||||
|
},
|
||||||
|
},
|
||||||
|
secondaryButton: {
|
||||||
|
children: 'Cancel',
|
||||||
|
},
|
||||||
|
})
|
||||||
|
},
|
||||||
|
runMaterialization: ({ viewId }) => {
|
||||||
|
actions.runDataWarehouseSavedQuery(viewId)
|
||||||
|
},
|
||||||
|
loadDataWarehouseSavedQueriesSuccess: () => {
|
||||||
|
// Load data for initially visible items
|
||||||
|
actions.loadVisibleData()
|
||||||
|
},
|
||||||
|
setMaterializedViewsPage: () => {
|
||||||
|
// Load data when page changes
|
||||||
|
actions.loadVisibleData()
|
||||||
|
},
|
||||||
|
setViewsPage: () => {
|
||||||
|
// Load data when page changes
|
||||||
|
actions.loadVisibleData()
|
||||||
|
},
|
||||||
|
setSearchTerm: () => {
|
||||||
|
// Load data when search changes (pagination resets to page 1)
|
||||||
|
actions.loadVisibleData()
|
||||||
|
},
|
||||||
|
loadVisibleData: () => {
|
||||||
|
// Get IDs of all currently visible items
|
||||||
|
const visibleMaterializedViewIds = values.visibleMaterializedViews.map((v) => v.id)
|
||||||
|
const visibleViewIds = values.visibleViews.map((v) => v.id)
|
||||||
|
const allVisibleIds = [...visibleMaterializedViewIds, ...visibleViewIds]
|
||||||
|
|
||||||
|
// Load dependencies for all visible items (if not already loaded)
|
||||||
|
if (allVisibleIds.length > 0) {
|
||||||
|
actions.loadDependenciesForViews(allVisibleIds)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load run history only for visible materialized views (if not already loaded)
|
||||||
|
if (visibleMaterializedViewIds.length > 0) {
|
||||||
|
actions.loadRunHistoryForViews(visibleMaterializedViewIds)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
afterMount(({ actions, values }) => {
|
||||||
|
// If views are already loaded (e.g., from cache), load visible data immediately
|
||||||
|
if (values.dataWarehouseSavedQueries.length > 0) {
|
||||||
|
actions.loadVisibleData()
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
])
|
||||||
@@ -4790,6 +4790,18 @@ export interface DataWarehouseTable {
|
|||||||
|
|
||||||
export type DataWarehouseTableTypes = 'CSV' | 'Parquet' | 'JSON' | 'CSVWithNames'
|
export type DataWarehouseTableTypes = 'CSV' | 'Parquet' | 'JSON' | 'CSVWithNames'
|
||||||
|
|
||||||
|
export type DataModelingJobStatus = 'Running' | 'Completed' | 'Failed' | 'Cancelled'
|
||||||
|
|
||||||
|
export interface DataWarehouseSavedQueryRunHistory {
|
||||||
|
status: DataModelingJobStatus
|
||||||
|
timestamp?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DataWarehouseSavedQueryDependencies {
|
||||||
|
upstream_count: number
|
||||||
|
downstream_count: number
|
||||||
|
}
|
||||||
|
|
||||||
export interface DataWarehouseSavedQuery {
|
export interface DataWarehouseSavedQuery {
|
||||||
/** UUID */
|
/** UUID */
|
||||||
id: string
|
id: string
|
||||||
@@ -4803,6 +4815,10 @@ export interface DataWarehouseSavedQuery {
|
|||||||
latest_error: string | null
|
latest_error: string | null
|
||||||
latest_history_id?: string
|
latest_history_id?: string
|
||||||
is_materialized?: boolean
|
is_materialized?: boolean
|
||||||
|
upstream_dependency_count?: number
|
||||||
|
downstream_dependency_count?: number
|
||||||
|
created_at?: string
|
||||||
|
run_history?: DataWarehouseSavedQueryRunHistory[]
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DataWarehouseSavedQueryDraft {
|
export interface DataWarehouseSavedQueryDraft {
|
||||||
|
|||||||
@@ -659,6 +659,57 @@ class DataWarehouseSavedQueryViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewS
|
|||||||
{"error": f"Failed to cancel workflow"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
{"error": f"Failed to cancel workflow"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@action(methods=["GET"], detail=True)
|
||||||
|
def dependencies(self, request: request.Request, *args, **kwargs) -> response.Response:
|
||||||
|
"""Return the count of immediate upstream and downstream dependencies for this saved query."""
|
||||||
|
saved_query = self.get_object()
|
||||||
|
saved_query_id = saved_query.id.hex
|
||||||
|
|
||||||
|
# Count immediate upstream (parents) - get unique parents from all paths to this node
|
||||||
|
upstream_paths = DataWarehouseModelPath.objects.filter(
|
||||||
|
team=saved_query.team, path__lquery=f"*.{saved_query_id}"
|
||||||
|
)
|
||||||
|
upstream_ids: set[str] = set()
|
||||||
|
for path in upstream_paths:
|
||||||
|
if len(path.path) >= 2:
|
||||||
|
# Get the immediate parent (second to last in path)
|
||||||
|
parent_id = path.path[-2]
|
||||||
|
upstream_ids.add(parent_id)
|
||||||
|
|
||||||
|
# Count immediate downstream (children) - get unique children that reference this node
|
||||||
|
downstream_paths = DataWarehouseModelPath.objects.filter(
|
||||||
|
team=saved_query.team, path__lquery=f"*.{saved_query_id}.*"
|
||||||
|
)
|
||||||
|
downstream_ids: set[str] = set()
|
||||||
|
for path in downstream_paths:
|
||||||
|
# Find position of current view in path
|
||||||
|
try:
|
||||||
|
idx = path.path.index(saved_query_id)
|
||||||
|
if idx + 1 < len(path.path):
|
||||||
|
# Get immediate child (next node after current)
|
||||||
|
child_id = path.path[idx + 1]
|
||||||
|
downstream_ids.add(child_id)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
return response.Response({"upstream_count": len(upstream_ids), "downstream_count": len(downstream_ids)})
|
||||||
|
|
||||||
|
@action(methods=["GET"], detail=True)
|
||||||
|
def run_history(self, request: request.Request, *args, **kwargs) -> response.Response:
|
||||||
|
"""Return the recent run history (up to 5 most recent) for this materialized view."""
|
||||||
|
saved_query = self.get_object()
|
||||||
|
|
||||||
|
# Get the 5 most recent runs
|
||||||
|
jobs = (
|
||||||
|
DataModelingJob.objects.filter(saved_query=saved_query)
|
||||||
|
.order_by("-last_run_at")[:5]
|
||||||
|
.values("status", "last_run_at")
|
||||||
|
)
|
||||||
|
|
||||||
|
run_history = [{"status": job["status"], "timestamp": job["last_run_at"]} for job in jobs]
|
||||||
|
|
||||||
|
return response.Response({"run_history": run_history})
|
||||||
|
|
||||||
|
|
||||||
def try_convert_to_uuid(s: str) -> uuid.UUID | str:
|
def try_convert_to_uuid(s: str) -> uuid.UUID | str:
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -7,7 +7,12 @@ from unittest.mock import patch
|
|||||||
|
|
||||||
from posthog.models import ActivityLog
|
from posthog.models import ActivityLog
|
||||||
|
|
||||||
from products.data_warehouse.backend.models import DataWarehouseModelPath, DataWarehouseSavedQuery, DataWarehouseTable
|
from products.data_warehouse.backend.models import (
|
||||||
|
DataModelingJob,
|
||||||
|
DataWarehouseModelPath,
|
||||||
|
DataWarehouseSavedQuery,
|
||||||
|
DataWarehouseTable,
|
||||||
|
)
|
||||||
from products.data_warehouse.backend.models.datawarehouse_managed_viewset import DataWarehouseManagedViewSet
|
from products.data_warehouse.backend.models.datawarehouse_managed_viewset import DataWarehouseManagedViewSet
|
||||||
from products.data_warehouse.backend.types import DataWarehouseManagedViewSetKind
|
from products.data_warehouse.backend.types import DataWarehouseManagedViewSetKind
|
||||||
|
|
||||||
@@ -1118,3 +1123,227 @@ class TestSavedQuery(APIBaseTest):
|
|||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
response.json()["detail"], "Cannot revert materialization of a query from a managed viewset."
|
response.json()["detail"], "Cannot revert materialization of a query from a managed viewset."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_dependencies_no_dependencies(self):
|
||||||
|
"""Test dependencies endpoint returns zero counts for a view with no dependencies"""
|
||||||
|
response = self.client.post(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/",
|
||||||
|
{
|
||||||
|
"name": "simple_view",
|
||||||
|
"query": {
|
||||||
|
"kind": "HogQLQuery",
|
||||||
|
"query": "select event as event from events LIMIT 100",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 201)
|
||||||
|
saved_query_id = response.json()["id"]
|
||||||
|
|
||||||
|
# Test dependencies endpoint
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query_id}/dependencies",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
data = response.json()
|
||||||
|
self.assertEqual(data["upstream_count"], 1) # events
|
||||||
|
self.assertEqual(data["downstream_count"], 0) # No downstream dependencies
|
||||||
|
|
||||||
|
def test_dependencies_with_upstream_and_downstream(self):
|
||||||
|
"""Test dependencies endpoint correctly counts immediate dependencies"""
|
||||||
|
# Create parent view
|
||||||
|
response_parent = self.client.post(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/",
|
||||||
|
{
|
||||||
|
"name": "parent_view",
|
||||||
|
"query": {
|
||||||
|
"kind": "HogQLQuery",
|
||||||
|
"query": "select event as event from events LIMIT 100",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(response_parent.status_code, 201)
|
||||||
|
parent_id = response_parent.json()["id"]
|
||||||
|
|
||||||
|
# Create child view that depends on parent
|
||||||
|
response_child = self.client.post(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/",
|
||||||
|
{
|
||||||
|
"name": "child_view",
|
||||||
|
"query": {
|
||||||
|
"kind": "HogQLQuery",
|
||||||
|
"query": f"select event as event from parent_view LIMIT 50",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(response_child.status_code, 201)
|
||||||
|
child_id = response_child.json()["id"]
|
||||||
|
|
||||||
|
# Create grandchild view that depends on child
|
||||||
|
response_grandchild = self.client.post(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/",
|
||||||
|
{
|
||||||
|
"name": "grandchild_view",
|
||||||
|
"query": {
|
||||||
|
"kind": "HogQLQuery",
|
||||||
|
"query": f"select event as event from child_view LIMIT 25",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(response_grandchild.status_code, 201)
|
||||||
|
grandchild_id = response_grandchild.json()["id"]
|
||||||
|
|
||||||
|
# Test parent dependencies (should have downstream but no upstream saved queries)
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/{parent_id}/dependencies",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
data = response.json()
|
||||||
|
self.assertEqual(data["upstream_count"], 1) # events table
|
||||||
|
self.assertEqual(data["downstream_count"], 1) # child_view
|
||||||
|
|
||||||
|
# Test child dependencies (should have both upstream and downstream)
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/{child_id}/dependencies",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
data = response.json()
|
||||||
|
self.assertEqual(data["upstream_count"], 1) # parent_view (only immediate parent)
|
||||||
|
self.assertEqual(data["downstream_count"], 1) # grandchild_view
|
||||||
|
|
||||||
|
# Test grandchild dependencies (should have upstream but no downstream)
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/{grandchild_id}/dependencies",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
data = response.json()
|
||||||
|
self.assertEqual(data["upstream_count"], 1) # child_view (only immediate parent)
|
||||||
|
self.assertEqual(data["downstream_count"], 0) # No downstream
|
||||||
|
|
||||||
|
def test_run_history_no_runs(self):
|
||||||
|
"""Test run_history endpoint returns empty array for a view with no runs"""
|
||||||
|
response = self.client.post(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/",
|
||||||
|
{
|
||||||
|
"name": "view_no_runs",
|
||||||
|
"query": {
|
||||||
|
"kind": "HogQLQuery",
|
||||||
|
"query": "select event as event from events LIMIT 100",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 201)
|
||||||
|
saved_query_id = response.json()["id"]
|
||||||
|
|
||||||
|
# Test run_history endpoint
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query_id}/run_history",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
data = response.json()
|
||||||
|
self.assertEqual(data["run_history"], [])
|
||||||
|
|
||||||
|
def test_run_history_with_runs(self):
|
||||||
|
"""Test run_history endpoint returns correct run history"""
|
||||||
|
# Create a materialized view
|
||||||
|
response = self.client.post(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/",
|
||||||
|
{
|
||||||
|
"name": "materialized_view",
|
||||||
|
"query": {
|
||||||
|
"kind": "HogQLQuery",
|
||||||
|
"query": "select event as event from events LIMIT 100",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 201)
|
||||||
|
saved_query_id = response.json()["id"]
|
||||||
|
saved_query = DataWarehouseSavedQuery.objects.get(id=saved_query_id)
|
||||||
|
|
||||||
|
# Create multiple runs with different statuses
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
now = timezone.now()
|
||||||
|
|
||||||
|
# Create 7 runs to test the limit of 5
|
||||||
|
runs = []
|
||||||
|
for i in range(7):
|
||||||
|
status = DataModelingJob.Status.COMPLETED if i % 2 == 0 else DataModelingJob.Status.FAILED
|
||||||
|
run = DataModelingJob.objects.create(
|
||||||
|
team=self.team,
|
||||||
|
saved_query=saved_query,
|
||||||
|
status=status,
|
||||||
|
last_run_at=now - timedelta(hours=i),
|
||||||
|
)
|
||||||
|
runs.append(run)
|
||||||
|
|
||||||
|
# Test run_history endpoint
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query_id}/run_history",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
# Should return only the 5 most recent runs
|
||||||
|
self.assertEqual(len(data["run_history"]), 5)
|
||||||
|
|
||||||
|
# Verify they are ordered by most recent first
|
||||||
|
for i in range(len(data["run_history"])):
|
||||||
|
expected_status = DataModelingJob.Status.COMPLETED if i % 2 == 0 else DataModelingJob.Status.FAILED
|
||||||
|
self.assertEqual(data["run_history"][i]["status"], expected_status)
|
||||||
|
self.assertIsNotNone(data["run_history"][i]["timestamp"])
|
||||||
|
|
||||||
|
# Verify the most recent run is first
|
||||||
|
most_recent_run = runs[0]
|
||||||
|
self.assertEqual(data["run_history"][0]["status"], most_recent_run.status)
|
||||||
|
|
||||||
|
def test_run_history_mixed_statuses(self):
|
||||||
|
"""Test run_history endpoint with various run statuses"""
|
||||||
|
response = self.client.post(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/",
|
||||||
|
{
|
||||||
|
"name": "mixed_status_view",
|
||||||
|
"query": {
|
||||||
|
"kind": "HogQLQuery",
|
||||||
|
"query": "select event as event from events LIMIT 100",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 201)
|
||||||
|
saved_query_id = response.json()["id"]
|
||||||
|
saved_query = DataWarehouseSavedQuery.objects.get(id=saved_query_id)
|
||||||
|
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
now = timezone.now()
|
||||||
|
|
||||||
|
# Create runs with different statuses
|
||||||
|
statuses = [
|
||||||
|
DataModelingJob.Status.COMPLETED,
|
||||||
|
DataModelingJob.Status.FAILED,
|
||||||
|
DataModelingJob.Status.RUNNING,
|
||||||
|
DataModelingJob.Status.CANCELLED,
|
||||||
|
]
|
||||||
|
|
||||||
|
for i, status in enumerate(statuses):
|
||||||
|
DataModelingJob.objects.create(
|
||||||
|
team=self.team,
|
||||||
|
saved_query=saved_query,
|
||||||
|
status=status,
|
||||||
|
last_run_at=now - timedelta(hours=i),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Test run_history endpoint
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/environments/{self.team.id}/warehouse_saved_queries/{saved_query_id}/run_history",
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
self.assertEqual(len(data["run_history"]), 4)
|
||||||
|
|
||||||
|
# Verify all different statuses are present
|
||||||
|
returned_statuses = [run["status"] for run in data["run_history"]]
|
||||||
|
self.assertIn("Completed", returned_statuses)
|
||||||
|
self.assertIn("Failed", returned_statuses)
|
||||||
|
self.assertIn("Running", returned_statuses)
|
||||||
|
self.assertIn("Cancelled", returned_statuses)
|
||||||
|
|||||||
Reference in New Issue
Block a user