feat(lineage-graph-materialization): Upstream graph view (#34670)

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Peter Hicks
2025-07-09 09:32:29 -07:00
committed by GitHub
parent ad51233fc1
commit f44bcf90e5
5 changed files with 358 additions and 97 deletions

View File

@@ -48,6 +48,7 @@
},
"dependencies": {
"@babel/runtime": "^7.24.0",
"@dagrejs/dagre": "^1.1.5",
"@dnd-kit/core": "^6.0.8",
"@dnd-kit/modifiers": "^6.0.1",
"@dnd-kit/sortable": "^7.0.2",

View File

@@ -230,6 +230,7 @@ export const multitabEditorLogic = kea<multitabEditorLogicType>([
types: string[][]
}
) => ({ view }),
setUpstreamViewMode: (mode: 'table' | 'graph') => ({ mode }),
})),
propsChanged(({ actions, props }, oldProps) => {
if (!oldProps.monaco && !oldProps.editor && props.monaco && props.editor) {
@@ -407,6 +408,12 @@ export const multitabEditorLogic = kea<multitabEditorLogicType>([
fixErrorsFailure: (_, { error }) => error,
},
],
upstreamViewMode: [
'table' as 'table' | 'graph',
{
setUpstreamViewMode: (_: 'table' | 'graph', { mode }: { mode: 'table' | 'graph' }) => mode,
},
],
})),
listeners(({ values, props, actions, asyncActions }) => ({
fixErrorsSuccess: ({ response }) => {

View File

@@ -1,10 +1,12 @@
import { IconRevert, IconTarget, IconX } from '@posthog/icons'
import { LemonDialog, LemonTable, Link, Spinner } from '@posthog/lemon-ui'
import { useActions } from 'kea'
import { useValues } from 'kea'
import { FEATURE_FLAGS } from 'lib/constants'
import { LemonButton } from 'lib/lemon-ui/LemonButton'
import { LemonSelect } from 'lib/lemon-ui/LemonSelect'
import { LemonSegmentedButton } from 'lib/lemon-ui/LemonSegmentedButton'
import { LemonTag, LemonTagType } from 'lib/lemon-ui/LemonTag'
import { Tooltip } from 'lib/lemon-ui/Tooltip'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
@@ -15,7 +17,7 @@ import { DataModelingJob, DataWarehouseSyncInterval, LineageNode, OrNever } from
import { multitabEditorLogic } from '../multitabEditorLogic'
import { infoTabLogic } from './infoTabLogic'
import { router } from 'kea-router'
import { UpstreamGraph } from './graph/UpstreamGraph'
interface QueryInfoProps {
codeEditorKey: string
@@ -62,15 +64,14 @@ const OPTIONS = [
export function QueryInfo({ codeEditorKey }: QueryInfoProps): JSX.Element {
const { sourceTableItems } = useValues(infoTabLogic({ codeEditorKey: codeEditorKey }))
const { editingView, upstream } = useValues(multitabEditorLogic)
const { runDataWarehouseSavedQuery, saveAsView } = useActions(multitabEditorLogic)
const { editingView, upstream, upstreamViewMode } = useValues(multitabEditorLogic)
const { runDataWarehouseSavedQuery, saveAsView, setUpstreamViewMode } = useActions(multitabEditorLogic)
const { featureFlags } = useValues(featureFlagLogic)
const isLineageDependencyViewEnabled = featureFlags[FEATURE_FLAGS.LINEAGE_DEPENDENCY_VIEW]
const {
dataWarehouseSavedQueryMapById,
dataWarehouseSavedQueryMapByIdStringMap,
updatingDataWarehouseSavedQuery,
initialDataWarehouseSavedQueryLoading,
dataModelingJobs,
@@ -84,7 +85,6 @@ export function QueryInfo({ codeEditorKey }: QueryInfoProps): JSX.Element {
} = useActions(dataWarehouseViewsLogic)
// note: editingView is stale, but dataWarehouseSavedQueryMapById gets updated
const currentViewIdHex = editingView?.id.replace(/-/g, '')
const savedQuery = editingView ? dataWarehouseSavedQueryMapById[editingView.id] : null
if (initialDataWarehouseSavedQueryLoading) {
@@ -387,41 +387,36 @@ export function QueryInfo({ codeEditorKey }: QueryInfoProps): JSX.Element {
{upstream && editingView && upstream.nodes.length > 0 && isLineageDependencyViewEnabled && (
<>
<div>
<h3>Tables we use</h3>
<p className="text-xs">Tables and views that this query relies on.</p>
<div className="flex items-center justify-between">
<div>
<h3 className="mb-1">Tables we use</h3>
<p className="text-xs mb-0">Tables and views that this query relies on.</p>
</div>
<LemonSegmentedButton
value={upstreamViewMode}
onChange={(mode) => setUpstreamViewMode(mode)}
options={[
{
value: 'table',
label: 'Table',
},
{
value: 'graph',
label: 'Graph',
},
]}
size="small"
/>
</div>
</div>
<LemonTable
size="small"
columns={[
{
key: 'name',
title: 'Name',
render: (_, { id, name, type }) => {
if (type === 'view' && currentViewIdHex !== id) {
const _view = dataWarehouseSavedQueryMapByIdStringMap[id]
return (
<div className="flex items-center gap-1">
{name === editingView?.name && (
<Tooltip
placement="right"
title="This is the currently viewed query"
>
<IconTarget className="text-warning" />
</Tooltip>
)}
<Link
onClick={() => {
multitabEditorLogic({
key: `hogQLQueryEditor/${router.values.location.pathname}`,
}).actions.editView(_view.query.query, _view)
}}
>
{name}
</Link>
</div>
)
}
return (
{upstreamViewMode === 'table' ? (
<LemonTable
size="small"
columns={[
{
key: 'name',
title: 'Name',
render: (_, { name }) => (
<div className="flex items-center gap-1">
{name === editingView?.name && (
<Tooltip
@@ -433,71 +428,75 @@ export function QueryInfo({ codeEditorKey }: QueryInfoProps): JSX.Element {
)}
{name}
</div>
)
),
},
},
{
key: 'type',
title: 'Type',
render: (_, { type, last_run_at }) => {
if (type === 'view') {
return last_run_at ? 'Mat. View' : 'View'
}
return 'Table'
{
key: 'type',
title: 'Type',
render: (_, { type, last_run_at }) => {
if (type === 'view') {
return last_run_at ? 'Mat. View' : 'View'
}
return 'Table'
},
},
},
{
key: 'upstream',
title: 'Direct Upstream',
render: (_, node) => {
const upstreamNodes = upstream.edges
.filter((edge) => edge.target === node.id)
.map((edge) => upstream.nodes.find((n) => n.id === edge.source))
.filter((n): n is LineageNode => n !== undefined)
{
key: 'upstream',
title: 'Direct Upstream',
render: (_, node) => {
const upstreamNodes = upstream.edges
.filter((edge) => edge.target === node.id)
.map((edge) => upstream.nodes.find((n) => n.id === edge.source))
.filter((n): n is LineageNode => n !== undefined)
if (upstreamNodes.length === 0) {
return <span className="text-secondary">None</span>
}
if (upstreamNodes.length === 0) {
return <span className="text-secondary">None</span>
}
return (
<div className="flex flex-wrap gap-1">
{upstreamNodes.map((upstreamNode) => (
<LemonTag key={upstreamNode.id} type="primary">
{upstreamNode.name}
</LemonTag>
))}
</div>
)
return (
<div className="flex flex-wrap gap-1">
{upstreamNodes.map((upstreamNode) => (
<LemonTag key={upstreamNode.id} type="primary">
{upstreamNode.name}
</LemonTag>
))}
</div>
)
},
},
},
{
key: 'last_run_at',
title: 'Last Run At',
render: (_, { last_run_at, sync_frequency }) => {
if (!last_run_at) {
return 'On demand'
}
const numericSyncFrequency = Number(sync_frequency)
const frequencyMap: Record<string, string> = {
300: '5 mins',
1800: '30 mins',
3600: '1 hour',
21600: '6 hours',
43200: '12 hours',
86400: '24 hours',
604800: '1 week',
}
{
key: 'last_run_at',
title: 'Last Run At',
render: (_, { last_run_at, sync_frequency }) => {
if (!last_run_at) {
return 'On demand'
}
const numericSyncFrequency = Number(sync_frequency)
const frequencyMap: Record<string, string> = {
300: '5 mins',
1800: '30 mins',
3600: '1 hour',
21600: '6 hours',
43200: '12 hours',
86400: '24 hours',
604800: '1 week',
}
return `${humanFriendlyDetailedTime(last_run_at)} ${
frequencyMap[numericSyncFrequency]
? `every ${frequencyMap[numericSyncFrequency]}`
: ''
}`
return `${humanFriendlyDetailedTime(last_run_at)} ${
frequencyMap[numericSyncFrequency]
? `every ${frequencyMap[numericSyncFrequency]}`
: ''
}`
},
},
},
]}
dataSource={upstream.nodes}
/>
]}
dataSource={upstream.nodes}
/>
) : (
<div className="h-96 border border-border rounded-lg overflow-hidden">
<UpstreamGraph codeEditorKey={codeEditorKey} />
</div>
)}
</>
)}
</div>

View File

@@ -0,0 +1,238 @@
import '@xyflow/react/dist/style.css'
import {
Background,
BackgroundVariant,
Controls,
Edge,
Handle,
MarkerType,
Node,
NodeTypes,
Position,
ReactFlow,
ReactFlowProvider,
MiniMap,
useReactFlow,
} from '@xyflow/react'
import dagre from '@dagrejs/dagre'
import { IconArchive, IconTarget } from '@posthog/icons'
import { LemonTag, LemonTagType } from '@posthog/lemon-ui'
import { useValues } from 'kea'
import { useEffect, useMemo } from 'react'
import { Tooltip } from 'lib/lemon-ui/Tooltip'
import { humanFriendlyDetailedTime } from 'lib/utils'
import { LineageNode as LineageNodeType } from '~/types'
import { multitabEditorLogic } from '../../multitabEditorLogic'
import { themeLogic } from '~/layout/navigation-3000/themeLogic'
interface UpstreamGraphProps {
codeEditorKey: string
}
interface LineageNodeProps {
data: LineageNodeType & { isCurrentView?: boolean }
edges: { source: string; target: string }[]
}
const MAT_VIEW_HEIGHT = 92
const TABLE_HEIGHT = 68
const NODE_WIDTH = 240
const MARKER_SIZE = 20
const NODE_SEP = 80
const RANK_SEP = 160
function LineageNode({ data, edges }: LineageNodeProps): JSX.Element {
const getNodeType = (type: string, lastRunAt?: string): string => {
if (type === 'view') {
return lastRunAt ? 'Mat. View' : 'View'
}
return 'Table'
}
const getTagType = (type: string): LemonTagType => {
if (type === 'view') {
return 'primary'
}
return 'highlight'
}
// Only show handles if there are edges to/from this node
const hasIncoming = edges.some((edge) => edge.target === data.id)
const hasOutgoing = edges.some((edge) => edge.source === data.id)
// Dynamic height: mat views get extra height for last run time
const isMatView = data.type === 'view' && !!data.last_run_at
const nodeHeight = isMatView ? MAT_VIEW_HEIGHT : TABLE_HEIGHT
return (
<div
className="bg-bg-light border border-border rounded-lg p-3 min-w-[240px] shadow-sm"
style={{ minHeight: nodeHeight }}
>
{hasIncoming && <Handle type="target" position={Position.Left} className="w-2 h-2 bg-primary" />}
<div className="flex items-center gap-2 mb-2">
{data.isCurrentView && (
<Tooltip placement="top" title="This is the currently viewed query">
<IconTarget className="text-warning text-sm" />
</Tooltip>
)}
<Tooltip title={data.name} placement="top">
<span className="font-medium text-sm truncate max-w-[180px] block">{data.name}</span>
</Tooltip>
</div>
<div className="flex items-center gap-2">
<LemonTag type={getTagType(data.type)} size="small">
{getNodeType(data.type, data.last_run_at)}
</LemonTag>
{data.status && (
<LemonTag
type={data.status === 'Failed' ? 'danger' : data.status === 'Running' ? 'warning' : 'success'}
size="small"
>
{data.status}
</LemonTag>
)}
</div>
{data.last_run_at && (
<div className="text-xs text-muted mt-2">Last run: {humanFriendlyDetailedTime(data.last_run_at)}</div>
)}
{hasOutgoing && <Handle type="source" position={Position.Right} className="w-2 h-2 bg-primary" />}
</div>
)
}
const getNodeTypes = (edges: { source: string; target: string }[]): NodeTypes => ({
lineageNode: (props) => <LineageNode {...props} edges={edges} />,
})
const getLayoutedElements = (
nodes: LineageNodeType[],
edges: { source: string; target: string }[],
currentViewName?: string
): { nodes: Node[]; edges: Edge[] } => {
const dagreGraph = new dagre.graphlib.Graph()
dagreGraph.setDefaultEdgeLabel(() => ({}))
dagreGraph.setGraph({ rankdir: 'LR', nodesep: NODE_SEP, ranksep: RANK_SEP })
// Add nodes and edges to dagre and layout with dagre
nodes.forEach((node) => {
const isMatView = node.type === 'view' && !!node.last_run_at
dagreGraph.setNode(node.id, { width: NODE_WIDTH, height: isMatView ? MAT_VIEW_HEIGHT : TABLE_HEIGHT })
})
edges.forEach((edge) => {
dagreGraph.setEdge(edge.source, edge.target)
})
dagre.layout(dagreGraph)
const layoutedNodes: Node[] = nodes.map((node) => {
const nodeWithPosition = dagreGraph.node(node.id)
const isMatView = node.type === 'view' && !!node.last_run_at
return {
id: node.id,
type: 'lineageNode',
position: {
x: nodeWithPosition.x - nodeWithPosition.width / 2,
y: nodeWithPosition.y - nodeWithPosition.height / 2,
},
data: {
...node,
isCurrentView: node.name === currentViewName,
},
width: NODE_WIDTH,
height: isMatView ? MAT_VIEW_HEIGHT : TABLE_HEIGHT,
}
})
const layoutedEdges: Edge[] = edges.map((edge, index) => ({
id: `edge-${index}`,
source: edge.source,
target: edge.target,
type: 'smoothstep',
animated: false,
markerEnd: {
type: MarkerType.ArrowClosed,
width: MARKER_SIZE,
height: MARKER_SIZE,
},
}))
return { nodes: layoutedNodes, edges: layoutedEdges }
}
function UpstreamGraphContent({ codeEditorKey }: UpstreamGraphProps): JSX.Element {
const { upstream, editingView } = useValues(multitabEditorLogic({ key: codeEditorKey }))
const { fitView } = useReactFlow()
const { isDarkModeOn } = useValues(themeLogic)
const { nodes, edges } = useMemo(() => {
if (!upstream || upstream.nodes.length === 0) {
return { nodes: [], edges: [] }
}
return getLayoutedElements(upstream.nodes, upstream.edges, editingView?.name)
}, [upstream, editingView?.name])
// Fit view when nodes change
useEffect(() => {
if (nodes.length > 0) {
setTimeout(() => fitView({ padding: 0.1 }), 100)
}
}, [nodes, fitView])
if (!upstream || upstream.nodes.length === 0) {
return (
<div
data-attr="upstream-graph-empty-state"
className="flex flex-col flex-1 rounded p-4 w-full items-center justify-center"
>
<IconArchive className="text-5xl mb-2 text-tertiary" />
<h2 className="text-xl leading-tight">No tables or views found</h2>
<p className="text-sm text-center text-balance text-tertiary">
This query doesn't depend on any other tables or views
</p>
</div>
)
}
return (
<div className="w-full h-full">
<ReactFlow
proOptions={{
hideAttribution: true,
}}
colorMode={isDarkModeOn ? 'dark' : 'light'}
nodes={nodes}
edges={edges}
nodeTypes={getNodeTypes(edges)}
fitView
fitViewOptions={{ padding: 0.1 }}
minZoom={0.1}
maxZoom={2}
>
<Background variant={BackgroundVariant.Dots} gap={20} size={1} />
<Controls showInteractive={false} position="bottom-right" />
<MiniMap zoomable pannable position="bottom-left" nodeStrokeWidth={2} />
</ReactFlow>
</div>
)
}
export function UpstreamGraph({ codeEditorKey }: UpstreamGraphProps): JSX.Element {
return (
<div className="w-full h-full">
<ReactFlowProvider>
<UpstreamGraphContent codeEditorKey={codeEditorKey} />
</ReactFlowProvider>
</div>
)
}

16
pnpm-lock.yaml generated
View File

@@ -493,6 +493,9 @@ importers:
'@babel/runtime':
specifier: ^7.24.0
version: 7.24.0
'@dagrejs/dagre':
specifier: ^1.1.5
version: 1.1.5
'@dnd-kit/core':
specifier: ^6.0.8
version: 6.0.8(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
@@ -3670,6 +3673,13 @@ packages:
'@cypress/xvfb@1.2.4':
resolution: {integrity: sha512-skbBzPggOVYCbnGgV+0dmBdW/s77ZkAOXIC1knS8NagwDjBrNC1LuXtQJeiN6l+m7lzmHtaoUw/ctJKdqkG57Q==}
'@dagrejs/dagre@1.1.5':
resolution: {integrity: sha512-Ghgrh08s12DCL5SeiR6AoyE80mQELTWhJBRmXfFoqDiFkR458vPEdgTbbjA0T+9ETNxUblnD0QW55tfdvi5pjQ==}
'@dagrejs/graphlib@2.2.4':
resolution: {integrity: sha512-mepCf/e9+SKYy1d02/UkvSy6+6MoyXhVxP8lLDfA7BPE1X1d4dR0sZznmbM8/XVJ1GPM+Svnx7Xj6ZweByWUkw==}
engines: {node: '>17.0.0'}
'@discoveryjs/json-ext@0.5.7':
resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==}
engines: {node: '>=10.0.0'}
@@ -19366,6 +19376,12 @@ snapshots:
transitivePeerDependencies:
- supports-color
'@dagrejs/dagre@1.1.5':
dependencies:
'@dagrejs/graphlib': 2.2.4
'@dagrejs/graphlib@2.2.4': {}
'@discoveryjs/json-ext@0.5.7': {}
'@dnd-kit/accessibility@3.0.1(react@18.2.0)':