From 519e7d98a5651564d4e35d6648f006cba4baaf4f Mon Sep 17 00:00:00 2001 From: Wang Qi Date: Thu, 1 Oct 2026 16:58:00 +0800 Subject: [PATCH] Refactor: use type_kwd instead of knowledge_graph_kwd (#20475) --- internal/engine/elasticsearch/chunk.go | 25 ++++++++++++++ internal/engine/elasticsearch/chunk_test.go | 21 ++++++++++++ internal/engine/infinity/chunk.go | 34 +++++++++++++++++++ internal/engine/infinity/common.go | 7 ++++ internal/engine/infinity/sql_test.go | 20 +++++++++++ internal/engine/oceanbase/filter.go | 23 +++++++++++++ internal/engine/oceanbase/schema.go | 3 +- internal/engine/oceanbase/sql_test.go | 15 ++++++++ internal/engine/serenedb/common.go | 14 ++++++++ internal/engine/serenedb/schema.go | 2 +- internal/engine/serenedb/serenedb_test.go | 8 +++++ internal/handler/dataset.go | 7 ++-- .../component/knowledge_compiler/component.go | 14 ++++---- .../knowledge_compiler/component_test.go | 16 ++++----- .../knowledge_compiler/golden_test.go | 4 +-- .../ingestion/knowledge_compile/reader.go | 10 +++--- .../ingestion/knowledge_compile/writer.go | 27 +++++++++------ .../knowledge_compile/writer_test.go | 17 +++++++++- .../agentic-rag/runtime/tool_navigation.go | 7 ++-- internal/service/dataset_structure_graph.go | 7 ++-- internal/service/nlp/retrieval.go | 2 +- 21 files changed, 241 insertions(+), 42 deletions(-) diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go index b1683cb7cc..e228aee305 100644 --- a/internal/engine/elasticsearch/chunk.go +++ b/internal/engine/elasticsearch/chunk.go @@ -1997,6 +1997,31 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string, } continue } + if k == "knowledge_graph_kwd" { + var graphTypeClauses []interface{} + for _, field := range []string{"type_kwd", "knowledge_graph_kwd"} { + switch val := v.(type) { + case []interface{}: + if len(val) > 0 { + graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"terms": map[string]interface{}{field: val}}) + } + case []string: + if len(val) > 0 { + graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"terms": map[string]interface{}{field: val}}) + } + case string: + if val != "" { + graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"term": map[string]interface{}{field: val}}) + } + } + } + if len(graphTypeClauses) > 0 { + filterClauses = append(filterClauses, map[string]interface{}{"bool": map[string]interface{}{ + "should": graphTypeClauses, "minimum_should_match": 1, + }}) + } + continue + } if k == "id" { if v == nil || v == "" { continue diff --git a/internal/engine/elasticsearch/chunk_test.go b/internal/engine/elasticsearch/chunk_test.go index 8a3f233a64..a4c5ca2837 100644 --- a/internal/engine/elasticsearch/chunk_test.go +++ b/internal/engine/elasticsearch/chunk_test.go @@ -367,6 +367,27 @@ func TestSearchAfterPaginateLimitLargerThanBatchSize(t *testing.T) { } } +func TestBuildBoolQueryFromConditionUsesLegacyGraphFallback(t *testing.T) { + got := buildBoolQueryFromCondition(map[string]interface{}{"knowledge_graph_kwd": "entity"}, nil, false, false) + outer, ok := got["bool"].(map[string]interface{}) + if !ok { + t.Fatalf("missing bool wrapper: %v", got) + } + filters, ok := outer["filter"].([]interface{}) + if !ok || len(filters) != 1 { + t.Fatalf("graph filter = %#v, want one compatibility clause", outer["filter"]) + } + encoded, err := json.Marshal(filters[0]) + if err != nil { + t.Fatal(err) + } + for _, field := range []string{"type_kwd", "knowledge_graph_kwd"} { + if !strings.Contains(string(encoded), "\""+field+"\":\"entity\"") { + t.Errorf("graph filter %s does not match %s: %s", field, field, encoded) + } + } +} + // TestBuildBoolQueryFromConditionIDFilter is the regression for the // "id filter encoded as a nested array inside bool.should" bug. // diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go index 28b4a6f5b2..b3d5507360 100644 --- a/internal/engine/infinity/chunk.go +++ b/internal/engine/infinity/chunk.go @@ -2505,6 +2505,33 @@ func floatsEqual(a, b float64) bool { return diff < 1e-9 } +func buildGraphTypeFilterCondition(value interface{}) string { + var values []string + switch val := value.(type) { + case []string: + values = val + case []interface{}: + for _, item := range val { + values = append(values, fmt.Sprintf("%v", item)) + } + case string: + values = []string{val} + default: + values = []string{fmt.Sprintf("%v", value)} + } + conditions := make([]string, 0, len(values)*2) + for _, value := range values { + if strings.TrimSpace(value) == "" { + continue + } + conditions = append(conditions, + keywordFilterCondition("type_kwd", value), + fmt.Sprintf("knowledge_graph_kwd='%s'", escapeFilterValue(value)), + ) + } + return joinBalanced(conditions, " OR ") +} + // equivalentConditionToStr converts a condition map to an Infinity filter string func equivalentConditionToStr(condition map[string]interface{}, tableColumns map[string]struct { Type string @@ -2540,6 +2567,13 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns map continue } + if k == "knowledge_graph_kwd" { + if graphTypeCondition := buildGraphTypeFilterCondition(v); graphTypeCondition != "" { + cond = append(cond, graphTypeCondition) + } + continue + } + // JSON-list fields use member containment. Legacy varchar columns keep // the old full-text behavior. if fieldJSONList(k) && tableColumns != nil { diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.go index 9a4c551d5b..8d648214d1 100644 --- a/internal/engine/infinity/common.go +++ b/internal/engine/infinity/common.go @@ -400,6 +400,13 @@ func buildFilterFromCondition(condition map[string]interface{}, tableColumns map continue } + if k == "knowledge_graph_kwd" { + if graphTypeCondition := buildGraphTypeFilterCondition(v); graphTypeCondition != "" { + conditions = append(conditions, graphTypeCondition) + } + continue + } + // JSON-list fields use member containment. Legacy tables stored these // columns as ###-joined varchar values, so retain their full-text fallback. if fieldJSONList(k) && tableColumns != nil { diff --git a/internal/engine/infinity/sql_test.go b/internal/engine/infinity/sql_test.go index 951132e11c..f66c51987a 100644 --- a/internal/engine/infinity/sql_test.go +++ b/internal/engine/infinity/sql_test.go @@ -644,6 +644,26 @@ func TestBuildFilterFromConditionMissingJSONListColumnNeverMatches(t *testing.T) // filter_fulltext() query, so a whitespace-bearing value matched NOTHING (nav's // parent_kwd child lookup, cluster updates and cleanup all no-op'd); the ES // columns are keywords and match exactly. +func TestGraphTypeFilterSupportsCurrentAndLegacyRows(t *testing.T) { + condition := map[string]interface{}{"knowledge_graph_kwd": []string{"entity", "relation"}} + for name, got := range map[string]string{ + "search": equivalentConditionToStr(condition, nil), + "mutation": buildFilterFromCondition(condition, nil), + } { + for _, want := range []string{ + "filter_fulltext('type_kwd', 'entity')", + "knowledge_graph_kwd='entity'", + "filter_fulltext('type_kwd', 'relation')", + "knowledge_graph_kwd='relation'", + } { + if !strings.Contains(got, want) { + t.Errorf("%s graph filter = %q, missing %q", name, got, want) + } + } + } +} + +// TestKeywordFilterConditionExactForWhitespaceValues covers exact matching for multi-token keywords. func TestKeywordFilterConditionExactForWhitespaceValues(t *testing.T) { cases := []struct { name, field, value, want string diff --git a/internal/engine/oceanbase/filter.go b/internal/engine/oceanbase/filter.go index ecd129a55f..0c5f3225f3 100644 --- a/internal/engine/oceanbase/filter.go +++ b/internal/engine/oceanbase/filter.go @@ -76,6 +76,29 @@ func buildFilter(condition map[string]interface{}, kind string) (string, []inter parts = append(parts, part) args = append(args, partArgs...) } + case "knowledge_graph_kwd": + if kind != "chunk" { + continue + } + values, isSlice := interfaceSlice(value) + if !isSlice { + values = []interface{}{value} + } + if len(values) == 0 { + continue + } + placeholders := make([]string, len(values)) + for i := range values { + placeholders[i] = "?" + } + predicate := "= ?" + if len(values) > 1 { + predicate = "IN (" + strings.Join(placeholders, ", ") + ")" + } + parts = append(parts, fmt.Sprintf("(%s %s OR %s %s)", + quoteIdentifier("type_kwd"), predicate, quoteIdentifier("knowledge_graph_kwd"), predicate)) + args = append(args, values...) + args = append(args, values...) default: if !valid[key] { continue diff --git a/internal/engine/oceanbase/schema.go b/internal/engine/oceanbase/schema.go index ac929d3133..ac07091b69 100644 --- a/internal/engine/oceanbase/schema.go +++ b/internal/engine/oceanbase/schema.go @@ -74,6 +74,7 @@ var chunkColumns = []columnDefinition{ {"page_num_int", "ARRAY(INTEGER) NULL"}, {"top_int", "ARRAY(INTEGER) NULL"}, {"knowledge_graph_kwd", "VARCHAR(256) NULL"}, + {"type_kwd", "VARCHAR(256) NULL"}, {"source_id", "ARRAY(VARCHAR(256)) NULL"}, {"entity_kwd", "VARCHAR(256) NULL"}, {"entity_type_kwd", "VARCHAR(256) NULL"}, @@ -148,7 +149,7 @@ var skillColumns = []columnDefinition{ } var chunkIndexColumns = []string{ - "kb_id", "doc_id", "available_int", "knowledge_graph_kwd", "entity_type_kwd", "removed_kwd", + "kb_id", "doc_id", "available_int", "knowledge_graph_kwd", "type_kwd", "entity_type_kwd", "removed_kwd", } var memoryIndexColumns = []string{"message_id", "memory_id", "status_int"} diff --git a/internal/engine/oceanbase/sql_test.go b/internal/engine/oceanbase/sql_test.go index b1a082f882..992fb86281 100644 --- a/internal/engine/oceanbase/sql_test.go +++ b/internal/engine/oceanbase/sql_test.go @@ -17,6 +17,7 @@ package oceanbase import ( + "reflect" "regexp" "strings" "testing" @@ -129,6 +130,20 @@ func TestMemorySourceIDUsesScalarSQL(t *testing.T) { } } +func TestKnowledgeGraphFilterSupportsCurrentAndLegacyRows(t *testing.T) { + where, args, err := buildFilter(map[string]interface{}{"knowledge_graph_kwd": []string{"entity", "relation"}}, "chunk") + if err != nil { + t.Fatal(err) + } + want := "(`type_kwd` IN (?, ?) OR `knowledge_graph_kwd` IN (?, ?))" + if where != want { + t.Fatalf("graph filter = %q, want %q", where, want) + } + if !reflect.DeepEqual(args, []interface{}{"entity", "relation", "entity", "relation"}) { + t.Fatalf("graph filter args = %#v", args) + } +} + func TestChunkSourceIDUsesArraySQL(t *testing.T) { where, args, err := buildFilter(map[string]interface{}{"source_id": "source-1"}, "chunk") if err != nil { diff --git a/internal/engine/serenedb/common.go b/internal/engine/serenedb/common.go index 31d510e970..af28630c4c 100644 --- a/internal/engine/serenedb/common.go +++ b/internal/engine/serenedb/common.go @@ -146,6 +146,20 @@ func buildFilters(condition map[string]interface{}) []string { } } } + case k == "knowledge_graph_kwd": + var values []interface{} + switch list := v.(type) { + case []interface{}: + values = list + case []string: + values = make([]interface{}, len(list)) + for i := range list { + values[i] = list[i] + } + default: + values = []interface{}{v} + } + filters = append(filters, "("+inList("type_kwd", values)+" OR "+inList("knowledge_graph_kwd", values)+")") case isArrayColumn(k): vals := toStringSlice(v) if len(vals) == 0 { diff --git a/internal/engine/serenedb/schema.go b/internal/engine/serenedb/schema.go index 34bdf674ce..f7bd4b480f 100644 --- a/internal/engine/serenedb/schema.go +++ b/internal/engine/serenedb/schema.go @@ -55,7 +55,7 @@ var ( textColumns = []string{ "docnm_kwd", "doc_type_kwd", "title_tks", "title_sm_tks", "content_with_weight", "content_ltks", "content_sm_ltks", "important_tks", "question_tks", "create_time", - "img_id", "knowledge_graph_kwd", "entity_kwd", "entity_type_kwd", "from_entity_kwd", + "img_id", "knowledge_graph_kwd", "type_kwd", "entity_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "removed_kwd", "raptor_kwd", "group_id", "mom_id", "n_hop_with_weight", } arrayColumns = []string{"important_kwd", "question_kwd", "tag_kwd", "source_id", "entities_kwd"} diff --git a/internal/engine/serenedb/serenedb_test.go b/internal/engine/serenedb/serenedb_test.go index e18e0e9cba..1418d8efc2 100644 --- a/internal/engine/serenedb/serenedb_test.go +++ b/internal/engine/serenedb/serenedb_test.go @@ -71,6 +71,14 @@ func TestBuildFiltersArrayUsesListContains(t *testing.T) { mustContain(t, got[0], "list_contains(tag_kwd, 'b')") } +func TestBuildFiltersKnowledgeGraphCompatibility(t *testing.T) { + got := buildFilters(map[string]interface{}{"knowledge_graph_kwd": []string{"entity", "relation"}}) + want := "(type_kwd IN ('entity', 'relation') OR knowledge_graph_kwd IN ('entity', 'relation'))" + if len(got) != 1 || got[0] != want { + t.Fatalf("graph filter = %v, want %q", got, want) + } +} + func TestBuildFiltersScalarAndIn(t *testing.T) { scalar := buildFilters(map[string]interface{}{"doc_id": "d1"}) if len(scalar) != 1 || scalar[0] != "doc_id = 'd1'" { diff --git a/internal/handler/dataset.go b/internal/handler/dataset.go index 2e09b00f46..7591c6528e 100644 --- a/internal/handler/dataset.go +++ b/internal/handler/dataset.go @@ -769,7 +769,7 @@ func (h *DatasetsHandler) GetKnowledgeGraph(c *gin.Context) { KbIDs: []string{datasetID}, Offset: 0, Limit: 1, - SelectFields: []string{"content_with_weight", "knowledge_graph_kwd"}, + SelectFields: []string{"content_with_weight", "type_kwd", "knowledge_graph_kwd"}, Filter: map[string]interface{}{ "kb_id": []string{datasetID}, "knowledge_graph_kwd": []string{"graph"}, @@ -785,7 +785,10 @@ func (h *DatasetsHandler) GetKnowledgeGraph(c *gin.Context) { } chunk := searchResult.Chunks[0] - graphType := firstStringValue(chunk["knowledge_graph_kwd"]) + graphType := firstStringValue(chunk["type_kwd"]) + if graphType == "" { + graphType = firstStringValue(chunk["knowledge_graph_kwd"]) + } contentWithWeight, _ := chunk["content_with_weight"].(string) if strings.TrimSpace(contentWithWeight) == "" { common.SuccessWithData(c, result, "success") diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go index 7eb1113b6e..5dbb2c0aed 100644 --- a/internal/ingestion/component/knowledge_compiler/component.go +++ b/internal/ingestion/component/knowledge_compiler/component.go @@ -512,7 +512,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error { switch p.Variant { case common.VariantStructure: - // knowledge_graph_kwd: "entity" | "relation" | "graph". + // type_kwd: "entity" | "relation" | "graph". return applyStructureGraphColumns(doc, p, kind) case common.VariantWiki: @@ -595,7 +595,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error { case "entity", "relation", "graph": // The tree is also projected onto the structure-graph shape (Python // raptor_tree_to_graph + _struct_upsert_tree_graph_rows): entity / - // relation rows carry knowledge_graph_kwd and the compact graph blob + // relation rows carry type_kwd and the compact graph blob // (kind "graph") is the /structure/graph discovery row. This is the // same storage contract as the structure variant, so both share // applyStructureGraphColumns. @@ -604,9 +604,9 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error { // Claim rows are searchable on their own (global KNN) but are NOT // part of the structure graph: they carry no relation, and a // relation-less row would be rendered as a root in the artifacts - // tree. So they deliberately skip knowledge_graph_kwd, which keeps + // tree. So they deliberately skip type_kwd, which keeps // them out of the artifacts query (it filters - // knowledge_graph_kwd=["entity","relation"]) without a frontend + // type_kwd=["entity","relation"]) without a frontend // change. Python mirrors this in _struct_upsert_tree_claim_rows. if v := metaString(p.Meta, "name"); v != "" { if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil { @@ -651,7 +651,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error { // Mindmap now emits entity/relation rows (plan §1.2) so it participates in // dataset-level merge exactly like graph/timeline: each node is an entity, // each parent→child edge is a relation. Reuse the shared structure-graph - // column contract (knowledge_graph_kwd + from/to_entity_kwd + name_kwd + + // column contract (type_kwd + from/to_entity_kwd + name_kwd + // entity_type_kwd + mention_count_int). The relation type lives in the // content_with_weight payload ({"from","to","type"}), matching Python — // NOT a dedicated relation_type_kwd column. @@ -663,7 +663,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error { // applyStructureGraphColumns emits the structure-graph row columns shared by the // structure and tree variants (Python _struct_to_doc_storage_doc contract): -// - knowledge_graph_kwd: "entity" | "relation" | "graph" +// - type_kwd: "entity" | "relation" | "graph" // - relations: from_entity_kwd / to_entity_kwd // - entities: name_kwd (lowercased) / entity_type_kwd // - mention_count_int @@ -672,7 +672,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error { // diverging (review Major). func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind string) error { if kind != "" { - if err := doc.SetExtraValue("knowledge_graph_kwd", kind); err != nil { + if err := doc.SetExtraValue("type_kwd", kind); err != nil { return err } } diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go index 58a8807d5c..3b06bd49da 100644 --- a/internal/ingestion/component/knowledge_compiler/component_test.go +++ b/internal/ingestion/component/knowledge_compiler/component_test.go @@ -217,7 +217,7 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) { // 2 input chunks + 3 entities + 2 relations = 7 total. The structure // variant no longer emits a separate compact graph blob (#19474): the // per-row entity/relation products are the whole output, mirroring the - // storage-model change that dropped the knowledge_graph_kwd="graph" row. + // storage-model change that dropped the type_kwd="graph" row. if len(chunks) != 7 { t.Fatalf("len(chunks) = %d, want 7 (2 input + 3 entities + 2 relations)", len(chunks)) } @@ -229,8 +229,8 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) { if !ok { continue } - // knowledge_graph_kwd discriminates the graph row. - if kind, _ := cm["knowledge_graph_kwd"].(string); kind == "graph" { + // type_kwd discriminates the graph row. + if kind, _ := cm["type_kwd"].(string); kind == "graph" { graphChunks++ } } @@ -567,18 +567,18 @@ func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) { // children). Mindmap now emits entity/relation rows (plan §1.2): every node // is an entity, every parent→child edge a relation. With a flat reply there // is at least one entity (the root) and it must carry name_kwd + - // knowledge_graph_kwd="entity" (the structure-graph storage contract). + // type_kwd="entity" (the structure-graph storage contract). chunks := runVariant(t, "mindmap", nil) entityCount := 0 for _, c := range chunks { - kind, _ := c["knowledge_graph_kwd"].(string) + kind, _ := c["type_kwd"].(string) if kind == "entity" { entityCount++ if _, ok := c["name_kwd"]; !ok { t.Fatalf("mindmap entity chunk missing name_kwd: %+v", c) } - if kg, _ := c["knowledge_graph_kwd"].(string); kg != "entity" { - t.Fatalf("mindmap entity chunk knowledge_graph_kwd = %q, want entity", kg) + if kg, _ := c["type_kwd"].(string); kg != "entity" { + t.Fatalf("mindmap entity chunk type_kwd = %q, want entity", kg) } } } @@ -690,7 +690,7 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) { t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want 1 (the resolved template id)", cm["id"], cm["compilation_template_ids"]) } // Entity rows must carry source_chunk_ids. - if kg, _ := cm["knowledge_graph_kwd"].(string); kg == "entity" { + if kg, _ := cm["type_kwd"].(string); kg == "entity" { var idsCount int switch ids := cm["source_chunk_ids"].(type) { case []any: diff --git a/internal/ingestion/component/knowledge_compiler/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden_test.go index cbde245b01..53b885c999 100644 --- a/internal/ingestion/component/knowledge_compiler/golden_test.go +++ b/internal/ingestion/component/knowledge_compiler/golden_test.go @@ -165,8 +165,8 @@ func TestGolden_Structure_ProductCount(t *testing.T) { // compact graph blob was removed by #19474). graphCount, nodeCount, edgeCount := 0, 0, 0 for _, p := range prods { - // knowledge_graph_kwd discriminates the structure graph rows. - kind, _ := p.GetExtraString("knowledge_graph_kwd") + // type_kwd discriminates the structure graph rows. + kind, _ := p.GetExtraString("type_kwd") switch kind { case "graph": graphCount++ diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go index 97c424ca6d..374dff8155 100644 --- a/internal/ingestion/knowledge_compile/reader.go +++ b/internal/ingestion/knowledge_compile/reader.go @@ -110,10 +110,10 @@ var compiledSelectFields = []string{ "create_timestamp_flt", "create_time", "compilation_template_kind_kwd", "compilation_template_ids", // The product kind discriminator for structure/tree: the component stores it - // under knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd + // under type_kwd (structure: graph/entity/relation) / raptor_kwd // (tree: root/summary). Without these the reader cannot restore Meta["kind"] // and the dataset-nav dispatch would skip structure/tree products (B2). - "knowledge_graph_kwd", "raptor_kwd", + "type_kwd", "knowledge_graph_kwd", "raptor_kwd", // mention_count_int round-trips the entity mention count for reprojection. // (relation type lives in the content_with_weight payload, matching Python — // there is NO relation_type_kwd column.) @@ -421,12 +421,14 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo } } // Restore the structure/tree product kind. The component stores it under - // knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd (tree: + // type_kwd (structure: graph/entity/relation) / raptor_kwd (tree: // root/summary), and the dataset-nav dispatch (B2) keys off Meta["kind"] to // pick the root/graph summary. Prefer these over the generic entity default. // Use asString (not a bare type assertion): the engine may return a // list-wrapped keyword column (review Major). - if v := asString(c["knowledge_graph_kwd"]); v != "" { + if v := asString(c["type_kwd"]); v != "" { + meta["kind"] = v + } else if v := asString(c["knowledge_graph_kwd"]); v != "" { meta["kind"] = v } else if v := asString(c["raptor_kwd"]); v != "" { meta["kind"] = v diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go index 3efbae47c2..85ec2c764d 100644 --- a/internal/ingestion/knowledge_compile/writer.go +++ b/internal/ingestion/knowledge_compile/writer.go @@ -160,7 +160,7 @@ func (w engineWriter) DeleteMergedWikiPages(ctx context.Context, tenant, kb stri } // datasetStructureSupported reports whether the running doc engine can filter -// the dataset-structure fields (knowledge_graph_kwd + scope_kwd + raw +// the dataset-structure fields (type_kwd + scope_kwd + raw // compile_kwd) this feature writes/deletes. Only infinity and elasticsearch // support them today; OceanBase/SeekDB/SereneDB have explicit schemas that lack // these filter keys and would reject the query or silently drop unknown fields. @@ -466,7 +466,7 @@ func pageEngineString(value any) string { // description, the union of source docs/chunks, and the bucket vector. Rows are // available_int=1 so the dataset-level structure index is searchable; the raw // compile kind (timeline/graph/mindmap) is stamped on compile_kwd and the -// entity/relation discriminator on knowledge_graph_kwd. +// entity/relation discriminator on type_kwd. func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb string, buckets []StructureBucket) error { if len(buckets) == 0 { return nil @@ -560,12 +560,10 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin } if b.FromEntity != "" || b.ToEntity != "" { // relation row: carries from/to entities; kind=relation, no name_kwd. - row["knowledge_graph_kwd"] = "relation" row["type_kwd"] = "relation" row["from_entity_kwd"] = b.FromEntity row["to_entity_kwd"] = b.ToEntity } else { - row["knowledge_graph_kwd"] = "entity" row["type_kwd"] = "entity" row["name_kwd"] = b.Name row["entity_type_kwd"] = b.Type @@ -585,7 +583,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin // Python _META_ROW_KWD legacy field NOT present in the Infinity/ES/OS schemas, // so writing it would fail with an undefined-column error — see review). It is // a write/delete-side marker — GET discovery does NOT read it (it scans - // knowledge_graph_kwd=["entity"] rows instead, per §6). + // type_kwd=["entity"] rows instead, per §6). seenKwd := map[string]bool{} for _, b := range buckets { ckwd := b.CompileKwd @@ -603,7 +601,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin "available_int": 0, "compile_kwd": ckwd, "scope_kwd": "dataset", - "knowledge_graph_kwd": "kg_build_meta", + "type_kwd": "kg_build_meta", "create_time": now.Format("2006-01-02 15:04:05"), "create_timestamp_flt": float64(now.Unix()), }) @@ -654,7 +652,7 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, ba KbIDs: []string{kb}, SelectFields: []string{ "id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd", - "knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", + "type_kwd", "knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "content_with_weight", "source_doc_ids", "source_chunk_ids", }, Filter: map[string]interface{}{ @@ -743,7 +741,7 @@ func structureRowIdentity(row map[string]interface{}) string { template := structureTemplateIdentity(templateID, structureString(row["compilation_template_kind_kwd"])) compileKwd := structureCompileKind(structureString(row["compile_kwd"])) payload := structureRowPayload(row) - if strings.EqualFold(structureString(row["knowledge_graph_kwd"]), "relation") { + if strings.EqualFold(structureRowType(row), "relation") { from := structureString(row["from_entity_kwd"]) to := structureString(row["to_entity_kwd"]) if from == "" { @@ -761,6 +759,13 @@ func structureRowIdentity(row map[string]interface{}) string { return "entity\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(name) } +func structureRowType(row map[string]interface{}) string { + if value := structureString(row["type_kwd"]); value != "" { + return value + } + return structureString(row["knowledge_graph_kwd"]) +} + func structureRowPayload(row map[string]interface{}) map[string]interface{} { // The payload is content_with_weight verbatim (Python json.dumps's it there); // there is no separate payload column. @@ -831,7 +836,7 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str SelectFields: []string{"id", "source_doc_ids", "compile_kwd"}, // Match dataset-scope entity/relation rows across ALL structure kinds // (timeline/graph/session_graph/mindmap), which now each stamp their - // raw compile_kwd; knowledge_graph_kwd ∈ {entity,relation} + scope_kwd + // raw compile_kwd; type_kwd ∈ {entity,relation} + scope_kwd // =dataset is the kind-agnostic predicate (plan §1, §4.2). Filter: map[string]interface{}{ "kb_id": kb, @@ -1614,8 +1619,8 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str return errDatasetStructureUnsupported() } // structure/mindmap dataset rows are scope_kwd="dataset" + - // knowledge_graph_kwd ∈ {entity,relation,kg_build_meta} with a raw - // compile_kwd (timeline/graph/session_graph/mindmap). knowledge_graph_kwd + // type_kwd ∈ {entity,relation,kg_build_meta} with a raw + // compile_kwd (timeline/graph/session_graph/mindmap). type_kwd // is required: wiki merged rows ALSO carry scope_kwd="dataset" (W5), so // a scope-only sweep would wrongly delete wiki merged rows too (review // Major). kg_build_meta (the build marker) is deleted together with the diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go index 69e2fb657e..caab047ff4 100644 --- a/internal/ingestion/knowledge_compile/writer_test.go +++ b/internal/ingestion/knowledge_compile/writer_test.go @@ -199,7 +199,7 @@ func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) { } var entity map[string]interface{} for _, row := range eng.insertedChunks { - if row["knowledge_graph_kwd"] == "entity" { + if row["type_kwd"] == "entity" { entity = row break } @@ -207,6 +207,21 @@ func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) { if entity == nil { t.Fatalf("inserted rows contain no entity: %+v", eng.insertedChunks) } + if _, ok := entity["knowledge_graph_kwd"]; ok { + t.Fatalf("new entity row still writes knowledge_graph_kwd: %+v", entity) + } + var foundBuildMeta bool + for _, row := range eng.insertedChunks { + if row["type_kwd"] == "kg_build_meta" { + foundBuildMeta = true + if _, ok := row["knowledge_graph_kwd"]; ok { + t.Fatalf("new build marker still writes knowledge_graph_kwd: %+v", row) + } + } + } + if !foundBuildMeta { + t.Fatalf("inserted rows contain no type_kwd=kg_build_meta marker: %+v", eng.insertedChunks) + } if entity["id"] == oldID { t.Fatalf("old type-scoped id was reused: %v", entity["id"]) } diff --git a/internal/rag/agentic-rag/runtime/tool_navigation.go b/internal/rag/agentic-rag/runtime/tool_navigation.go index 33a1afe66a..72dfc98424 100644 --- a/internal/rag/agentic-rag/runtime/tool_navigation.go +++ b/internal/rag/agentic-rag/runtime/tool_navigation.go @@ -95,7 +95,7 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[ return nil, nil } idx := indexName - selectFields := []string{"content_with_weight", "compile_kwd", "compilation_template_kind_kwd", "knowledge_graph_kwd"} + selectFields := []string{"content_with_weight", "compile_kwd", "compilation_template_kind_kwd", "type_kwd", "knowledge_graph_kwd"} if vecField != "" { selectFields = append(selectFields, vecField) } @@ -111,7 +111,10 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[ } rows := make([]StructureRow, 0, len(res.Chunks)) for _, row := range res.Chunks { - kg, _ := row["knowledge_graph_kwd"].(string) + kg, _ := row["type_kwd"].(string) + if kg == "" { + kg, _ = row["knowledge_graph_kwd"].(string) + } sr := StructureRow{ CompileKwd: fmt.Sprint(row["compile_kwd"]), TemplateKind: fmt.Sprint(row["compilation_template_kind_kwd"]), diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.go index dd686cfb96..8c0a657590 100644 --- a/internal/service/dataset_structure_graph.go +++ b/internal/service/dataset_structure_graph.go @@ -33,7 +33,7 @@ var graphEntityFields = []string{"id", "content_with_weight", "name_kwd", "menti var graphRelationFields = []string{"id", "content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids"} var graphAllFields = []string{ "id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", - "from_entity_kwd", "to_entity_kwd", "knowledge_graph_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids", + "from_entity_kwd", "to_entity_kwd", "type_kwd", "knowledge_graph_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd", } @@ -481,7 +481,10 @@ func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, data if !rowHasEnabledSource(row, excludedDocIDs) { continue } - kg := firstStringValue(row["knowledge_graph_kwd"]) + kg := firstStringValue(row["type_kwd"]) + if kg == "" { + kg = firstStringValue(row["knowledge_graph_kwd"]) + } if kg == "relation" { if edge := projectRelation(row); edge != nil { relations = append(relations, edge) diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go index 290500c136..748315b386 100644 --- a/internal/service/nlp/retrieval.go +++ b/internal/service/nlp/retrieval.go @@ -698,7 +698,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque // Build Source field list src := []string{ "docnm_kwd", "content_ltks", "kb_id", "img_id", "title_tks", "important_kwd", "position_int", - "doc_id", "chunk_order_int", "page_num_int", "top_int", "create_timestamp_flt", "knowledge_graph_kwd", + "doc_id", "chunk_order_int", "page_num_int", "top_int", "create_timestamp_flt", "type_kwd", "knowledge_graph_kwd", // Fields the projection below reads off each chunk. This list IS the ES // _source filter, so omitting one returns it empty even though the // document carries it: content_with_weight is the text the caller