Refactor: use type_kwd instead of knowledge_graph_kwd (#20475)

This commit is contained in:
Wang Qi
2026-10-01 16:58:00 +08:00
committed by GitHub
parent 9766c8c548
commit 519e7d98a5
21 changed files with 241 additions and 42 deletions
+25
View File
@@ -1997,6 +1997,31 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,
}
continue
}
if k == "knowledge_graph_kwd" {
var graphTypeClauses []interface{}
for _, field := range []string{"type_kwd", "knowledge_graph_kwd"} {
switch val := v.(type) {
case []interface{}:
if len(val) > 0 {
graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"terms": map[string]interface{}{field: val}})
}
case []string:
if len(val) > 0 {
graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"terms": map[string]interface{}{field: val}})
}
case string:
if val != "" {
graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"term": map[string]interface{}{field: val}})
}
}
}
if len(graphTypeClauses) > 0 {
filterClauses = append(filterClauses, map[string]interface{}{"bool": map[string]interface{}{
"should": graphTypeClauses, "minimum_should_match": 1,
}})
}
continue
}
if k == "id" {
if v == nil || v == "" {
continue
@@ -367,6 +367,27 @@ func TestSearchAfterPaginateLimitLargerThanBatchSize(t *testing.T) {
}
}
func TestBuildBoolQueryFromConditionUsesLegacyGraphFallback(t *testing.T) {
got := buildBoolQueryFromCondition(map[string]interface{}{"knowledge_graph_kwd": "entity"}, nil, false, false)
outer, ok := got["bool"].(map[string]interface{})
if !ok {
t.Fatalf("missing bool wrapper: %v", got)
}
filters, ok := outer["filter"].([]interface{})
if !ok || len(filters) != 1 {
t.Fatalf("graph filter = %#v, want one compatibility clause", outer["filter"])
}
encoded, err := json.Marshal(filters[0])
if err != nil {
t.Fatal(err)
}
for _, field := range []string{"type_kwd", "knowledge_graph_kwd"} {
if !strings.Contains(string(encoded), "\""+field+"\":\"entity\"") {
t.Errorf("graph filter %s does not match %s: %s", field, field, encoded)
}
}
}
// TestBuildBoolQueryFromConditionIDFilter is the regression for the
// "id filter encoded as a nested array inside bool.should" bug.
//
+34
View File
@@ -2505,6 +2505,33 @@ func floatsEqual(a, b float64) bool {
return diff < 1e-9
}
func buildGraphTypeFilterCondition(value interface{}) string {
var values []string
switch val := value.(type) {
case []string:
values = val
case []interface{}:
for _, item := range val {
values = append(values, fmt.Sprintf("%v", item))
}
case string:
values = []string{val}
default:
values = []string{fmt.Sprintf("%v", value)}
}
conditions := make([]string, 0, len(values)*2)
for _, value := range values {
if strings.TrimSpace(value) == "" {
continue
}
conditions = append(conditions,
keywordFilterCondition("type_kwd", value),
fmt.Sprintf("knowledge_graph_kwd='%s'", escapeFilterValue(value)),
)
}
return joinBalanced(conditions, " OR ")
}
// equivalentConditionToStr converts a condition map to an Infinity filter string
func equivalentConditionToStr(condition map[string]interface{}, tableColumns map[string]struct {
Type string
@@ -2540,6 +2567,13 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns map
continue
}
if k == "knowledge_graph_kwd" {
if graphTypeCondition := buildGraphTypeFilterCondition(v); graphTypeCondition != "" {
cond = append(cond, graphTypeCondition)
}
continue
}
// JSON-list fields use member containment. Legacy varchar columns keep
// the old full-text behavior.
if fieldJSONList(k) && tableColumns != nil {
+7
View File
@@ -400,6 +400,13 @@ func buildFilterFromCondition(condition map[string]interface{}, tableColumns map
continue
}
if k == "knowledge_graph_kwd" {
if graphTypeCondition := buildGraphTypeFilterCondition(v); graphTypeCondition != "" {
conditions = append(conditions, graphTypeCondition)
}
continue
}
// JSON-list fields use member containment. Legacy tables stored these
// columns as ###-joined varchar values, so retain their full-text fallback.
if fieldJSONList(k) && tableColumns != nil {
+20
View File
@@ -644,6 +644,26 @@ func TestBuildFilterFromConditionMissingJSONListColumnNeverMatches(t *testing.T)
// filter_fulltext() query, so a whitespace-bearing value matched NOTHING (nav's
// parent_kwd child lookup, cluster updates and cleanup all no-op'd); the ES
// columns are keywords and match exactly.
func TestGraphTypeFilterSupportsCurrentAndLegacyRows(t *testing.T) {
condition := map[string]interface{}{"knowledge_graph_kwd": []string{"entity", "relation"}}
for name, got := range map[string]string{
"search": equivalentConditionToStr(condition, nil),
"mutation": buildFilterFromCondition(condition, nil),
} {
for _, want := range []string{
"filter_fulltext('type_kwd', 'entity')",
"knowledge_graph_kwd='entity'",
"filter_fulltext('type_kwd', 'relation')",
"knowledge_graph_kwd='relation'",
} {
if !strings.Contains(got, want) {
t.Errorf("%s graph filter = %q, missing %q", name, got, want)
}
}
}
}
// TestKeywordFilterConditionExactForWhitespaceValues covers exact matching for multi-token keywords.
func TestKeywordFilterConditionExactForWhitespaceValues(t *testing.T) {
cases := []struct {
name, field, value, want string
+23
View File
@@ -76,6 +76,29 @@ func buildFilter(condition map[string]interface{}, kind string) (string, []inter
parts = append(parts, part)
args = append(args, partArgs...)
}
case "knowledge_graph_kwd":
if kind != "chunk" {
continue
}
values, isSlice := interfaceSlice(value)
if !isSlice {
values = []interface{}{value}
}
if len(values) == 0 {
continue
}
placeholders := make([]string, len(values))
for i := range values {
placeholders[i] = "?"
}
predicate := "= ?"
if len(values) > 1 {
predicate = "IN (" + strings.Join(placeholders, ", ") + ")"
}
parts = append(parts, fmt.Sprintf("(%s %s OR %s %s)",
quoteIdentifier("type_kwd"), predicate, quoteIdentifier("knowledge_graph_kwd"), predicate))
args = append(args, values...)
args = append(args, values...)
default:
if !valid[key] {
continue
+2 -1
View File
@@ -74,6 +74,7 @@ var chunkColumns = []columnDefinition{
{"page_num_int", "ARRAY(INTEGER) NULL"},
{"top_int", "ARRAY(INTEGER) NULL"},
{"knowledge_graph_kwd", "VARCHAR(256) NULL"},
{"type_kwd", "VARCHAR(256) NULL"},
{"source_id", "ARRAY(VARCHAR(256)) NULL"},
{"entity_kwd", "VARCHAR(256) NULL"},
{"entity_type_kwd", "VARCHAR(256) NULL"},
@@ -148,7 +149,7 @@ var skillColumns = []columnDefinition{
}
var chunkIndexColumns = []string{
"kb_id", "doc_id", "available_int", "knowledge_graph_kwd", "entity_type_kwd", "removed_kwd",
"kb_id", "doc_id", "available_int", "knowledge_graph_kwd", "type_kwd", "entity_type_kwd", "removed_kwd",
}
var memoryIndexColumns = []string{"message_id", "memory_id", "status_int"}
+15
View File
@@ -17,6 +17,7 @@
package oceanbase
import (
"reflect"
"regexp"
"strings"
"testing"
@@ -129,6 +130,20 @@ func TestMemorySourceIDUsesScalarSQL(t *testing.T) {
}
}
func TestKnowledgeGraphFilterSupportsCurrentAndLegacyRows(t *testing.T) {
where, args, err := buildFilter(map[string]interface{}{"knowledge_graph_kwd": []string{"entity", "relation"}}, "chunk")
if err != nil {
t.Fatal(err)
}
want := "(`type_kwd` IN (?, ?) OR `knowledge_graph_kwd` IN (?, ?))"
if where != want {
t.Fatalf("graph filter = %q, want %q", where, want)
}
if !reflect.DeepEqual(args, []interface{}{"entity", "relation", "entity", "relation"}) {
t.Fatalf("graph filter args = %#v", args)
}
}
func TestChunkSourceIDUsesArraySQL(t *testing.T) {
where, args, err := buildFilter(map[string]interface{}{"source_id": "source-1"}, "chunk")
if err != nil {
+14
View File
@@ -146,6 +146,20 @@ func buildFilters(condition map[string]interface{}) []string {
}
}
}
case k == "knowledge_graph_kwd":
var values []interface{}
switch list := v.(type) {
case []interface{}:
values = list
case []string:
values = make([]interface{}, len(list))
for i := range list {
values[i] = list[i]
}
default:
values = []interface{}{v}
}
filters = append(filters, "("+inList("type_kwd", values)+" OR "+inList("knowledge_graph_kwd", values)+")")
case isArrayColumn(k):
vals := toStringSlice(v)
if len(vals) == 0 {
+1 -1
View File
@@ -55,7 +55,7 @@ var (
textColumns = []string{
"docnm_kwd", "doc_type_kwd", "title_tks", "title_sm_tks", "content_with_weight",
"content_ltks", "content_sm_ltks", "important_tks", "question_tks", "create_time",
"img_id", "knowledge_graph_kwd", "entity_kwd", "entity_type_kwd", "from_entity_kwd",
"img_id", "knowledge_graph_kwd", "type_kwd", "entity_kwd", "entity_type_kwd", "from_entity_kwd",
"to_entity_kwd", "removed_kwd", "raptor_kwd", "group_id", "mom_id", "n_hop_with_weight",
}
arrayColumns = []string{"important_kwd", "question_kwd", "tag_kwd", "source_id", "entities_kwd"}
@@ -71,6 +71,14 @@ func TestBuildFiltersArrayUsesListContains(t *testing.T) {
mustContain(t, got[0], "list_contains(tag_kwd, 'b')")
}
func TestBuildFiltersKnowledgeGraphCompatibility(t *testing.T) {
got := buildFilters(map[string]interface{}{"knowledge_graph_kwd": []string{"entity", "relation"}})
want := "(type_kwd IN ('entity', 'relation') OR knowledge_graph_kwd IN ('entity', 'relation'))"
if len(got) != 1 || got[0] != want {
t.Fatalf("graph filter = %v, want %q", got, want)
}
}
func TestBuildFiltersScalarAndIn(t *testing.T) {
scalar := buildFilters(map[string]interface{}{"doc_id": "d1"})
if len(scalar) != 1 || scalar[0] != "doc_id = 'd1'" {
+5 -2
View File
@@ -769,7 +769,7 @@ func (h *DatasetsHandler) GetKnowledgeGraph(c *gin.Context) {
KbIDs: []string{datasetID},
Offset: 0,
Limit: 1,
SelectFields: []string{"content_with_weight", "knowledge_graph_kwd"},
SelectFields: []string{"content_with_weight", "type_kwd", "knowledge_graph_kwd"},
Filter: map[string]interface{}{
"kb_id": []string{datasetID},
"knowledge_graph_kwd": []string{"graph"},
@@ -785,7 +785,10 @@ func (h *DatasetsHandler) GetKnowledgeGraph(c *gin.Context) {
}
chunk := searchResult.Chunks[0]
graphType := firstStringValue(chunk["knowledge_graph_kwd"])
graphType := firstStringValue(chunk["type_kwd"])
if graphType == "" {
graphType = firstStringValue(chunk["knowledge_graph_kwd"])
}
contentWithWeight, _ := chunk["content_with_weight"].(string)
if strings.TrimSpace(contentWithWeight) == "" {
common.SuccessWithData(c, result, "success")
@@ -512,7 +512,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
switch p.Variant {
case common.VariantStructure:
// knowledge_graph_kwd: "entity" | "relation" | "graph".
// type_kwd: "entity" | "relation" | "graph".
return applyStructureGraphColumns(doc, p, kind)
case common.VariantWiki:
@@ -595,7 +595,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
case "entity", "relation", "graph":
// The tree is also projected onto the structure-graph shape (Python
// raptor_tree_to_graph + _struct_upsert_tree_graph_rows): entity /
// relation rows carry knowledge_graph_kwd and the compact graph blob
// relation rows carry type_kwd and the compact graph blob
// (kind "graph") is the /structure/graph discovery row. This is the
// same storage contract as the structure variant, so both share
// applyStructureGraphColumns.
@@ -604,9 +604,9 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
// Claim rows are searchable on their own (global KNN) but are NOT
// part of the structure graph: they carry no relation, and a
// relation-less row would be rendered as a root in the artifacts
// tree. So they deliberately skip knowledge_graph_kwd, which keeps
// tree. So they deliberately skip type_kwd, which keeps
// them out of the artifacts query (it filters
// knowledge_graph_kwd=["entity","relation"]) without a frontend
// type_kwd=["entity","relation"]) without a frontend
// change. Python mirrors this in _struct_upsert_tree_claim_rows.
if v := metaString(p.Meta, "name"); v != "" {
if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {
@@ -651,7 +651,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
// Mindmap now emits entity/relation rows (plan §1.2) so it participates in
// dataset-level merge exactly like graph/timeline: each node is an entity,
// each parent→child edge is a relation. Reuse the shared structure-graph
// column contract (knowledge_graph_kwd + from/to_entity_kwd + name_kwd +
// column contract (type_kwd + from/to_entity_kwd + name_kwd +
// entity_type_kwd + mention_count_int). The relation type lives in the
// content_with_weight payload ({"from","to","type"}), matching Python —
// NOT a dedicated relation_type_kwd column.
@@ -663,7 +663,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
// applyStructureGraphColumns emits the structure-graph row columns shared by the
// structure and tree variants (Python _struct_to_doc_storage_doc contract):
// - knowledge_graph_kwd: "entity" | "relation" | "graph"
// - type_kwd: "entity" | "relation" | "graph"
// - relations: from_entity_kwd / to_entity_kwd
// - entities: name_kwd (lowercased) / entity_type_kwd
// - mention_count_int
@@ -672,7 +672,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
// diverging (review Major).
func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind string) error {
if kind != "" {
if err := doc.SetExtraValue("knowledge_graph_kwd", kind); err != nil {
if err := doc.SetExtraValue("type_kwd", kind); err != nil {
return err
}
}
@@ -217,7 +217,7 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {
// 2 input chunks + 3 entities + 2 relations = 7 total. The structure
// variant no longer emits a separate compact graph blob (#19474): the
// per-row entity/relation products are the whole output, mirroring the
// storage-model change that dropped the knowledge_graph_kwd="graph" row.
// storage-model change that dropped the type_kwd="graph" row.
if len(chunks) != 7 {
t.Fatalf("len(chunks) = %d, want 7 (2 input + 3 entities + 2 relations)", len(chunks))
}
@@ -229,8 +229,8 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {
if !ok {
continue
}
// knowledge_graph_kwd discriminates the graph row.
if kind, _ := cm["knowledge_graph_kwd"].(string); kind == "graph" {
// type_kwd discriminates the graph row.
if kind, _ := cm["type_kwd"].(string); kind == "graph" {
graphChunks++
}
}
@@ -567,18 +567,18 @@ func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) {
// children). Mindmap now emits entity/relation rows (plan §1.2): every node
// is an entity, every parent→child edge a relation. With a flat reply there
// is at least one entity (the root) and it must carry name_kwd +
// knowledge_graph_kwd="entity" (the structure-graph storage contract).
// type_kwd="entity" (the structure-graph storage contract).
chunks := runVariant(t, "mindmap", nil)
entityCount := 0
for _, c := range chunks {
kind, _ := c["knowledge_graph_kwd"].(string)
kind, _ := c["type_kwd"].(string)
if kind == "entity" {
entityCount++
if _, ok := c["name_kwd"]; !ok {
t.Fatalf("mindmap entity chunk missing name_kwd: %+v", c)
}
if kg, _ := c["knowledge_graph_kwd"].(string); kg != "entity" {
t.Fatalf("mindmap entity chunk knowledge_graph_kwd = %q, want entity", kg)
if kg, _ := c["type_kwd"].(string); kg != "entity" {
t.Fatalf("mindmap entity chunk type_kwd = %q, want entity", kg)
}
}
}
@@ -690,7 +690,7 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {
t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want 1 (the resolved template id)", cm["id"], cm["compilation_template_ids"])
}
// Entity rows must carry source_chunk_ids.
if kg, _ := cm["knowledge_graph_kwd"].(string); kg == "entity" {
if kg, _ := cm["type_kwd"].(string); kg == "entity" {
var idsCount int
switch ids := cm["source_chunk_ids"].(type) {
case []any:
@@ -165,8 +165,8 @@ func TestGolden_Structure_ProductCount(t *testing.T) {
// compact graph blob was removed by #19474).
graphCount, nodeCount, edgeCount := 0, 0, 0
for _, p := range prods {
// knowledge_graph_kwd discriminates the structure graph rows.
kind, _ := p.GetExtraString("knowledge_graph_kwd")
// type_kwd discriminates the structure graph rows.
kind, _ := p.GetExtraString("type_kwd")
switch kind {
case "graph":
graphCount++
@@ -110,10 +110,10 @@ var compiledSelectFields = []string{
"create_timestamp_flt", "create_time",
"compilation_template_kind_kwd", "compilation_template_ids",
// The product kind discriminator for structure/tree: the component stores it
// under knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd
// under type_kwd (structure: graph/entity/relation) / raptor_kwd
// (tree: root/summary). Without these the reader cannot restore Meta["kind"]
// and the dataset-nav dispatch would skip structure/tree products (B2).
"knowledge_graph_kwd", "raptor_kwd",
"type_kwd", "knowledge_graph_kwd", "raptor_kwd",
// mention_count_int round-trips the entity mention count for reprojection.
// (relation type lives in the content_with_weight payload, matching Python —
// there is NO relation_type_kwd column.)
@@ -421,12 +421,14 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
}
}
// Restore the structure/tree product kind. The component stores it under
// knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd (tree:
// type_kwd (structure: graph/entity/relation) / raptor_kwd (tree:
// root/summary), and the dataset-nav dispatch (B2) keys off Meta["kind"] to
// pick the root/graph summary. Prefer these over the generic entity default.
// Use asString (not a bare type assertion): the engine may return a
// list-wrapped keyword column (review Major).
if v := asString(c["knowledge_graph_kwd"]); v != "" {
if v := asString(c["type_kwd"]); v != "" {
meta["kind"] = v
} else if v := asString(c["knowledge_graph_kwd"]); v != "" {
meta["kind"] = v
} else if v := asString(c["raptor_kwd"]); v != "" {
meta["kind"] = v
+16 -11
View File
@@ -160,7 +160,7 @@ func (w engineWriter) DeleteMergedWikiPages(ctx context.Context, tenant, kb stri
}
// datasetStructureSupported reports whether the running doc engine can filter
// the dataset-structure fields (knowledge_graph_kwd + scope_kwd + raw
// the dataset-structure fields (type_kwd + scope_kwd + raw
// compile_kwd) this feature writes/deletes. Only infinity and elasticsearch
// support them today; OceanBase/SeekDB/SereneDB have explicit schemas that lack
// these filter keys and would reject the query or silently drop unknown fields.
@@ -466,7 +466,7 @@ func pageEngineString(value any) string {
// description, the union of source docs/chunks, and the bucket vector. Rows are
// available_int=1 so the dataset-level structure index is searchable; the raw
// compile kind (timeline/graph/mindmap) is stamped on compile_kwd and the
// entity/relation discriminator on knowledge_graph_kwd.
// entity/relation discriminator on type_kwd.
func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb string, buckets []StructureBucket) error {
if len(buckets) == 0 {
return nil
@@ -560,12 +560,10 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
}
if b.FromEntity != "" || b.ToEntity != "" {
// relation row: carries from/to entities; kind=relation, no name_kwd.
row["knowledge_graph_kwd"] = "relation"
row["type_kwd"] = "relation"
row["from_entity_kwd"] = b.FromEntity
row["to_entity_kwd"] = b.ToEntity
} else {
row["knowledge_graph_kwd"] = "entity"
row["type_kwd"] = "entity"
row["name_kwd"] = b.Name
row["entity_type_kwd"] = b.Type
@@ -585,7 +583,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
// Python _META_ROW_KWD legacy field NOT present in the Infinity/ES/OS schemas,
// so writing it would fail with an undefined-column error — see review). It is
// a write/delete-side marker — GET discovery does NOT read it (it scans
// knowledge_graph_kwd=["entity"] rows instead, per §6).
// type_kwd=["entity"] rows instead, per §6).
seenKwd := map[string]bool{}
for _, b := range buckets {
ckwd := b.CompileKwd
@@ -603,7 +601,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
"available_int": 0,
"compile_kwd": ckwd,
"scope_kwd": "dataset",
"knowledge_graph_kwd": "kg_build_meta",
"type_kwd": "kg_build_meta",
"create_time": now.Format("2006-01-02 15:04:05"),
"create_timestamp_flt": float64(now.Unix()),
})
@@ -654,7 +652,7 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, ba
KbIDs: []string{kb},
SelectFields: []string{
"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd",
"knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
"type_kwd", "knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
"content_with_weight", "source_doc_ids", "source_chunk_ids",
},
Filter: map[string]interface{}{
@@ -743,7 +741,7 @@ func structureRowIdentity(row map[string]interface{}) string {
template := structureTemplateIdentity(templateID, structureString(row["compilation_template_kind_kwd"]))
compileKwd := structureCompileKind(structureString(row["compile_kwd"]))
payload := structureRowPayload(row)
if strings.EqualFold(structureString(row["knowledge_graph_kwd"]), "relation") {
if strings.EqualFold(structureRowType(row), "relation") {
from := structureString(row["from_entity_kwd"])
to := structureString(row["to_entity_kwd"])
if from == "" {
@@ -761,6 +759,13 @@ func structureRowIdentity(row map[string]interface{}) string {
return "entity\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(name)
}
func structureRowType(row map[string]interface{}) string {
if value := structureString(row["type_kwd"]); value != "" {
return value
}
return structureString(row["knowledge_graph_kwd"])
}
func structureRowPayload(row map[string]interface{}) map[string]interface{} {
// The payload is content_with_weight verbatim (Python json.dumps's it there);
// there is no separate payload column.
@@ -831,7 +836,7 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str
SelectFields: []string{"id", "source_doc_ids", "compile_kwd"},
// Match dataset-scope entity/relation rows across ALL structure kinds
// (timeline/graph/session_graph/mindmap), which now each stamp their
// raw compile_kwd; knowledge_graph_kwd ∈ {entity,relation} + scope_kwd
// raw compile_kwd; type_kwd ∈ {entity,relation} + scope_kwd
// =dataset is the kind-agnostic predicate (plan §1, §4.2).
Filter: map[string]interface{}{
"kb_id": kb,
@@ -1614,8 +1619,8 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
return errDatasetStructureUnsupported()
}
// structure/mindmap dataset rows are scope_kwd="dataset" +
// knowledge_graph_kwd ∈ {entity,relation,kg_build_meta} with a raw
// compile_kwd (timeline/graph/session_graph/mindmap). knowledge_graph_kwd
// type_kwd ∈ {entity,relation,kg_build_meta} with a raw
// compile_kwd (timeline/graph/session_graph/mindmap). type_kwd
// is required: wiki merged rows ALSO carry scope_kwd="dataset" (W5), so
// a scope-only sweep would wrongly delete wiki merged rows too (review
// Major). kg_build_meta (the build marker) is deleted together with the
@@ -199,7 +199,7 @@ func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {
}
var entity map[string]interface{}
for _, row := range eng.insertedChunks {
if row["knowledge_graph_kwd"] == "entity" {
if row["type_kwd"] == "entity" {
entity = row
break
}
@@ -207,6 +207,21 @@ func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {
if entity == nil {
t.Fatalf("inserted rows contain no entity: %+v", eng.insertedChunks)
}
if _, ok := entity["knowledge_graph_kwd"]; ok {
t.Fatalf("new entity row still writes knowledge_graph_kwd: %+v", entity)
}
var foundBuildMeta bool
for _, row := range eng.insertedChunks {
if row["type_kwd"] == "kg_build_meta" {
foundBuildMeta = true
if _, ok := row["knowledge_graph_kwd"]; ok {
t.Fatalf("new build marker still writes knowledge_graph_kwd: %+v", row)
}
}
}
if !foundBuildMeta {
t.Fatalf("inserted rows contain no type_kwd=kg_build_meta marker: %+v", eng.insertedChunks)
}
if entity["id"] == oldID {
t.Fatalf("old type-scoped id was reused: %v", entity["id"])
}
@@ -95,7 +95,7 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[
return nil, nil
}
idx := indexName
selectFields := []string{"content_with_weight", "compile_kwd", "compilation_template_kind_kwd", "knowledge_graph_kwd"}
selectFields := []string{"content_with_weight", "compile_kwd", "compilation_template_kind_kwd", "type_kwd", "knowledge_graph_kwd"}
if vecField != "" {
selectFields = append(selectFields, vecField)
}
@@ -111,7 +111,10 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[
}
rows := make([]StructureRow, 0, len(res.Chunks))
for _, row := range res.Chunks {
kg, _ := row["knowledge_graph_kwd"].(string)
kg, _ := row["type_kwd"].(string)
if kg == "" {
kg, _ = row["knowledge_graph_kwd"].(string)
}
sr := StructureRow{
CompileKwd: fmt.Sprint(row["compile_kwd"]),
TemplateKind: fmt.Sprint(row["compilation_template_kind_kwd"]),
+5 -2
View File
@@ -33,7 +33,7 @@ var graphEntityFields = []string{"id", "content_with_weight", "name_kwd", "menti
var graphRelationFields = []string{"id", "content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids"}
var graphAllFields = []string{
"id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids",
"from_entity_kwd", "to_entity_kwd", "knowledge_graph_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids",
"from_entity_kwd", "to_entity_kwd", "type_kwd", "knowledge_graph_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids",
"compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd",
}
@@ -481,7 +481,10 @@ func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, data
if !rowHasEnabledSource(row, excludedDocIDs) {
continue
}
kg := firstStringValue(row["knowledge_graph_kwd"])
kg := firstStringValue(row["type_kwd"])
if kg == "" {
kg = firstStringValue(row["knowledge_graph_kwd"])
}
if kg == "relation" {
if edge := projectRelation(row); edge != nil {
relations = append(relations, edge)
+1 -1
View File
@@ -698,7 +698,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque
// Build Source field list
src := []string{
"docnm_kwd", "content_ltks", "kb_id", "img_id", "title_tks", "important_kwd", "position_int",
"doc_id", "chunk_order_int", "page_num_int", "top_int", "create_timestamp_flt", "knowledge_graph_kwd",
"doc_id", "chunk_order_int", "page_num_int", "top_int", "create_timestamp_flt", "type_kwd", "knowledge_graph_kwd",
// Fields the projection below reads off each chunk. This list IS the ES
// _source filter, so omitting one returns it empty even though the
// document carries it: content_with_weight is the text the caller