mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-10-11 21:39:54 +00:00
Refactor: use type_kwd instead of knowledge_graph_kwd (#20475)
This commit is contained in:
@@ -1997,6 +1997,31 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,
|
||||
}
|
||||
continue
|
||||
}
|
||||
if k == "knowledge_graph_kwd" {
|
||||
var graphTypeClauses []interface{}
|
||||
for _, field := range []string{"type_kwd", "knowledge_graph_kwd"} {
|
||||
switch val := v.(type) {
|
||||
case []interface{}:
|
||||
if len(val) > 0 {
|
||||
graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"terms": map[string]interface{}{field: val}})
|
||||
}
|
||||
case []string:
|
||||
if len(val) > 0 {
|
||||
graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"terms": map[string]interface{}{field: val}})
|
||||
}
|
||||
case string:
|
||||
if val != "" {
|
||||
graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"term": map[string]interface{}{field: val}})
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(graphTypeClauses) > 0 {
|
||||
filterClauses = append(filterClauses, map[string]interface{}{"bool": map[string]interface{}{
|
||||
"should": graphTypeClauses, "minimum_should_match": 1,
|
||||
}})
|
||||
}
|
||||
continue
|
||||
}
|
||||
if k == "id" {
|
||||
if v == nil || v == "" {
|
||||
continue
|
||||
|
||||
@@ -367,6 +367,27 @@ func TestSearchAfterPaginateLimitLargerThanBatchSize(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBoolQueryFromConditionUsesLegacyGraphFallback(t *testing.T) {
|
||||
got := buildBoolQueryFromCondition(map[string]interface{}{"knowledge_graph_kwd": "entity"}, nil, false, false)
|
||||
outer, ok := got["bool"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("missing bool wrapper: %v", got)
|
||||
}
|
||||
filters, ok := outer["filter"].([]interface{})
|
||||
if !ok || len(filters) != 1 {
|
||||
t.Fatalf("graph filter = %#v, want one compatibility clause", outer["filter"])
|
||||
}
|
||||
encoded, err := json.Marshal(filters[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, field := range []string{"type_kwd", "knowledge_graph_kwd"} {
|
||||
if !strings.Contains(string(encoded), "\""+field+"\":\"entity\"") {
|
||||
t.Errorf("graph filter %s does not match %s: %s", field, field, encoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildBoolQueryFromConditionIDFilter is the regression for the
|
||||
// "id filter encoded as a nested array inside bool.should" bug.
|
||||
//
|
||||
|
||||
@@ -2505,6 +2505,33 @@ func floatsEqual(a, b float64) bool {
|
||||
return diff < 1e-9
|
||||
}
|
||||
|
||||
func buildGraphTypeFilterCondition(value interface{}) string {
|
||||
var values []string
|
||||
switch val := value.(type) {
|
||||
case []string:
|
||||
values = val
|
||||
case []interface{}:
|
||||
for _, item := range val {
|
||||
values = append(values, fmt.Sprintf("%v", item))
|
||||
}
|
||||
case string:
|
||||
values = []string{val}
|
||||
default:
|
||||
values = []string{fmt.Sprintf("%v", value)}
|
||||
}
|
||||
conditions := make([]string, 0, len(values)*2)
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
continue
|
||||
}
|
||||
conditions = append(conditions,
|
||||
keywordFilterCondition("type_kwd", value),
|
||||
fmt.Sprintf("knowledge_graph_kwd='%s'", escapeFilterValue(value)),
|
||||
)
|
||||
}
|
||||
return joinBalanced(conditions, " OR ")
|
||||
}
|
||||
|
||||
// equivalentConditionToStr converts a condition map to an Infinity filter string
|
||||
func equivalentConditionToStr(condition map[string]interface{}, tableColumns map[string]struct {
|
||||
Type string
|
||||
@@ -2540,6 +2567,13 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns map
|
||||
continue
|
||||
}
|
||||
|
||||
if k == "knowledge_graph_kwd" {
|
||||
if graphTypeCondition := buildGraphTypeFilterCondition(v); graphTypeCondition != "" {
|
||||
cond = append(cond, graphTypeCondition)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// JSON-list fields use member containment. Legacy varchar columns keep
|
||||
// the old full-text behavior.
|
||||
if fieldJSONList(k) && tableColumns != nil {
|
||||
|
||||
@@ -400,6 +400,13 @@ func buildFilterFromCondition(condition map[string]interface{}, tableColumns map
|
||||
continue
|
||||
}
|
||||
|
||||
if k == "knowledge_graph_kwd" {
|
||||
if graphTypeCondition := buildGraphTypeFilterCondition(v); graphTypeCondition != "" {
|
||||
conditions = append(conditions, graphTypeCondition)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// JSON-list fields use member containment. Legacy tables stored these
|
||||
// columns as ###-joined varchar values, so retain their full-text fallback.
|
||||
if fieldJSONList(k) && tableColumns != nil {
|
||||
|
||||
@@ -644,6 +644,26 @@ func TestBuildFilterFromConditionMissingJSONListColumnNeverMatches(t *testing.T)
|
||||
// filter_fulltext() query, so a whitespace-bearing value matched NOTHING (nav's
|
||||
// parent_kwd child lookup, cluster updates and cleanup all no-op'd); the ES
|
||||
// columns are keywords and match exactly.
|
||||
func TestGraphTypeFilterSupportsCurrentAndLegacyRows(t *testing.T) {
|
||||
condition := map[string]interface{}{"knowledge_graph_kwd": []string{"entity", "relation"}}
|
||||
for name, got := range map[string]string{
|
||||
"search": equivalentConditionToStr(condition, nil),
|
||||
"mutation": buildFilterFromCondition(condition, nil),
|
||||
} {
|
||||
for _, want := range []string{
|
||||
"filter_fulltext('type_kwd', 'entity')",
|
||||
"knowledge_graph_kwd='entity'",
|
||||
"filter_fulltext('type_kwd', 'relation')",
|
||||
"knowledge_graph_kwd='relation'",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("%s graph filter = %q, missing %q", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeywordFilterConditionExactForWhitespaceValues covers exact matching for multi-token keywords.
|
||||
func TestKeywordFilterConditionExactForWhitespaceValues(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, field, value, want string
|
||||
|
||||
@@ -76,6 +76,29 @@ func buildFilter(condition map[string]interface{}, kind string) (string, []inter
|
||||
parts = append(parts, part)
|
||||
args = append(args, partArgs...)
|
||||
}
|
||||
case "knowledge_graph_kwd":
|
||||
if kind != "chunk" {
|
||||
continue
|
||||
}
|
||||
values, isSlice := interfaceSlice(value)
|
||||
if !isSlice {
|
||||
values = []interface{}{value}
|
||||
}
|
||||
if len(values) == 0 {
|
||||
continue
|
||||
}
|
||||
placeholders := make([]string, len(values))
|
||||
for i := range values {
|
||||
placeholders[i] = "?"
|
||||
}
|
||||
predicate := "= ?"
|
||||
if len(values) > 1 {
|
||||
predicate = "IN (" + strings.Join(placeholders, ", ") + ")"
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("(%s %s OR %s %s)",
|
||||
quoteIdentifier("type_kwd"), predicate, quoteIdentifier("knowledge_graph_kwd"), predicate))
|
||||
args = append(args, values...)
|
||||
args = append(args, values...)
|
||||
default:
|
||||
if !valid[key] {
|
||||
continue
|
||||
|
||||
@@ -74,6 +74,7 @@ var chunkColumns = []columnDefinition{
|
||||
{"page_num_int", "ARRAY(INTEGER) NULL"},
|
||||
{"top_int", "ARRAY(INTEGER) NULL"},
|
||||
{"knowledge_graph_kwd", "VARCHAR(256) NULL"},
|
||||
{"type_kwd", "VARCHAR(256) NULL"},
|
||||
{"source_id", "ARRAY(VARCHAR(256)) NULL"},
|
||||
{"entity_kwd", "VARCHAR(256) NULL"},
|
||||
{"entity_type_kwd", "VARCHAR(256) NULL"},
|
||||
@@ -148,7 +149,7 @@ var skillColumns = []columnDefinition{
|
||||
}
|
||||
|
||||
var chunkIndexColumns = []string{
|
||||
"kb_id", "doc_id", "available_int", "knowledge_graph_kwd", "entity_type_kwd", "removed_kwd",
|
||||
"kb_id", "doc_id", "available_int", "knowledge_graph_kwd", "type_kwd", "entity_type_kwd", "removed_kwd",
|
||||
}
|
||||
|
||||
var memoryIndexColumns = []string{"message_id", "memory_id", "status_int"}
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
package oceanbase
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -129,6 +130,20 @@ func TestMemorySourceIDUsesScalarSQL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestKnowledgeGraphFilterSupportsCurrentAndLegacyRows(t *testing.T) {
|
||||
where, args, err := buildFilter(map[string]interface{}{"knowledge_graph_kwd": []string{"entity", "relation"}}, "chunk")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "(`type_kwd` IN (?, ?) OR `knowledge_graph_kwd` IN (?, ?))"
|
||||
if where != want {
|
||||
t.Fatalf("graph filter = %q, want %q", where, want)
|
||||
}
|
||||
if !reflect.DeepEqual(args, []interface{}{"entity", "relation", "entity", "relation"}) {
|
||||
t.Fatalf("graph filter args = %#v", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkSourceIDUsesArraySQL(t *testing.T) {
|
||||
where, args, err := buildFilter(map[string]interface{}{"source_id": "source-1"}, "chunk")
|
||||
if err != nil {
|
||||
|
||||
@@ -146,6 +146,20 @@ func buildFilters(condition map[string]interface{}) []string {
|
||||
}
|
||||
}
|
||||
}
|
||||
case k == "knowledge_graph_kwd":
|
||||
var values []interface{}
|
||||
switch list := v.(type) {
|
||||
case []interface{}:
|
||||
values = list
|
||||
case []string:
|
||||
values = make([]interface{}, len(list))
|
||||
for i := range list {
|
||||
values[i] = list[i]
|
||||
}
|
||||
default:
|
||||
values = []interface{}{v}
|
||||
}
|
||||
filters = append(filters, "("+inList("type_kwd", values)+" OR "+inList("knowledge_graph_kwd", values)+")")
|
||||
case isArrayColumn(k):
|
||||
vals := toStringSlice(v)
|
||||
if len(vals) == 0 {
|
||||
|
||||
@@ -55,7 +55,7 @@ var (
|
||||
textColumns = []string{
|
||||
"docnm_kwd", "doc_type_kwd", "title_tks", "title_sm_tks", "content_with_weight",
|
||||
"content_ltks", "content_sm_ltks", "important_tks", "question_tks", "create_time",
|
||||
"img_id", "knowledge_graph_kwd", "entity_kwd", "entity_type_kwd", "from_entity_kwd",
|
||||
"img_id", "knowledge_graph_kwd", "type_kwd", "entity_kwd", "entity_type_kwd", "from_entity_kwd",
|
||||
"to_entity_kwd", "removed_kwd", "raptor_kwd", "group_id", "mom_id", "n_hop_with_weight",
|
||||
}
|
||||
arrayColumns = []string{"important_kwd", "question_kwd", "tag_kwd", "source_id", "entities_kwd"}
|
||||
|
||||
@@ -71,6 +71,14 @@ func TestBuildFiltersArrayUsesListContains(t *testing.T) {
|
||||
mustContain(t, got[0], "list_contains(tag_kwd, 'b')")
|
||||
}
|
||||
|
||||
func TestBuildFiltersKnowledgeGraphCompatibility(t *testing.T) {
|
||||
got := buildFilters(map[string]interface{}{"knowledge_graph_kwd": []string{"entity", "relation"}})
|
||||
want := "(type_kwd IN ('entity', 'relation') OR knowledge_graph_kwd IN ('entity', 'relation'))"
|
||||
if len(got) != 1 || got[0] != want {
|
||||
t.Fatalf("graph filter = %v, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFiltersScalarAndIn(t *testing.T) {
|
||||
scalar := buildFilters(map[string]interface{}{"doc_id": "d1"})
|
||||
if len(scalar) != 1 || scalar[0] != "doc_id = 'd1'" {
|
||||
|
||||
@@ -769,7 +769,7 @@ func (h *DatasetsHandler) GetKnowledgeGraph(c *gin.Context) {
|
||||
KbIDs: []string{datasetID},
|
||||
Offset: 0,
|
||||
Limit: 1,
|
||||
SelectFields: []string{"content_with_weight", "knowledge_graph_kwd"},
|
||||
SelectFields: []string{"content_with_weight", "type_kwd", "knowledge_graph_kwd"},
|
||||
Filter: map[string]interface{}{
|
||||
"kb_id": []string{datasetID},
|
||||
"knowledge_graph_kwd": []string{"graph"},
|
||||
@@ -785,7 +785,10 @@ func (h *DatasetsHandler) GetKnowledgeGraph(c *gin.Context) {
|
||||
}
|
||||
|
||||
chunk := searchResult.Chunks[0]
|
||||
graphType := firstStringValue(chunk["knowledge_graph_kwd"])
|
||||
graphType := firstStringValue(chunk["type_kwd"])
|
||||
if graphType == "" {
|
||||
graphType = firstStringValue(chunk["knowledge_graph_kwd"])
|
||||
}
|
||||
contentWithWeight, _ := chunk["content_with_weight"].(string)
|
||||
if strings.TrimSpace(contentWithWeight) == "" {
|
||||
common.SuccessWithData(c, result, "success")
|
||||
|
||||
@@ -512,7 +512,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
|
||||
|
||||
switch p.Variant {
|
||||
case common.VariantStructure:
|
||||
// knowledge_graph_kwd: "entity" | "relation" | "graph".
|
||||
// type_kwd: "entity" | "relation" | "graph".
|
||||
return applyStructureGraphColumns(doc, p, kind)
|
||||
|
||||
case common.VariantWiki:
|
||||
@@ -595,7 +595,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
|
||||
case "entity", "relation", "graph":
|
||||
// The tree is also projected onto the structure-graph shape (Python
|
||||
// raptor_tree_to_graph + _struct_upsert_tree_graph_rows): entity /
|
||||
// relation rows carry knowledge_graph_kwd and the compact graph blob
|
||||
// relation rows carry type_kwd and the compact graph blob
|
||||
// (kind "graph") is the /structure/graph discovery row. This is the
|
||||
// same storage contract as the structure variant, so both share
|
||||
// applyStructureGraphColumns.
|
||||
@@ -604,9 +604,9 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
|
||||
// Claim rows are searchable on their own (global KNN) but are NOT
|
||||
// part of the structure graph: they carry no relation, and a
|
||||
// relation-less row would be rendered as a root in the artifacts
|
||||
// tree. So they deliberately skip knowledge_graph_kwd, which keeps
|
||||
// tree. So they deliberately skip type_kwd, which keeps
|
||||
// them out of the artifacts query (it filters
|
||||
// knowledge_graph_kwd=["entity","relation"]) without a frontend
|
||||
// type_kwd=["entity","relation"]) without a frontend
|
||||
// change. Python mirrors this in _struct_upsert_tree_claim_rows.
|
||||
if v := metaString(p.Meta, "name"); v != "" {
|
||||
if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {
|
||||
@@ -651,7 +651,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
|
||||
// Mindmap now emits entity/relation rows (plan §1.2) so it participates in
|
||||
// dataset-level merge exactly like graph/timeline: each node is an entity,
|
||||
// each parent→child edge is a relation. Reuse the shared structure-graph
|
||||
// column contract (knowledge_graph_kwd + from/to_entity_kwd + name_kwd +
|
||||
// column contract (type_kwd + from/to_entity_kwd + name_kwd +
|
||||
// entity_type_kwd + mention_count_int). The relation type lives in the
|
||||
// content_with_weight payload ({"from","to","type"}), matching Python —
|
||||
// NOT a dedicated relation_type_kwd column.
|
||||
@@ -663,7 +663,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
|
||||
|
||||
// applyStructureGraphColumns emits the structure-graph row columns shared by the
|
||||
// structure and tree variants (Python _struct_to_doc_storage_doc contract):
|
||||
// - knowledge_graph_kwd: "entity" | "relation" | "graph"
|
||||
// - type_kwd: "entity" | "relation" | "graph"
|
||||
// - relations: from_entity_kwd / to_entity_kwd
|
||||
// - entities: name_kwd (lowercased) / entity_type_kwd
|
||||
// - mention_count_int
|
||||
@@ -672,7 +672,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
|
||||
// diverging (review Major).
|
||||
func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind string) error {
|
||||
if kind != "" {
|
||||
if err := doc.SetExtraValue("knowledge_graph_kwd", kind); err != nil {
|
||||
if err := doc.SetExtraValue("type_kwd", kind); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -217,7 +217,7 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {
|
||||
// 2 input chunks + 3 entities + 2 relations = 7 total. The structure
|
||||
// variant no longer emits a separate compact graph blob (#19474): the
|
||||
// per-row entity/relation products are the whole output, mirroring the
|
||||
// storage-model change that dropped the knowledge_graph_kwd="graph" row.
|
||||
// storage-model change that dropped the type_kwd="graph" row.
|
||||
if len(chunks) != 7 {
|
||||
t.Fatalf("len(chunks) = %d, want 7 (2 input + 3 entities + 2 relations)", len(chunks))
|
||||
}
|
||||
@@ -229,8 +229,8 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// knowledge_graph_kwd discriminates the graph row.
|
||||
if kind, _ := cm["knowledge_graph_kwd"].(string); kind == "graph" {
|
||||
// type_kwd discriminates the graph row.
|
||||
if kind, _ := cm["type_kwd"].(string); kind == "graph" {
|
||||
graphChunks++
|
||||
}
|
||||
}
|
||||
@@ -567,18 +567,18 @@ func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) {
|
||||
// children). Mindmap now emits entity/relation rows (plan §1.2): every node
|
||||
// is an entity, every parent→child edge a relation. With a flat reply there
|
||||
// is at least one entity (the root) and it must carry name_kwd +
|
||||
// knowledge_graph_kwd="entity" (the structure-graph storage contract).
|
||||
// type_kwd="entity" (the structure-graph storage contract).
|
||||
chunks := runVariant(t, "mindmap", nil)
|
||||
entityCount := 0
|
||||
for _, c := range chunks {
|
||||
kind, _ := c["knowledge_graph_kwd"].(string)
|
||||
kind, _ := c["type_kwd"].(string)
|
||||
if kind == "entity" {
|
||||
entityCount++
|
||||
if _, ok := c["name_kwd"]; !ok {
|
||||
t.Fatalf("mindmap entity chunk missing name_kwd: %+v", c)
|
||||
}
|
||||
if kg, _ := c["knowledge_graph_kwd"].(string); kg != "entity" {
|
||||
t.Fatalf("mindmap entity chunk knowledge_graph_kwd = %q, want entity", kg)
|
||||
if kg, _ := c["type_kwd"].(string); kg != "entity" {
|
||||
t.Fatalf("mindmap entity chunk type_kwd = %q, want entity", kg)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -690,7 +690,7 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {
|
||||
t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want 1 (the resolved template id)", cm["id"], cm["compilation_template_ids"])
|
||||
}
|
||||
// Entity rows must carry source_chunk_ids.
|
||||
if kg, _ := cm["knowledge_graph_kwd"].(string); kg == "entity" {
|
||||
if kg, _ := cm["type_kwd"].(string); kg == "entity" {
|
||||
var idsCount int
|
||||
switch ids := cm["source_chunk_ids"].(type) {
|
||||
case []any:
|
||||
|
||||
@@ -165,8 +165,8 @@ func TestGolden_Structure_ProductCount(t *testing.T) {
|
||||
// compact graph blob was removed by #19474).
|
||||
graphCount, nodeCount, edgeCount := 0, 0, 0
|
||||
for _, p := range prods {
|
||||
// knowledge_graph_kwd discriminates the structure graph rows.
|
||||
kind, _ := p.GetExtraString("knowledge_graph_kwd")
|
||||
// type_kwd discriminates the structure graph rows.
|
||||
kind, _ := p.GetExtraString("type_kwd")
|
||||
switch kind {
|
||||
case "graph":
|
||||
graphCount++
|
||||
|
||||
@@ -110,10 +110,10 @@ var compiledSelectFields = []string{
|
||||
"create_timestamp_flt", "create_time",
|
||||
"compilation_template_kind_kwd", "compilation_template_ids",
|
||||
// The product kind discriminator for structure/tree: the component stores it
|
||||
// under knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd
|
||||
// under type_kwd (structure: graph/entity/relation) / raptor_kwd
|
||||
// (tree: root/summary). Without these the reader cannot restore Meta["kind"]
|
||||
// and the dataset-nav dispatch would skip structure/tree products (B2).
|
||||
"knowledge_graph_kwd", "raptor_kwd",
|
||||
"type_kwd", "knowledge_graph_kwd", "raptor_kwd",
|
||||
// mention_count_int round-trips the entity mention count for reprojection.
|
||||
// (relation type lives in the content_with_weight payload, matching Python —
|
||||
// there is NO relation_type_kwd column.)
|
||||
@@ -421,12 +421,14 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
|
||||
}
|
||||
}
|
||||
// Restore the structure/tree product kind. The component stores it under
|
||||
// knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd (tree:
|
||||
// type_kwd (structure: graph/entity/relation) / raptor_kwd (tree:
|
||||
// root/summary), and the dataset-nav dispatch (B2) keys off Meta["kind"] to
|
||||
// pick the root/graph summary. Prefer these over the generic entity default.
|
||||
// Use asString (not a bare type assertion): the engine may return a
|
||||
// list-wrapped keyword column (review Major).
|
||||
if v := asString(c["knowledge_graph_kwd"]); v != "" {
|
||||
if v := asString(c["type_kwd"]); v != "" {
|
||||
meta["kind"] = v
|
||||
} else if v := asString(c["knowledge_graph_kwd"]); v != "" {
|
||||
meta["kind"] = v
|
||||
} else if v := asString(c["raptor_kwd"]); v != "" {
|
||||
meta["kind"] = v
|
||||
|
||||
@@ -160,7 +160,7 @@ func (w engineWriter) DeleteMergedWikiPages(ctx context.Context, tenant, kb stri
|
||||
}
|
||||
|
||||
// datasetStructureSupported reports whether the running doc engine can filter
|
||||
// the dataset-structure fields (knowledge_graph_kwd + scope_kwd + raw
|
||||
// the dataset-structure fields (type_kwd + scope_kwd + raw
|
||||
// compile_kwd) this feature writes/deletes. Only infinity and elasticsearch
|
||||
// support them today; OceanBase/SeekDB/SereneDB have explicit schemas that lack
|
||||
// these filter keys and would reject the query or silently drop unknown fields.
|
||||
@@ -466,7 +466,7 @@ func pageEngineString(value any) string {
|
||||
// description, the union of source docs/chunks, and the bucket vector. Rows are
|
||||
// available_int=1 so the dataset-level structure index is searchable; the raw
|
||||
// compile kind (timeline/graph/mindmap) is stamped on compile_kwd and the
|
||||
// entity/relation discriminator on knowledge_graph_kwd.
|
||||
// entity/relation discriminator on type_kwd.
|
||||
func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb string, buckets []StructureBucket) error {
|
||||
if len(buckets) == 0 {
|
||||
return nil
|
||||
@@ -560,12 +560,10 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
|
||||
}
|
||||
if b.FromEntity != "" || b.ToEntity != "" {
|
||||
// relation row: carries from/to entities; kind=relation, no name_kwd.
|
||||
row["knowledge_graph_kwd"] = "relation"
|
||||
row["type_kwd"] = "relation"
|
||||
row["from_entity_kwd"] = b.FromEntity
|
||||
row["to_entity_kwd"] = b.ToEntity
|
||||
} else {
|
||||
row["knowledge_graph_kwd"] = "entity"
|
||||
row["type_kwd"] = "entity"
|
||||
row["name_kwd"] = b.Name
|
||||
row["entity_type_kwd"] = b.Type
|
||||
@@ -585,7 +583,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
|
||||
// Python _META_ROW_KWD legacy field NOT present in the Infinity/ES/OS schemas,
|
||||
// so writing it would fail with an undefined-column error — see review). It is
|
||||
// a write/delete-side marker — GET discovery does NOT read it (it scans
|
||||
// knowledge_graph_kwd=["entity"] rows instead, per §6).
|
||||
// type_kwd=["entity"] rows instead, per §6).
|
||||
seenKwd := map[string]bool{}
|
||||
for _, b := range buckets {
|
||||
ckwd := b.CompileKwd
|
||||
@@ -603,7 +601,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
|
||||
"available_int": 0,
|
||||
"compile_kwd": ckwd,
|
||||
"scope_kwd": "dataset",
|
||||
"knowledge_graph_kwd": "kg_build_meta",
|
||||
"type_kwd": "kg_build_meta",
|
||||
"create_time": now.Format("2006-01-02 15:04:05"),
|
||||
"create_timestamp_flt": float64(now.Unix()),
|
||||
})
|
||||
@@ -654,7 +652,7 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, ba
|
||||
KbIDs: []string{kb},
|
||||
SelectFields: []string{
|
||||
"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd",
|
||||
"knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
|
||||
"type_kwd", "knowledge_graph_kwd", "name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
|
||||
"content_with_weight", "source_doc_ids", "source_chunk_ids",
|
||||
},
|
||||
Filter: map[string]interface{}{
|
||||
@@ -743,7 +741,7 @@ func structureRowIdentity(row map[string]interface{}) string {
|
||||
template := structureTemplateIdentity(templateID, structureString(row["compilation_template_kind_kwd"]))
|
||||
compileKwd := structureCompileKind(structureString(row["compile_kwd"]))
|
||||
payload := structureRowPayload(row)
|
||||
if strings.EqualFold(structureString(row["knowledge_graph_kwd"]), "relation") {
|
||||
if strings.EqualFold(structureRowType(row), "relation") {
|
||||
from := structureString(row["from_entity_kwd"])
|
||||
to := structureString(row["to_entity_kwd"])
|
||||
if from == "" {
|
||||
@@ -761,6 +759,13 @@ func structureRowIdentity(row map[string]interface{}) string {
|
||||
return "entity\x00" + template + "\x00" + compileKwd + "\x00" + normalizedStructureEntityName(name)
|
||||
}
|
||||
|
||||
func structureRowType(row map[string]interface{}) string {
|
||||
if value := structureString(row["type_kwd"]); value != "" {
|
||||
return value
|
||||
}
|
||||
return structureString(row["knowledge_graph_kwd"])
|
||||
}
|
||||
|
||||
func structureRowPayload(row map[string]interface{}) map[string]interface{} {
|
||||
// The payload is content_with_weight verbatim (Python json.dumps's it there);
|
||||
// there is no separate payload column.
|
||||
@@ -831,7 +836,7 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str
|
||||
SelectFields: []string{"id", "source_doc_ids", "compile_kwd"},
|
||||
// Match dataset-scope entity/relation rows across ALL structure kinds
|
||||
// (timeline/graph/session_graph/mindmap), which now each stamp their
|
||||
// raw compile_kwd; knowledge_graph_kwd ∈ {entity,relation} + scope_kwd
|
||||
// raw compile_kwd; type_kwd ∈ {entity,relation} + scope_kwd
|
||||
// =dataset is the kind-agnostic predicate (plan §1, §4.2).
|
||||
Filter: map[string]interface{}{
|
||||
"kb_id": kb,
|
||||
@@ -1614,8 +1619,8 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
|
||||
return errDatasetStructureUnsupported()
|
||||
}
|
||||
// structure/mindmap dataset rows are scope_kwd="dataset" +
|
||||
// knowledge_graph_kwd ∈ {entity,relation,kg_build_meta} with a raw
|
||||
// compile_kwd (timeline/graph/session_graph/mindmap). knowledge_graph_kwd
|
||||
// type_kwd ∈ {entity,relation,kg_build_meta} with a raw
|
||||
// compile_kwd (timeline/graph/session_graph/mindmap). type_kwd
|
||||
// is required: wiki merged rows ALSO carry scope_kwd="dataset" (W5), so
|
||||
// a scope-only sweep would wrongly delete wiki merged rows too (review
|
||||
// Major). kg_build_meta (the build marker) is deleted together with the
|
||||
|
||||
@@ -199,7 +199,7 @@ func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {
|
||||
}
|
||||
var entity map[string]interface{}
|
||||
for _, row := range eng.insertedChunks {
|
||||
if row["knowledge_graph_kwd"] == "entity" {
|
||||
if row["type_kwd"] == "entity" {
|
||||
entity = row
|
||||
break
|
||||
}
|
||||
@@ -207,6 +207,21 @@ func TestWriteMergedStructureMigratesTypeScopedEntityID(t *testing.T) {
|
||||
if entity == nil {
|
||||
t.Fatalf("inserted rows contain no entity: %+v", eng.insertedChunks)
|
||||
}
|
||||
if _, ok := entity["knowledge_graph_kwd"]; ok {
|
||||
t.Fatalf("new entity row still writes knowledge_graph_kwd: %+v", entity)
|
||||
}
|
||||
var foundBuildMeta bool
|
||||
for _, row := range eng.insertedChunks {
|
||||
if row["type_kwd"] == "kg_build_meta" {
|
||||
foundBuildMeta = true
|
||||
if _, ok := row["knowledge_graph_kwd"]; ok {
|
||||
t.Fatalf("new build marker still writes knowledge_graph_kwd: %+v", row)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundBuildMeta {
|
||||
t.Fatalf("inserted rows contain no type_kwd=kg_build_meta marker: %+v", eng.insertedChunks)
|
||||
}
|
||||
if entity["id"] == oldID {
|
||||
t.Fatalf("old type-scoped id was reused: %v", entity["id"])
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[
|
||||
return nil, nil
|
||||
}
|
||||
idx := indexName
|
||||
selectFields := []string{"content_with_weight", "compile_kwd", "compilation_template_kind_kwd", "knowledge_graph_kwd"}
|
||||
selectFields := []string{"content_with_weight", "compile_kwd", "compilation_template_kind_kwd", "type_kwd", "knowledge_graph_kwd"}
|
||||
if vecField != "" {
|
||||
selectFields = append(selectFields, vecField)
|
||||
}
|
||||
@@ -111,7 +111,10 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[
|
||||
}
|
||||
rows := make([]StructureRow, 0, len(res.Chunks))
|
||||
for _, row := range res.Chunks {
|
||||
kg, _ := row["knowledge_graph_kwd"].(string)
|
||||
kg, _ := row["type_kwd"].(string)
|
||||
if kg == "" {
|
||||
kg, _ = row["knowledge_graph_kwd"].(string)
|
||||
}
|
||||
sr := StructureRow{
|
||||
CompileKwd: fmt.Sprint(row["compile_kwd"]),
|
||||
TemplateKind: fmt.Sprint(row["compilation_template_kind_kwd"]),
|
||||
|
||||
@@ -33,7 +33,7 @@ var graphEntityFields = []string{"id", "content_with_weight", "name_kwd", "menti
|
||||
var graphRelationFields = []string{"id", "content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids"}
|
||||
var graphAllFields = []string{
|
||||
"id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids",
|
||||
"from_entity_kwd", "to_entity_kwd", "knowledge_graph_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids",
|
||||
"from_entity_kwd", "to_entity_kwd", "type_kwd", "knowledge_graph_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids",
|
||||
"compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd",
|
||||
}
|
||||
|
||||
@@ -481,7 +481,10 @@ func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, data
|
||||
if !rowHasEnabledSource(row, excludedDocIDs) {
|
||||
continue
|
||||
}
|
||||
kg := firstStringValue(row["knowledge_graph_kwd"])
|
||||
kg := firstStringValue(row["type_kwd"])
|
||||
if kg == "" {
|
||||
kg = firstStringValue(row["knowledge_graph_kwd"])
|
||||
}
|
||||
if kg == "relation" {
|
||||
if edge := projectRelation(row); edge != nil {
|
||||
relations = append(relations, edge)
|
||||
|
||||
@@ -698,7 +698,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque
|
||||
// Build Source field list
|
||||
src := []string{
|
||||
"docnm_kwd", "content_ltks", "kb_id", "img_id", "title_tks", "important_kwd", "position_int",
|
||||
"doc_id", "chunk_order_int", "page_num_int", "top_int", "create_timestamp_flt", "knowledge_graph_kwd",
|
||||
"doc_id", "chunk_order_int", "page_num_int", "top_int", "create_timestamp_flt", "type_kwd", "knowledge_graph_kwd",
|
||||
// Fields the projection below reads off each chunk. This list IS the ES
|
||||
// _source filter, so omitting one returns it empty even though the
|
||||
// document carries it: content_with_weight is the text the caller
|
||||
|
||||
Reference in New Issue
Block a user