ragflow/internal/engine/elasticsearch/chunk.go

//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package elasticsearch

import (
	"bytes"
	"cmp"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"reflect"
	"regexp"
	"slices"
	"sort"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"

	"ragflow/internal/common"
	"ragflow/internal/engine/types"
	"ragflow/internal/tokenizer"

	"github.com/bytedance/sonic"
	"github.com/elastic/go-elasticsearch/v8/esapi"

	"go.uber.org/zap"
)

var jsonIterator = sonic.Config{
	SortMapKeys: false,
}.Froze()

var memoryMessageVectorFieldRE = regexp.MustCompile(`^q_\d+_vec$`)

// CreateChunkStore creates an index
func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {
	if baseName == "" {
		return fmt.Errorf("index name cannot be empty")
	}

	// Check if index already exists
	exists, err := e.indexExists(ctx, baseName)
	if err != nil {
		return fmt.Errorf("failed to check index existence: %w", err)
	}
	if exists {
		if strings.HasPrefix(baseName, "memory_") {
			if err := e.ensureMemoryMessageVectorMapping(ctx, baseName, vectorSize); err != nil {
				return fmt.Errorf("failed to ensure memory vector mapping: %w", err)
			}
			common.Info("Memory index already exists, ensured vector mapping", zap.String("index_name", baseName), zap.Int("vector_size", vectorSize))
			return nil
		}
		common.Info("Index already exists, skipping creation", zap.String("index_name", baseName))
		return nil
	}

	// Load mapping based on index type
	var mapping map[string]interface{}
	if strings.HasPrefix(baseName, "memory_") {
		mapping = getMemoryMessageMapping(vectorSize)
	} else if datasetID == "skill" {
		// Load skill-specific mapping
		skillMapping, err := loadSkillMapping()
		if err != nil {
			return fmt.Errorf("failed to load skill mapping: %w", err)
		}
		mapping = skillMapping
	} else {
		// Default mapping for dataset
		mapping = map[string]interface{}{
			"settings": map[string]interface{}{
				"number_of_shards":   1,
				"number_of_replicas": 0,
			},
			"mappings": map[string]interface{}{
				"properties": map[string]interface{}{
					// tag_feas must be an object of numeric values (the
					// "rank_features" ES type) so tag-based ranking works and
					// inserts succeed. Without this, dynamic mapping could
					// infer an incompatible type when a JSON string is sent.
					"tag_feas": map[string]interface{}{
						"type": "rank_features",
					},
				},
			},
		}
	}

	// Prepare request body
	var body io.Reader
	if mapping != nil {
		data, err := json.Marshal(mapping)
		if err != nil {
			return fmt.Errorf("failed to marshal mapping: %w", err)
		}
		body = bytes.NewReader(data)
	}

	// Create index
	req := esapi.IndicesCreateRequest{
		Index: baseName,
		Body:  body,
	}

	res, err := req.Do(ctx, e.client)
	if err != nil {
		return fmt.Errorf("failed to create index: %w", err)
	}
	defer res.Body.Close()

	if res.IsError() {
		bodyBytes, _ := io.ReadAll(res.Body)
		reason := extractErrorReason(bodyBytes)
		if reason != "" {
			return fmt.Errorf("elasticsearch error: %s", reason)
		}
		return fmt.Errorf("elasticsearch returned error: %s, body: %s", res.Status(), string(bodyBytes))
	}

	// Parse response
	var result map[string]interface{}
	if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
		return fmt.Errorf("failed to parse response: %w", err)
	}

	acknowledged, ok := result["acknowledged"].(bool)
	if !ok || !acknowledged {
		return fmt.Errorf("index creation not acknowledged")
	}

	common.Info("Successfully created Elasticsearch index", zap.String("index_name", baseName))
	return nil
}

// InsertChunks inserts chunks into a chunk index
// If a chunk with the same id + doc_id + kb_id already exists, it will be updated with the new value
//
// It waits for an index refresh before returning, so a caller that reads the
// chunks straight back (the chunk APIs, the debug endpoints) sees them right
// away. Ingestion must use InsertChunksNoRefresh instead - see that method for
// why the wait is the wrong trade there.
func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {
	return e.insertChunks(ctx, chunks, baseName, datasetID, "wait_for")
}

// InsertChunksNoRefresh inserts chunks without waiting for an index refresh:
// they become visible on the index's normal refresh cycle (the KB index ships
// with refresh_interval=1000ms), so the difference is at most a second of
// visibility - against a measured ~0.5s median (1.0s p90, 5.0s max) of pure
// latency per write when waiting.
//
// This is the ingestion path's contract, and it matches Python's: the Python
// ingestion inserts chunks with refresh=False (rag/svr/task_executor_refactor/
// chunk_service.py:386 and :423), while its API default stays "wait_for".
func (e *Engine) InsertChunksNoRefresh(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) {
	return e.insertChunks(ctx, chunks, baseName, datasetID, "")
}

// insertChunks is the shared bulk-insert body. An empty refresh omits the
// parameter, which leaves the decision to the index (no forced refresh).
func (e *Engine) insertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, refresh string) ([]string, error) {
	common.Info("ElasticsearchConnection.InsertChunks called", zap.String("index_name", baseName), zap.Int("chunkCount", len(chunks)))

	if len(chunks) == 0 {
		return []string{}, nil
	}

	if baseName == "" {
		return nil, fmt.Errorf("index name cannot be empty")
	}

	isMemoryIndex := strings.HasPrefix(baseName, "memory_")
	if isMemoryIndex {
		if err := e.ensureMemoryMessageVectorMappingsForDocs(ctx, baseName, chunks); err != nil {
			return nil, err
		}
	}

	// Bulk requests are executed in bounded batches so a single large document
	// (thousands of chunks) never sends a payload above the Elasticsearch HTTP
	// body limit (default http.max_content_length = 100MB), which surfaces as a
	// 413 Request Entity Too Large. We flush a batch by chunk count AND byte
	// size to stay well within the limit.
	const (
		bulkBatchChunks = 500
		bulkBatchBytes  = 20 * 1024 * 1024 // 20MB, comfortably under the 100MB default
	)

	flush := func(buf *bytes.Buffer) error {
		if buf.Len() == 0 {
			return nil
		}
		req := esapi.BulkRequest{
			Body:    bytes.NewReader(buf.Bytes()),
			Refresh: refresh,
		}
		res, err := req.Do(ctx, e.client)
		if err != nil {
			common.Error("Failed to execute bulk request", err)
			return fmt.Errorf("failed to execute bulk request: %w", err)
		}
		defer res.Body.Close()

		if res.IsError() {
			bodyBytes, _ := io.ReadAll(res.Body)
			common.Sugar.Errorw("Elasticsearch bulk request returned error", "status", res.Status(), "body", string(bodyBytes))
			return fmt.Errorf("elasticsearch bulk request returned error: %s, body: %s", res.Status(), string(bodyBytes))
		}

		// Parse bulk response
		var bulkResponse map[string]interface{}
		if err := json.NewDecoder(res.Body).Decode(&bulkResponse); err != nil {
			common.Error("Failed to parse bulk response", err)
			return fmt.Errorf("failed to parse bulk response: %w", err)
		}

		// Check for errors in bulk response. ES reports a 200 at the
		// top level even when individual bulk items fail, so we must
		// inspect the items and surface the reasons instead of returning
		// success while the chunks were silently dropped.
		if hasErrs, ok := bulkResponse["errors"].(bool); ok && hasErrs {
			var reasons []string
			if items, ok := bulkResponse["items"].([]interface{}); ok {
				for _, it := range items {
					im, ok := it.(map[string]interface{})
					if !ok {
						continue
					}
					for _, op := range im {
						om, ok := op.(map[string]interface{})
						if !ok {
							continue
						}
						if errObj, ok := om["error"].(map[string]interface{}); ok {
							if reason, ok := errObj["reason"].(string); ok && reason != "" {
								reasons = append(reasons, reason)
							}
						}
					}
				}
			}
			if len(reasons) > 5 {
				reasons = reasons[:5]
			}
			common.Error("Elasticsearch bulk request had item errors",
				fmt.Errorf("index %s: %d failed items; first reasons: %v", baseName, len(reasons), reasons))
			return fmt.Errorf("elasticsearch bulk request had %d item errors (index %s); first reasons: %v",
				len(reasons), baseName, reasons)
		}
		return nil
	}

	// Build and execute bulk request bodies in bounded batches.
	var buf bytes.Buffer
	batchChunkCount := 0
	for _, doc := range chunks {
		if isMemoryIndex {
			// Memory messages arrive with logical fields; map them to
			// the storage document (tokenization included) before write.
			doc = mapMemoryMessageToESDoc(doc)
		}
		docID, _ := doc["doc_id"].(string)
		chunkID, _ := doc["id"].(string)
		if docID == "" || chunkID == "" {
			common.Warn("Skipping chunk without doc_id or id")
			continue
		}

		// Action line: use json.Marshal to properly escape string values
		action, err := json.Marshal(map[string]interface{}{
			"index": map[string]interface{}{
				"_index": baseName,
				"_id":    chunkID,
			},
		})
		if err != nil {
			common.Error("Failed to marshal bulk action", err)
			return nil, fmt.Errorf("failed to marshal bulk action: %w", err)
		}
		buf.Write(action)
		buf.WriteByte('\n')

		// Document line: work with a copy to avoid mutating the original
		docCopy := copyFields(doc)
		docCopy["kb_id"] = datasetID
		if raw, ok := formatOrderedTagFeas(docCopy["tag_feas"]); ok {
			docCopy["tag_feas"] = raw
		}
		if err := jsonIterator.NewEncoder(&buf).Encode(docCopy); err != nil {
			return nil, fmt.Errorf("failed to encode document: %w", err)
		}

		batchChunkCount++
		if batchChunkCount >= bulkBatchChunks || buf.Len() >= bulkBatchBytes {
			// Flush the current batch before adding the next chunk.
			if err := flush(&buf); err != nil {
				return nil, err
			}
			buf.Reset()
			batchChunkCount = 0
		}
	}
	// Flush the final partial batch.
	if err := flush(&buf); err != nil {
		return nil, err
	}

	common.Info("ElasticsearchConnection.InsertChunks result", zap.String("index_name", baseName), zap.Int("count", len(chunks)))
	return []string{}, nil
}

// UpdateChunks updates chunks by condition
func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error {
	fullIndexName := baseName
	common.Info("ElasticsearchConnection.UpdateChunks called", zap.String("index_name", fullIndexName), zap.Any("condition", condition), zap.Any("new_value", newValue))

	if fullIndexName == "" {
		return fmt.Errorf("index name cannot be empty")
	}

	// Check if index exists
	exists, err := e.indexExists(ctx, fullIndexName)
	if err != nil {
		common.Error("Failed to check index existence", err)
		return fmt.Errorf("failed to check index existence: %w", err)
	}
	if !exists {
		return fmt.Errorf("%w: '%s'", types.ErrIndexNotFound, fullIndexName)
	}

	if strings.HasPrefix(fullIndexName, "memory_") {
		condition["memory_id"] = datasetID
		if messageDocID, ok := condition["id"].(string); ok {
			return e.updateSingleMemoryMessage(ctx, fullIndexName, messageDocID, newValue)
		}
		return e.updateChunksByQuery(ctx, fullIndexName, mapMemoryMessageESConditionFields(condition), mapMemoryMessageESUpdateFields(newValue))
	}

	// Add kb_id to condition
	condition["kb_id"] = datasetID

	// Case 1: Single document update (when condition["id"] is a string)
	if chunkID, ok := condition["id"].(string); ok {
		return e.updateSingleChunk(ctx, fullIndexName, chunkID, newValue)
	}

	// Case 2: Multi-document update via UpdateByQuery
	return e.updateChunksByQuery(ctx, fullIndexName, condition, newValue)
}

// AdjustChunkPagerank atomically adjusts pagerank_fea and clamps it to
// [minWeight, maxWeight].
func (e *Engine) AdjustChunkPagerank(ctx context.Context, indexName, chunkID, kbID string, delta, minWeight, maxWeight float64) error {
	if indexName == "" {
		return fmt.Errorf("index name cannot be empty")
	}
	if chunkID == "" {
		return fmt.Errorf("chunk id cannot be empty")
	}
	script := `
		if (ctx._source.kb_id == null || !ctx._source.kb_id.equals(params.kb_id)) {
			ctx.op = 'noop';
		} else {
			double current = 0.0;
			if (ctx._source.containsKey(params.field) && ctx._source[params.field] != null) {
				Object currentValue = ctx._source[params.field];
				if (currentValue instanceof Number) {
					current = ((Number)currentValue).doubleValue();
				} else {
					try {
						current = Double.parseDouble(currentValue.toString());
					} catch (Exception e) {
						current = 0.0;
					}
				}
			}
			double next = current + params.delta;
			if (next < params.min_weight) {
				next = params.min_weight;
			}
			if (next > params.max_weight) {
				next = params.max_weight;
			}
			if (next <= 0.0) {
				ctx._source.remove(params.field);
			} else {
				ctx._source[params.field] = next;
			}
		}
	`
	body, err := json.Marshal(map[string]interface{}{
		"script": map[string]interface{}{
			"source": script,
			"lang":   "painless",
			"params": map[string]interface{}{
				"field":      common.PAGERANK_FLD,
				"kb_id":      kbID,
				"delta":      delta,
				"min_weight": minWeight,
				"max_weight": maxWeight,
			},
		},
	})
	if err != nil {
		return fmt.Errorf("failed to marshal pagerank adjust request: %w", err)
	}
	retryOnConflict := 3
	req := esapi.UpdateRequest{
		Index:           indexName,
		DocumentID:      chunkID,
		Body:            bytes.NewReader(body),
		RetryOnConflict: &retryOnConflict,
	}
	res, err := req.Do(ctx, e.client)
	if err != nil {
		return fmt.Errorf("failed to adjust chunk pagerank: %w", err)
	}
	defer res.Body.Close()
	if res.IsError() {
		if res.StatusCode == http.StatusNotFound {
			return fmt.Errorf("%w: %s", types.ErrDocumentNotFound, chunkID)
		}
		bodyBytes, _ := io.ReadAll(res.Body)
		return fmt.Errorf("elasticsearch pagerank adjust error: %s, body: %s", res.Status(), string(bodyBytes))
	}
	var updateResp struct {
		Result string `json:"result"`
	}
	if err := json.NewDecoder(res.Body).Decode(&updateResp); err != nil {
		return fmt.Errorf("failed to decode pagerank adjust response: %w", err)
	}
	if updateResp.Result == "noop" {
		return fmt.Errorf("chunk %s does not belong to dataset %s", chunkID, kbID)
	}
	return nil
}

func (e *Engine) updateSingleMemoryMessage(ctx context.Context, indexName, messageDocID string, newValue map[string]interface{}) error {
	doc := mapMemoryMessageESUpdateFields(newValue)
	delete(doc, "id")
	if len(doc) == 0 {
		return nil
	}

	updateBody := map[string]interface{}{"doc": doc}
	body, err := json.Marshal(updateBody)
	if err != nil {
		return fmt.Errorf("failed to marshal memory message update request: %w", err)
	}
	req := esapi.UpdateRequest{
		Index:      indexName,
		DocumentID: messageDocID,
		Body:       bytes.NewReader(body),
		Refresh:    "wait_for",
	}
	res, err := req.Do(ctx, e.client)
	if err != nil {
		return fmt.Errorf("failed to update memory message: %w", err)
	}
	defer res.Body.Close()
	if res.IsError() {
		if res.StatusCode == http.StatusNotFound {
			return fmt.Errorf("%w: %s", types.ErrDocumentNotFound, messageDocID)
		}
		bodyBytes, _ := io.ReadAll(res.Body)
		return fmt.Errorf("elasticsearch memory message update error: %s, body: %s", res.Status(), string(bodyBytes))
	}
	return nil
}

func mapMemoryMessageESUpdateFields(newValue map[string]interface{}) map[string]interface{} {
	doc := make(map[string]interface{}, len(newValue))
	for k, v := range newValue {
		switch k {
		case "remove", "add":
			doc[k] = v
		default:
			doc[mapMemoryMessageESField(k, false)] = v
		}
	}
	// Mirror Python memory/utils/es_conn.py update: writing content
	// refreshes tokenized_content_ltks before write.
	if content, ok := newValue["content"].(string); ok {
		doc["tokenized_content_ltks"] = tokenizeMemoryMessageContent(content)
	}
	return doc
}

func mapMemoryMessageESConditionFields(condition map[string]interface{}) map[string]interface{} {
	mapped := make(map[string]interface{}, len(condition))
	for k, v := range condition {
		mapped[mapMemoryMessageESField(k, false)] = v
	}
	return mapped
}

// updateSingleChunk handles single document update
func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID string, newValue map[string]interface{}) error {
	common.Debug("ElasticsearchConnection.updateSingleChunk called", zap.String("indexName", indexName), zap.String("chunkID", chunkID))

	// First find the document by id field to get the actual _id
	searchReq := map[string]interface{}{
		"query": map[string]interface{}{
			"term": map[string]interface{}{"id": chunkID},
		},
	}

	body, err := json.Marshal(searchReq)
	if err != nil {
		return fmt.Errorf("failed to marshal search request: %w", err)
	}

	res, err := e.client.Search(
		e.client.Search.WithContext(ctx),
		e.client.Search.WithIndex(indexName),
		e.client.Search.WithBody(bytes.NewReader(body)),
	)
	if err != nil {
		return fmt.Errorf("failed to search for chunk: %w", err)
	}
	defer res.Body.Close()

	if res.IsError() {
		return fmt.Errorf("failed to search for chunk: %s", res.Status())
	}

	var searchResult map[string]interface{}
	if err := json.NewDecoder(res.Body).Decode(&searchResult); err != nil {
		return fmt.Errorf("failed to parse search response: %w", err)
	}

	hits, ok := searchResult["hits"].(map[string]interface{})
	if !ok {
		return fmt.Errorf("%w: %s", types.ErrDocumentNotFound, chunkID)
	}

	hitList, ok := hits["hits"].([]interface{})
	if !ok || len(hitList) == 0 {
		return fmt.Errorf("%w: %s", types.ErrDocumentNotFound, chunkID)
	}

	firstHit, ok := hitList[0].(map[string]interface{})
	if !ok {
		return fmt.Errorf("%w: %s", types.ErrDocumentNotFound, chunkID)
	}

	actualID, ok := firstHit["_id"].(string)
	if !ok {
		return fmt.Errorf("%w: %s", types.ErrDocumentNotFound, chunkID)
	}

	doc := copyFields(newValue)
	delete(doc, "id")

	removeValue, _ := doc["remove"]
	delete(doc, "remove")
	removeField, _ := removeValue.(string)
	removeDict, _ := removeValue.(map[string]interface{})

	// Remove *_feas fields
	var feasFields []string
	for k := range doc {
		if strings.HasSuffix(k, "feas") {
			feasFields = append(feasFields, k)
		}
	}
	for _, k := range feasFields {
		scriptBody := map[string]interface{}{
			"script": map[string]interface{}{
				"source": fmt.Sprintf("ctx._source.remove(\"%s\");", k),
			},
		}
		body, _ = json.Marshal(scriptBody)
		req := esapi.UpdateRequest{
			Index:      indexName,
			DocumentID: actualID,
			Body:       bytes.NewReader(body),
		}
		res, err = req.Do(ctx, e.client)
		if err != nil {
			common.Warn("Failed to remove feas field", zap.String("field", k), zap.Error(err))
		}
		closeESBody(res)
	}

	// Remove specific field if removeField is set
	if removeField != "" {
		scriptBody := map[string]interface{}{
			"script": map[string]interface{}{
				"source": fmt.Sprintf("ctx._source.remove('%s');", removeField),
			},
		}
		body, _ = json.Marshal(scriptBody)
		req := esapi.UpdateRequest{
			Index:      indexName,
			DocumentID: actualID,
			Body:       bytes.NewReader(body),
		}
		res, err = req.Do(ctx, e.client)
		if err != nil {
			common.Warn("Failed to remove field", zap.String("field", removeField), zap.Error(err))
		}
		closeESBody(res)
	}

	// Remove specific values from array fields (removeDict)
	if removeDict != nil {
		scripts := []string{}
		params := make(map[string]interface{})
		for kk, vv := range removeDict {
			scripts = append(scripts,
				fmt.Sprintf("if (ctx._source.containsKey('%s') && ctx._source.%s != null) { int i = ctx._source.%s.indexOf(params.p_%s); if (i >= 0) { ctx._source.%s.remove(i); }}",
					kk, kk, kk, kk, kk))
			params[fmt.Sprintf("p_%s", kk)] = vv
		}
		if scripts != nil {
			scriptBody := map[string]interface{}{
				"script": map[string]interface{}{
					"source": strings.Join(scripts, ""),
					"params": params,
				},
			}
			body, _ = json.Marshal(scriptBody)
			req := esapi.UpdateRequest{
				Index:      indexName,
				DocumentID: actualID,
				Body:       bytes.NewReader(body),
			}
			res, err = req.Do(ctx, e.client)
			if err != nil {
				common.Warn("Failed to remove dict fields", zap.Error(err))
			}
			closeESBody(res)
		}
	}

	// Update document fields if any remain
	if len(doc) > 0 {
		if raw, ok := formatOrderedTagFeas(doc["tag_feas"]); ok {
			doc["tag_feas"] = raw
		}
		updateBody := map[string]interface{}{"doc": doc}
		body, _ := json.Marshal(updateBody)
		req := esapi.UpdateRequest{
			Index:      indexName,
			DocumentID: actualID,
			Body:       bytes.NewReader(body),
			Refresh:    "wait_for",
		}
		res, err := req.Do(ctx, e.client)
		if err != nil {
			return fmt.Errorf("failed to update document: %w", err)
		}
		defer res.Body.Close()
		if res.IsError() {
			if res.StatusCode == http.StatusNotFound {
				return fmt.Errorf("%w: %s", types.ErrDocumentNotFound, chunkID)
			}
			return fmt.Errorf("elasticsearch update error: %s", res.Status())
		}
	}

	common.Debug("ElasticsearchConnection.updateSingleChunk completed", zap.String("indexName", indexName), zap.String("chunkID", chunkID))
	return nil
}

// conditionTermsList normalizes a condition or new-value field into the value
// list a `terms` query / stored array field needs. Callers hand over either
// []interface{} (decoded JSON payloads) or a typed slice such as []string
// (internal callers, e.g. the document availability switch). A typed slice the
// builder does not recognize silently drops the field, which turns a bounded
// update/delete by query into an unfiltered one over the whole index.
func conditionTermsList(value interface{}) ([]interface{}, bool) {
	items, ok := value.([]interface{})
	if ok {
		return items, true
	}
	rv := reflect.ValueOf(value)
	if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array {
		return nil, false
	}
	if rv.Type().Elem().Kind() == reflect.Uint8 {
		// []byte is a scalar blob, not a list of ids/keywords.
		return nil, false
	}
	out := make([]interface{}, 0, rv.Len())
	for i := 0; i < rv.Len(); i++ {
		out = append(out, rv.Index(i).Interface())
	}
	return out, true
}

// mustNotExistsClauses renders the `must_not` exclusions a condition carries.
// The only form RAGFlow produces is must_not={"exists": <field>} (e.g.
// hybrid_search excluding compiled products); any other shape yields nothing.
func mustNotExistsClauses(value interface{}) []map[string]interface{} {
	exclusions, ok := value.(map[string]interface{})
	if !ok {
		return nil
	}
	clauses := make([]map[string]interface{}, 0, len(exclusions))
	for field, value := range exclusions {
		if field != "exists" {
			continue
		}
		clauses = append(clauses, map[string]interface{}{
			"exists": map[string]interface{}{"field": value},
		})
	}
	return clauses
}

// updateChunksByQuery handles multi-document update
func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, condition map[string]interface{}, newValue map[string]interface{}) error {
	common.Debug("ElasticsearchConnection.updateChunksByQuery called", zap.String("indexName", indexName))

	// Build bool query from condition
	var mustClauses []map[string]interface{}
	var mustNotClauses []map[string]interface{}
	for k, v := range condition {
		if k == "exists" {
			mustClauses = append(mustClauses, map[string]interface{}{
				"exists": map[string]interface{}{"field": v},
			})
			continue
		}
		if k == "must_not" {
			mustNotClauses = append(mustNotClauses, mustNotExistsClauses(v)...)
			continue
		}
		if v == nil || v == "" {
			continue
		}
		if items, ok := conditionTermsList(v); ok {
			mustClauses = append(mustClauses, map[string]interface{}{
				"terms": map[string]interface{}{k: items},
			})
			continue
		}
		if _, ok := v.(string); ok {
			mustClauses = append(mustClauses, map[string]interface{}{
				"term": map[string]interface{}{k: v},
			})
			continue
		}
		if _, ok := v.(int); ok {
			mustClauses = append(mustClauses, map[string]interface{}{
				"term": map[string]interface{}{k: v},
			})
		}
	}
	if len(mustClauses) == 0 && len(mustNotClauses) == 0 && len(condition) > 0 {
		// Every condition key was dropped (empty value or unsupported type).
		// update-by-query without a filter rewrites the whole index, so refuse
		// instead — mirrors DeleteChunks.
		return fmt.Errorf("ES update aborted: non-empty condition yielded unfiltered update on index %s", indexName)
	}

	boolClauses := map[string]interface{}{}
	if len(mustClauses) > 0 {
		boolClauses["filter"] = mustClauses
	}
	if len(mustNotClauses) > 0 {
		boolClauses["must_not"] = mustNotClauses
	}
	boolQuery := map[string]interface{}{"bool": boolClauses}

	// Build painless scripts from newValue
	var scripts []string
	params := make(map[string]interface{})

	for k, v := range newValue {
		if k == "remove" {
			if removeStr, ok := v.(string); ok {
				scripts = append(scripts, fmt.Sprintf("ctx._source.remove('%s');", removeStr))
				continue
			}
			if removeDict, ok := v.(map[string]interface{}); ok {
				for kk, vv := range removeDict {
					scripts = append(scripts,
						fmt.Sprintf("if (ctx._source.containsKey('%s') && ctx._source.%s != null) { int i = ctx._source.%s.indexOf(params.p_%s); if (i >= 0) { ctx._source.%s.remove(i); }}",
							kk, kk, kk, kk, kk))
					params[fmt.Sprintf("p_%s", kk)] = vv
				}
			}
			continue
		}
		if k == "add" {
			if addDict, ok := v.(map[string]interface{}); ok {
				for kk, vv := range addDict {
					vvStr, ok := vv.(string)
					if ok {
						vvStr = strings.TrimSpace(vvStr)
						scripts = append(scripts, fmt.Sprintf("ctx._source.%s.add(params.pp_%s);", kk, kk))
						params[fmt.Sprintf("pp_%s", kk)] = vvStr
					}
				}
			}
			continue
		}
		if (k == "" || v == nil) && k != "available_int" {
			continue
		}

		if items, ok := conditionTermsList(v); ok {
			params[fmt.Sprintf("pp_%s", k)] = items
			scripts = append(scripts, fmt.Sprintf("ctx._source.%s=params.pp_%s;", k, k))
			continue
		}
		switch val := v.(type) {
		case string:
			// Sanitize: replace ' \n \r with space
			sanitized := sanitizeString(val)
			params[fmt.Sprintf("pp_%s", k)] = sanitized
			scripts = append(scripts, fmt.Sprintf("ctx._source.%s=params.pp_%s;", k, k))
		case int, int8, int16, int32, int64, float32, float64:
			scripts = append(scripts, fmt.Sprintf("ctx._source.%s=%v;", k, val))
		}
	}

	scriptSource := strings.Join(scripts, "")
	if scriptSource == "" {
		return fmt.Errorf("no supported update fields for update by query")
	}

	// Build update by query body
	updateBody := map[string]interface{}{
		"query": boolQuery,
		"script": map[string]interface{}{
			"source": scriptSource,
			"params": params,
		},
	}

	bodyBytes, err := json.Marshal(updateBody)
	if err != nil {
		return fmt.Errorf("failed to marshal update body: %w", err)
	}

	// Execute update by query with refresh=true, slices=5, conflicts="proceed"
	refreshTrue := true
	req := esapi.UpdateByQueryRequest{
		Index:     []string{indexName},
		Body:      bytes.NewReader(bodyBytes),
		Refresh:   &refreshTrue,
		Slices:    5,
		Conflicts: "proceed",
	}

	res, err := req.Do(ctx, e.client)
	if err != nil {
		common.Error("Failed to execute update by query", err)
		return fmt.Errorf("failed to execute update by query: %w", err)
	}
	defer res.Body.Close()

	if res.IsError() {
		bodyBytes, _ := io.ReadAll(res.Body)
		return fmt.Errorf("elasticsearch update by query error: %s, body: %s", res.Status(), string(bodyBytes))
	}

	common.Debug("ElasticsearchConnection.updateChunksByQuery completed", zap.String("indexName", indexName))
	return nil
}

// sanitizeString replaces ' \n \r with space
func sanitizeString(s string) string {
	s = strings.ReplaceAll(s, "'", " ")
	s = strings.ReplaceAll(s, "\n", " ")
	s = strings.ReplaceAll(s, "\r", " ")
	return strings.TrimSpace(s)
}

// closeESBody releases an ES response body, tolerating the nil response the
// client returns alongside a transport error. Callers that check `err` first
// and close only on the success branch leak the body whenever the client ever
// hands back a non-nil response with an error, so every Do/Search call site
// funnels its close through here.
func closeESBody(res *esapi.Response) {
	if res != nil && res.Body != nil {
		_ = res.Body.Close()
	}
}

// copyFields creates a shallow copy of a map
func copyFields(m map[string]interface{}) map[string]interface{} {
	result := make(map[string]interface{})
	for k, v := range m {
		result[k] = v
	}
	return result
}

// formatOrderedTagFeas formats tag_feas as json.RawMessage with keys ordered
// strictly descending by score, preventing jsoniter map-key randomized output.
func formatOrderedTagFeas(v any) (json.RawMessage, bool) {
	if v == nil {
		return nil, false
	}
	if raw, ok := v.(json.RawMessage); ok {
		return raw, true
	}
	if marshaler, ok := v.(json.Marshaler); ok {
		b, err := marshaler.MarshalJSON()
		if err == nil {
			return json.RawMessage(b), true
		}
	}
	type kv struct {
		k string
		v int
	}
	var kvs []kv
	roundScore := func(f float64) int {
		if f < 0 {
			return int(f - 0.5)
		}
		return int(f + 0.5)
	}

	switch m := v.(type) {
	case map[string]int:
		kvs = make([]kv, 0, len(m))
		for k, val := range m {
			kvs = append(kvs, kv{k, val})
		}
	case map[string]float64:
		kvs = make([]kv, 0, len(m))
		for k, val := range m {
			kvs = append(kvs, kv{k, roundScore(val)})
		}
	case map[string]any:
		kvs = make([]kv, 0, len(m))
		for k, val := range m {
			switch score := val.(type) {
			case int:
				kvs = append(kvs, kv{k, score})
			case float64:
				kvs = append(kvs, kv{k, roundScore(score)})
			case int64:
				kvs = append(kvs, kv{k, int(score)})
			case json.Number:
				if f, err := score.Float64(); err == nil {
					kvs = append(kvs, kv{k, roundScore(f)})
				}
			}
		}
	default:
		return nil, false
	}

	if len(kvs) == 0 {
		return json.RawMessage("{}"), true
	}

	sort.Slice(kvs, func(i, j int) bool {
		if kvs[i].v != kvs[j].v {
			return kvs[i].v > kvs[j].v
		}
		return kvs[i].k < kvs[j].k
	})

	var buf bytes.Buffer
	buf.WriteByte('{')
	for i, item := range kvs {
		if i > 0 {
			buf.WriteByte(',')
		}
		kb, _ := json.Marshal(item.k)
		buf.Write(kb)
		buf.WriteByte(':')
		buf.WriteString(strconv.Itoa(item.v))
	}
	buf.WriteByte('}')
	return json.RawMessage(buf.Bytes()), true
}

// DeleteChunks deletes chunks from a dataset index by condition
func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
	// For ES, index name is just indexName (e.g., "ragflow_{tenantID}"), not indexName_datasetID
	fullIndexName := indexName
	common.Info("Deleting chunks from Elasticsearch index", zap.String("index_name", fullIndexName), zap.Any("condition", condition))

	// Check if index exists
	exists, err := e.indexExists(ctx, fullIndexName)
	if err != nil {
		return 0, fmt.Errorf("failed to check index existence: %w", err)
	}
	if !exists {
		common.Warn(fmt.Sprintf("Index %s does not exist, skipping delete", fullIndexName))
		return 0, nil
	}

	// Build bool query from condition
	var mustClauses []map[string]interface{}
	var filterClauses []map[string]interface{}
	var mustNotClauses []map[string]interface{}

	// Handle chunk IDs - use terms query on "id" field instead of ids query on _id
	if idVal, ok := condition["id"]; ok && idVal != nil {
		if id, isString := idVal.(string); isString {
			mustClauses = append(mustClauses, map[string]interface{}{
				"term": map[string]interface{}{"id": id},
			})
		} else if items, isList := conditionTermsList(idVal); isList {
			ids := make([]interface{}, 0, len(items))
			for _, item := range items {
				if id, isString := item.(string); isString {
					ids = append(ids, id)
				}
			}
			if len(ids) > 0 {
				mustClauses = append(mustClauses, map[string]interface{}{
					"terms": map[string]interface{}{"id": ids},
				})
			}
		}
	}

	// Handle kb_id - add as term filter
	if kbID, ok := condition["kb_id"].(string); ok && kbID != "" {
		filterClauses = append(filterClauses, map[string]interface{}{
			"term": map[string]interface{}{"kb_id": kbID},
		})
	}

	// Add all other conditions as filters/must/must_not
	for k, v := range condition {
		if k == "id" || k == "kb_id" {
			continue // Already handled above
		}
		if k == "exists" {
			filterClauses = append(filterClauses, map[string]interface{}{
				"exists": map[string]interface{}{"field": v},
			})
		} else if k == "must_not" {
			mustNotClauses = append(mustNotClauses, mustNotExistsClauses(v)...)
		} else if v != nil {
			if items, ok := conditionTermsList(v); ok {
				mustClauses = append(mustClauses, map[string]interface{}{
					"terms": map[string]interface{}{k: items},
				})
			} else if _, ok := v.(string); ok {
				mustClauses = append(mustClauses, map[string]interface{}{
					"term": map[string]interface{}{k: v},
				})
			} else if _, ok := v.(int); ok {
				mustClauses = append(mustClauses, map[string]interface{}{
					"term": map[string]interface{}{k: v},
				})
			}
		}
	}

	// Build the query
	var qry map[string]interface{}
	if len(filterClauses) == 0 && len(mustClauses) == 0 && len(mustNotClauses) == 0 {
		if len(condition) > 0 {
			return 0, fmt.Errorf("ES delete aborted: non-empty condition yielded match_all query on index %s", fullIndexName)
		}
		qry = map[string]interface{}{"match_all": map[string]interface{}{}}
	} else {
		boolMap := map[string]interface{}{}
		if len(filterClauses) > 0 {
			boolMap["filter"] = filterClauses
		}
		if len(mustClauses) > 0 {
			boolMap["must"] = mustClauses
		}
		if len(mustNotClauses) > 0 {
			boolMap["must_not"] = mustNotClauses
		}
		qry = map[string]interface{}{"bool": boolMap}
	}

	// Build delete by query body
	deleteBody := map[string]interface{}{
		"query": qry,
	}

	bodyBytes, err := json.Marshal(deleteBody)
	if err != nil {
		return 0, fmt.Errorf("failed to marshal delete body: %w", err)
	}

	// Execute delete by query with refresh=true
	refreshTrue := true
	req := esapi.DeleteByQueryRequest{
		Index:   []string{fullIndexName},
		Body:    bytes.NewReader(bodyBytes),
		Refresh: &refreshTrue,
	}

	res, err := req.Do(ctx, e.client)
	if err != nil {
		common.Error("Failed to execute delete by query", err)
		if strings.Contains(err.Error(), "not_found") {
			return 0, nil
		}
		return 0, fmt.Errorf("failed to execute delete by query: %w", err)
	}
	defer res.Body.Close()

	if res.IsError() {
		bodyBytes, _ := io.ReadAll(res.Body)
		errStr := string(bodyBytes)
		if strings.Contains(errStr, "not_found") {
			return 0, nil
		}
		common.Sugar.Errorw("Elasticsearch delete by query returned error", "status", res.Status())
		return 0, fmt.Errorf("elasticsearch delete by query returned error: %s", res.Status())
	}

	// Parse response
	var result map[string]interface{}
	if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
		common.Error("Failed to parse delete response", err)
		return 0, fmt.Errorf("failed to parse delete response: %w", err)
	}

	deleted := int64(0)
	if d, ok := result["deleted"].(float64); ok {
		deleted = int64(d)
	}

	common.Info("Successfully deleted chunks", zap.String("index_name", fullIndexName), zap.Int64("deleted_count", deleted))
	return deleted, nil
}

// SearchResponse Elasticsearch search response
type SearchResponse struct {
	Hits struct {
		Total struct {
			Value int64 `json:"value"`
		} `json:"total"`
		Hits []struct {
			ID        string                 `json:"_id"`
			Index     string                 `json:"_index"`
			Score     float64                `json:"_score"`
			Source    map[string]interface{} `json:"_source"`
			Fields    map[string]interface{} `json:"fields"` // ES 9.x stores dense_vector here
			Highlight map[string]interface{} `json:"highlight,omitempty"`
			// Sort is populated when the request body specifies a `sort`
			// clause. The last hit's Sort is the cursor for the next
			// search_after request — without it, deep pagination can't
			// advance.
			Sort []interface{} `json:"sort,omitempty"`
		} `json:"hits"`
	} `json:"hits"`
	Aggregations map[string]interface{} `json:"aggregations"`
}

// Search executes search with unified types.SearchRequest
func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
	types.LogSearchRequest("Elasticsearch", req)

	// Validate inputs and set defaults
	if len(req.IndexNames) == 0 {
		return nil, fmt.Errorf("index names cannot be empty")
	}

	offset := max(req.Offset, 0)
	limit := req.Limit
	if limit <= 0 {
		limit = 30
	}

	// Detect index types
	isSkillIndex := false
	isMemoryIndex := false
	for _, idx := range req.IndexNames {
		if strings.HasPrefix(idx, "skill_") {
			isSkillIndex = true
		}
		if strings.HasPrefix(idx, "memory_") {
			isMemoryIndex = true
		}
	}

	// Build bool query from condition
	boolQuery := buildBoolQueryFromCondition(req.Filter, req.KbIDs, isSkillIndex, isMemoryIndex)

	// Extract vector_similarity_weight from FusionExpr
	var matchText *types.MatchTextExpr
	var matchDense *types.MatchDenseExpr
	vectorSimilarityWeight := 0.5
	for _, expr := range req.MatchExprs {
		if expr == nil {
			continue
		}
		switch m := expr.(type) {
		case *types.FusionExpr:
			if m.Method == "weighted_sum" {
				if weights, ok := m.FusionParams["weights"].(string); ok {
					// Assert structure only when FusionExpr has weighted_sum with weights
					if len(req.MatchExprs) != 3 {
						return nil, fmt.Errorf("match_expressions must have exactly 3 elements with FusionExpr, got %d", len(req.MatchExprs))
					}
					if _, ok := req.MatchExprs[0].(*types.MatchTextExpr); !ok {
						return nil, fmt.Errorf("match_expressions[0] must be MatchTextExpr")
					}
					if _, ok := req.MatchExprs[1].(*types.MatchDenseExpr); !ok {
						return nil, fmt.Errorf("match_expressions[1] must be MatchDenseExpr")
					}
					if _, ok := req.MatchExprs[2].(*types.FusionExpr); !ok {
						return nil, fmt.Errorf("match_expressions[2] must be FusionExpr")
					}
					parts := strings.Split(weights, ",")
					if len(parts) == 2 {
						if w, err := strconv.ParseFloat(parts[1], 64); err == nil {
							vectorSimilarityWeight = w
						}
					}
				}
			}
		case *types.MatchTextExpr:
			matchText = m
		case *types.MatchDenseExpr:
			matchDense = m
		}
	}

	// Build query body with text match and/or knn match
	queryBody := make(map[string]interface{})

	hasVectorMatch := matchDense != nil && len(matchDense.EmbeddingData) > 0

	if matchText != nil {
		textQuery := buildQueryStringQuery(matchText, isSkillIndex, isMemoryIndex)
		if boolQuery != nil {
			if boolMap, ok := boolQuery["bool"].(map[string]interface{}); ok {
				if must, ok := boolMap["must"].([]interface{}); ok {
					must = append(must, textQuery)
					boolMap["must"] = must
				} else {
					boolMap["must"] = []interface{}{textQuery}
				}
				// Weighted-sum fusion scales the text leg by (1 - w) against the
				// knn leg's w; a text-only query has no such pairing, so the
				// boolean filter wrapper must not apply the constant boost —
				// otherwise every BM25 score is silently multiplied by 0.5.
				//
				// Logged because it is the boost that orders the first-stage
				// result set, and so decides which candidates survive the
				// caller's size cap.
				if hasVectorMatch {
					boolMap["boost"] = 1.0 - vectorSimilarityWeight
				}
			}
		} else {
			boolQuery = textQuery
		}
	}

	if hasVectorMatch {
		if isMemoryIndex {
			if err := e.ensureMemoryMessageSearchVectorMappings(ctx, req.IndexNames, matchDense.VectorColumnName, len(matchDense.EmbeddingData)); err != nil {
				return nil, err
			}
		}

		k := matchDense.TopN
		if k <= 0 {
			k = limit
		}
		if k <= 0 {
			k = 1024
		}
		k = min(k, 10000)
		numCandidates := min(k*2, 10000)

		similarity := 0.0
		if matchDense.ExtraOptions != nil {
			if sim, ok := matchDense.ExtraOptions["similarity"].(float64); ok {
				similarity = sim
			}
			if n, ok := common.GetInt(matchDense.ExtraOptions["num_candidates"]); ok {
				numCandidates = max(k, min(n, 10000))
			}
		}

		vectorFieldName := matchDense.VectorColumnName

		knnQuery := map[string]interface{}{
			"field":          vectorFieldName,
			"query_vector":   matchDense.EmbeddingData,
			"k":              k,
			"num_candidates": numCandidates,
			"similarity":     similarity,
			"filter":         boolQuery,
		}

		queryBody["knn"] = knnQuery
		// A DENSE-ONLY request (no text expression: the vector-only search and
		// the knowledge-compile lookups) leaves the top-level query unset. The
		// same scope conditions already ride inside knn.filter, so repeating
		// them as a scored query adds one scoring pass per search AND blends
		// that score into _score — which is exactly the number a vector-only
		// caller reads as the cosine similarity.
		if boolQuery != nil && matchText != nil {
			queryBody["query"] = boolQuery
		}
	} else if boolQuery != nil {
		queryBody["query"] = boolQuery
	} else {
		queryBody["query"] = map[string]interface{}{
			"match_all": map[string]interface{}{},
		}
	}

	// Add rank_feature queries
	if req.RankFeature != nil && len(req.RankFeature) > 0 && !isSkillIndex && !isMemoryIndex {
		rankFeatureQuery := buildRankFeatureQuery(req.RankFeature)
		if rankFeatureQuery != nil {
			if boolQuery, ok := queryBody["query"].(map[string]interface{}); ok {
				if boolMap, ok := boolQuery["bool"].(map[string]interface{}); ok {
					if should, ok := boolMap["should"].([]interface{}); ok {
						for _, q := range rankFeatureQuery {
							boolMap["should"] = append(should, q)
						}
					} else {
						interfaceSlice := make([]interface{}, len(rankFeatureQuery))
						for i, q := range rankFeatureQuery {
							interfaceSlice[i] = q
						}
						boolMap["should"] = interfaceSlice
					}
				}
			}
		}
	}

	// Add sorting if order_by specified
	if req.OrderBy != nil && len(req.OrderBy.Fields) > 0 {
		sort := parseOrderByExpr(req.OrderBy)
		if len(sort) > 0 {
			queryBody["sort"] = sort
		}
	}

	// Determine use_search_after for deep pagination
	//
	// ES rejects from + size combinations where from + size > index.max_result_window
	// (default 10,000) — see https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html.
	// For those requests we must drop `from` and walk the result set with
	// `search_after` instead. The preconditions mirror the Python reference
	// (rag/utils/es_conn.py):
	//   - explicit OrderBy is required (search_after needs a stable cursor,
	//     and _score / KNN-similarity sorts are not unique enough to be safe)
	//   - no dense/KNN match (knn queries do not honour `search_after` in the
	//     same way and the Python path explicitly disallows them here)
	hasDense := hasVectorMatch
	hasExplicitSort := req.OrderBy != nil && len(req.OrderBy.Fields) > 0
	useSearchAfter := limit > 0 && (offset+limit > common.MAX_RESULT_WINDOW) && hasExplicitSort && !hasDense

	// Apply offset/limit pagination. When useSearchAfter is true, the
	// caller is going to drive pagination via searchAfterCursor()
	// instead, so we must NOT emit from/size here — leaving them out
	// is the whole point of routing to the search_after path.
	if !useSearchAfter && limit > 0 {
		queryBody["size"] = limit
		queryBody["from"] = offset
	}

	// Set _source and fields for vector fields
	hasTextMatch := matchText != nil
	selectFields := req.SelectFields
	if isMemoryIndex {
		selectFields = mapMemoryMessageESFields(req.SelectFields, false)
	}
	if len(req.SelectFields) > 0 {
		// Use caller-specified fields, add pagerank_fld/tag_fld if needed
		queryBody["_source"] = selectFields
		if hasTextMatch || hasVectorMatch {
			if !isSkillIndex && !isMemoryIndex {
				if !slices.Contains(selectFields, common.PAGERANK_FLD) {
					queryBody["_source"] = append(queryBody["_source"].([]string), common.PAGERANK_FLD)
				}
				if !slices.Contains(selectFields, common.TAG_FLD) {
					queryBody["_source"] = append(queryBody["_source"].([]string), common.TAG_FLD)
				}
			}
		}
		var vectorFields []string
		for _, f := range selectFields {
			if strings.HasSuffix(f, "_vec") {
				vectorFields = append(vectorFields, f)
			}
		}
		if len(vectorFields) > 0 {
			queryBody["fields"] = vectorFields
		}
	} else {
		// No explicit SelectFields - use match_all, but add pagerank_fld/tag_fld for scoring if needed
		if hasTextMatch || hasVectorMatch {
			if !isSkillIndex && !isMemoryIndex {
				queryBody["_source"] = []string{common.PAGERANK_FLD, common.TAG_FLD}
			}
		}
	}

	// Serialize query
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(queryBody); err != nil {
		return nil, fmt.Errorf("error encoding query: %w", err)
	}

	// Two legs (text + knn) decide the result set; when the window comes back short
	// the body is logged below, to tell "the index has less" from "the request
	// asked for less".
	hybrid := hasTextMatch && hasVectorMatch

	// Execute search. When useSearchAfter is true we must NOT send
	// from/size (we dropped them above) and instead walk the result set
	// page-by-page with the search_after cursor — ES otherwise returns
	// the first page and the caller gets the wrong page.
	var (
		totalHits  int64
		allResults []map[string]interface{}
		err        error
	)

	if useSearchAfter {
		allResults, totalHits, err = e.searchAfterCursor(ctx, req, queryBody, offset, limit)
		if err != nil {
			return nil, err
		}
	} else {
		// WithBody takes an io.Reader that the Go client streams
		// directly into the request. Reusing &buf across iterations
		// would drain it on the first request and leave the rest
		// with an empty body — so we copy the bytes once and hand
		// each iteration a fresh bytes.NewReader.
		payload := append([]byte(nil), buf.Bytes()...)
		for _, indexName := range req.IndexNames {
			searchChunks, indexTotal, esErr := e.searchOneIndex(ctx, indexName, payload)
			if esErr != nil {
				common.Warn("Elasticsearch query failed", zap.String("index", indexName), zap.Error(esErr))
				continue
			}
			if hybrid {
				// A window the backend could not fill is a candidate shortfall
				// upstream; only then is the request body worth its size.
				common.InfoCtx(ctx, "Elasticsearch hybrid response",
					zap.String("index", indexName),
					zap.Int64("total", indexTotal),
					zap.Int("returned", len(searchChunks)),
					zap.Int("window", limit))
				if limit > 0 && len(searchChunks) < limit {
					common.InfoCtx(ctx, "Elasticsearch hybrid body (window not filled)",
						zap.Strings("indexes", req.IndexNames),
						zap.String("body", elideQueryVector(payload)))
				}
			}
			totalHits += indexTotal
			allResults = append(allResults, searchChunks...)
		}
	}

	if isMemoryIndex {
		normalizeMemoryMessageChunks(allResults)
	}

	// Post-processing: Sort results by score
	if len(allResults) > 0 && (matchText != nil || hasVectorMatch) {
		// "_score" is the ES hit score. "SCORE" is Infinity's column and reads
		// back empty here, which flattened every hybrid chunk's score to 0.
		scoreColumn := "_score"

		pagerankField := common.PAGERANK_FLD
		if isSkillIndex {
			pagerankField = ""
		}
		if isMemoryIndex {
			pagerankField = ""
		}

		allResults = calculateScores(allResults, scoreColumn, pagerankField)
		allResults = sortByScore(allResults, limit)
	}

	return &types.SearchResult{
		Chunks: allResults,
		Total:  totalHits,
	}, nil
}

// esIndexNameForTenant maps one tenant id to its physical ES index. Regexp
// search deliberately has no multi-tenant mode.
func esIndexNameForTenant(tenantID string) (string, error) {
	tenantID = strings.TrimSpace(tenantID)
	if tenantID == "" {
		return "", fmt.Errorf("tenant id cannot be empty")
	}
	if strings.Contains(tenantID, ",") {
		return "", fmt.Errorf("regexp search supports exactly one tenant")
	}
	return "ragflow_" + tenantID, nil
}

// SearchByRegexp executes a regex-match-only search over chunk content. It
// reuses the same scope-filter builder and response converter as Search but
// emits a Lucene `regexp` query on the chunk content field instead of text /
// dense match expressions. This backs the agent's grep_chunks tool.
//
// The caller is responsible for translating a user regex into Lucene syntax;
// patterns using unsupported constructs should be detected and handled by the
// caller (fallback to broad recall + in-memory filtering).
func (e *Engine) SearchByRegexp(ctx context.Context, req *types.RegexpSearchRequest) (*types.SearchResult, error) {
	if req == nil {
		return nil, fmt.Errorf("regexp search request cannot be nil")
	}
	// Map the engine-agnostic tenant to this engine's physical index. ES
	// stores one index per tenant (ragflow_<tenant>).
	indexName, err := esIndexNameForTenant(req.TenantID)
	if err != nil {
		return nil, err
	}
	if strings.TrimSpace(req.Pattern) == "" {
		return nil, fmt.Errorf("regexp pattern cannot be empty")
	}

	const maxRegexpResults = 30
	maxResults := maxRegexpResults
	if req.ReturnAll {
		maxResults = 10000
	}

	// Build the scope filter (kb_id terms + available_int + explicit filters).
	boolQuery := buildBoolQueryFromCondition(req.Filter, req.KbIDs, false, false)

	// Attach the regexp query as a must clause on the chunk content field.
	// Two critical ES keyword-regexp behaviours:
	//  1. ES regexp matches the WHOLE field value, not a substring. To emulate
	//     "contains" (what grep_chunks and the old in-memory RE2 did), wrap the
	//     pattern as ".*(pattern).*" — a bare "何进" would only match a chunk
	//     whose content is exactly "何进" (yielding 0 hits).
	//  2. Set case_insensitive explicitly so the public grep contract remains
	//     case-insensitive for Latin text; it has no effect on CJK characters.
	regexpPattern := ".*(" + req.Pattern + ").*"
	regexpClause := map[string]interface{}{
		"regexp": map[string]interface{}{
			"content_with_weight": map[string]interface{}{
				"value":            regexpPattern,
				"flags":            "NONE",
				"case_insensitive": true,
			},
		},
	}

	if boolQuery == nil {
		boolQuery = map[string]interface{}{}
	}
	if boolMap, ok := boolQuery["bool"].(map[string]interface{}); ok {
		if must, ok := boolMap["must"].([]interface{}); ok {
			boolMap["must"] = append(must, regexpClause)
		} else {
			boolMap["must"] = []interface{}{regexpClause}
		}
	} else {
		boolQuery = map[string]interface{}{
			"bool": map[string]interface{}{
				"must": []interface{}{regexpClause},
			},
		}
	}

	// Regexp search is intentionally a first-page operation. Callers that need
	// deep reading use chunk anchors rather than offset pagination.
	queryBody := map[string]interface{}{
		"query": boolQuery,
		"size":  maxResults,
		"from":  0,
	}

	// When an explicit sort is requested (e.g. a document's reading order), push
	// it down so the capped first page is deterministic.
	if req.Sort != nil && len(req.Sort.Fields) > 0 {
		if sortClause := parseOrderByExpr(req.Sort); len(sortClause) > 0 {
			queryBody["sort"] = sortClause
		}
	}

	// Narrow the returned _source to the requested fields when given. The regexp
	// matches content_with_weight, so callers must keep it in SelectFields or the
	// hits will carry no content.
	if len(req.SelectFields) > 0 {
		queryBody["_source"] = req.SelectFields
	}

	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(queryBody); err != nil {
		return nil, fmt.Errorf("error encoding regexp query: %w", err)
	}

	payload := append([]byte(nil), buf.Bytes()...)

	res, err := e.client.Search(
		e.client.Search.WithContext(ctx),
		e.client.Search.WithIndex(indexName),
		e.client.Search.WithBody(bytes.NewReader(payload)),
		e.client.Search.WithTrackTotalHits(true),
	)
	if err != nil {
		common.Warn("Elasticsearch regexp query failed", zap.String("index", indexName), zap.Error(err))
		return nil, err
	}

	if res.IsError() {
		bodyBytes, _ := io.ReadAll(res.Body)
		res.Body.Close()
		common.Warn("Elasticsearch regexp error response", zap.String("index", indexName), zap.String("body", string(bodyBytes)))
		return nil, fmt.Errorf("elasticsearch regexp error on index %q: %s", indexName, strings.TrimSpace(string(bodyBytes)))
	}

	var esResp SearchResponse
	decodeErr := json.NewDecoder(res.Body).Decode(&esResp)
	res.Body.Close()
	if decodeErr != nil {
		common.Warn("Elasticsearch regexp failed to parse response", zap.String("index", indexName), zap.Error(decodeErr))
		return nil, fmt.Errorf("elasticsearch regexp parse error on index %q: %w", indexName, decodeErr)
	}

	allResults := convertESResponse(&esResp, "")
	totalHits := esResp.Hits.Total.Value
	if req.Sort != nil && len(req.Sort.Fields) > 0 {
		allResults = sortByFields(allResults, req.Sort)
	} else {
		// Regexp matches have no meaningful relevance score. Preserve ES's
		// returned order when no explicit deterministic sort was requested.
	}
	if len(allResults) > maxResults {
		allResults = allResults[:maxResults]
	}

	return &types.SearchResult{
		Chunks: allResults,
		Total:  totalHits,
	}, nil
}

// sortByFields orders chunks by the given OrderByExpr fields ascending. It is
// used for deterministic ordering of regexp results. Numeric fields sort
// numerically, string fields lexicographically.
func sortByFields(chunks []map[string]interface{}, expr *types.OrderByExpr) []map[string]interface{} {
	if expr == nil || len(expr.Fields) == 0 {
		return chunks
	}
	sort.SliceStable(chunks, func(i, j int) bool {
		for _, field := range expr.Fields {
			vi, oki := chunks[i][field.Field]
			vj, okj := chunks[j][field.Field]
			if !oki || !okj {
				// Missing field sorts before a present one.
				if !oki && !okj {
					continue
				}
				return !oki
			}
			// Compare the two values and derive a stable equality key. The raw
			// interface{} equality test below would panic on uncomparable types
			// ([]interface{}, map from ES source), so the equality check always
			// runs on string forms instead.
			var less bool
			var keyI, keyJ string
			if fi, ei := toFloat64(vi); ei {
				if fj, ej := toFloat64(vj); ej {
					less = fi < fj
				} else {
					less = false // numeric before non-numeric
				}
				keyI, keyJ = fmt.Sprintf("%v", vi), fmt.Sprintf("%v", vj)
			} else {
				si, siOk := vi.(string)
				sj, sjOk := vj.(string)
				if siOk && sjOk {
					less = si < sj
					keyI, keyJ = si, sj
				} else {
					keyI, keyJ = fmt.Sprintf("%v", vi), fmt.Sprintf("%v", vj)
					less = keyI < keyJ
				}
			}
			if keyI != keyJ {
				if field.Type == types.SortDesc {
					return !less
				}
				return less
			}
		}
		return false
	})
	return chunks
}

func (e *Engine) searchOneIndex(ctx context.Context, indexName string, payload []byte) ([]map[string]interface{}, int64, error) {
	res, err := e.client.Search(
		e.client.Search.WithContext(ctx),
		e.client.Search.WithIndex(indexName),
		e.client.Search.WithBody(bytes.NewReader(payload)),
		e.client.Search.WithTrackTotalHits(true),
	)
	if err != nil {
		common.Warn("Elasticsearch query failed", zap.String("index", indexName), zap.Error(err))
		return nil, 0, err
	}
	defer res.Body.Close()

	if res.IsError() {
		bodyBytes, _ := io.ReadAll(res.Body)
		common.Warn("Elasticsearch error response", zap.String("index", indexName), zap.String("body", string(bodyBytes)))
		return nil, 0, fmt.Errorf("elasticsearch error response: %s", string(bodyBytes))
	}

	// Parse response and return results
	var esResp SearchResponse
	if err = json.NewDecoder(res.Body).Decode(&esResp); err != nil {
		common.Warn("Elasticsearch failed to parse response", zap.String("index", indexName), zap.Error(err))
		return nil, 0, err
	}

	return convertESResponse(&esResp, ""), esResp.Hits.Total.Value, nil
}

// searchAfterFetcher issues one ES search request with the given batch
// size and search_after cursor, returning the decoded response. Defined
// as a function type so the pagination logic below can be unit-tested
// with a mock fetcher instead of a real Elasticsearch client.
type searchAfterFetcher func(
	ctx context.Context,
	baseQuery map[string]interface{},
	batch int,
	cursor []interface{},
	trackTotalHits bool,
) (SearchResponse, error)

// searchAfterCursor walks ES with the search_after pagination protocol,
// returning the page [offset, offset+limit) of an explicitly-sorted
// result set. Used when offset+limit exceeds common.MAX_RESULT_WINDOW
// and ES would otherwise reject the from/size combination.
//
// Mirrors rag/utils/es_conn.py:ESConnection._search_with_search_after:
//
//  1. Drop from/size from the base query (the caller has already omitted
//     them on this path; this is a defensive no-op).
//  2. Skip phase: discard hits until we have skipped `offset` of them.
//  3. Take phase: collect hits until we have `limit` of them, or the
//     index is exhausted.
//  4. After each batch, advance the cursor with the last hit's `sort`
//     field. If `sort` is missing or unchanged, the index is exhausted.
//
// The first request carries trackTotalHits=true so the caller still
// gets an accurate total; subsequent requests skip it for efficiency.
// Returns the (possibly empty) collected hits and the total hit count
// from the first response.
func (e *Engine) searchAfterCursor(
	ctx context.Context,
	req *types.SearchRequest,
	baseQuery map[string]interface{},
	offset, limit int,
) ([]map[string]interface{}, int64, error) {
	// Defensive: strip from/size if the caller left them in. In the
	// current code path they are never set when useSearchAfter is true,
	// but the base query is a shared map and future callers may forget.
	delete(baseQuery, "from")
	delete(baseQuery, "size")

	return searchAfterPaginate(ctx, baseQuery, offset, limit, e.buildSearchAfterFetcher(req))
}

// buildSearchAfterFetcher returns a fetcher that delegates each
// iteration to executeSearchRequest, which talks to the real ES client.
func (e *Engine) buildSearchAfterFetcher(req *types.SearchRequest) searchAfterFetcher {
	return func(
		ctx context.Context,
		baseQuery map[string]interface{},
		batch int,
		cursor []interface{},
		trackTotalHits bool,
	) (SearchResponse, error) {
		return e.executeSearchRequest(ctx, req, baseQuery, batch, cursor, trackTotalHits)
	}
}

// searchAfterPaginate is the pure, callback-driven pagination loop
// shared by the engine and the unit tests. See searchAfterCursor for
// the semantics.
func searchAfterPaginate(
	ctx context.Context,
	baseQuery map[string]interface{},
	offset, limit int,
	fetch searchAfterFetcher,
) ([]map[string]interface{}, int64, error) {
	var (
		cursor        []interface{}
		totalHits     int64
		collected     []map[string]interface{}
		collectedTake int
		firstCall     = true
	)

	// Skip phase: walk past `offset` hits without retaining them.
	remainingSkip := offset
	for remainingSkip > 0 {
		batch := min(remainingSkip, common.SearchAfterBatchSize)

		resp, err := fetch(ctx, baseQuery, batch, cursor, firstCall)
		firstCall = false
		if err != nil {
			return nil, 0, err
		}
		if totalHits == 0 {
			totalHits = resp.Hits.Total.Value
		}
		if len(resp.Hits.Hits) == 0 {
			break
		}
		nextCursor := resp.Hits.Hits[len(resp.Hits.Hits)-1].Sort
		if len(nextCursor) == 0 || sortValuesEqual(nextCursor, cursor) {
			// ES returned hits but no usable cursor (e.g. sort field
			// missing or unchanged). The index is exhausted from our
			// point of view.
			break
		}
		cursor = nextCursor
		remainingSkip -= len(resp.Hits.Hits)
		if len(resp.Hits.Hits) < batch {
			// Short batch — we asked for more than was available, so
			// the cursor is at the end of the index.
			break
		}
	}

	// Take phase: collect up to `limit` hits. ES may return up to
	// `batch` hits per request, but we stop at `limit` (the absolute
	// target) regardless of how many we asked for in this iteration.
	for collectedTake < limit {
		want := limit - collectedTake
		batch := min(want, common.SearchAfterBatchSize)

		resp, err := fetch(ctx, baseQuery, batch, cursor, firstCall)
		firstCall = false
		if err != nil {
			return nil, 0, err
		}
		if totalHits == 0 {
			totalHits = resp.Hits.Total.Value
		}
		if len(resp.Hits.Hits) == 0 {
			break
		}

		// Convert and append. We could parallelize the conversion with
		// the next request, but conversion is cheap relative to the
		// ES round-trip, so keep the loop straightforward.
		for _, hit := range resp.Hits.Hits {
			if collectedTake >= limit {
				break
			}
			chunk := hit.Source
			if chunk == nil {
				chunk = map[string]interface{}{}
			}
			chunk["_score"] = hit.Score
			chunk["_id"] = hit.ID
			chunk["id"] = hit.ID // shim id to _id, matching Python es_conn.py behavior
			chunk["_index"] = hit.Index
			collected = append(collected, chunk)
			collectedTake++
		}

		// Reached the absolute limit — stop without advancing the
		// cursor (we already have what was asked for).
		if collectedTake >= limit {
			break
		}

		nextCursor := resp.Hits.Hits[len(resp.Hits.Hits)-1].Sort
		if len(nextCursor) == 0 || sortValuesEqual(nextCursor, cursor) {
			break
		}
		cursor = nextCursor
		if len(resp.Hits.Hits) < batch {
			break
		}
	}

	// If we never sent a request (e.g. offset == 0 and limit == 0) we
	// still need a total. Issue one count-only request.
	if totalHits == 0 {
		resp, err := fetch(ctx, baseQuery, 0, nil, true)
		if err != nil {
			return nil, 0, err
		}
		totalHits = resp.Hits.Total.Value
	}

	return collected, totalHits, nil
}

// executeSearchRequest sends one ES search request with the given
// batch size and search_after cursor. If trackTotalHits is true the
// request asks ES to compute an exact total (cheap to omit on
// pagination iterations after the first).
func (e *Engine) executeSearchRequest(
	ctx context.Context,
	req *types.SearchRequest,
	baseQuery map[string]interface{},
	batch int,
	cursor []interface{},
	trackTotalHits bool,
) (SearchResponse, error) {
	queryBody := make(map[string]interface{}, len(baseQuery)+2)
	for k, v := range baseQuery {
		queryBody[k] = v
	}
	if batch > 0 {
		queryBody["size"] = batch
	}
	if len(cursor) > 0 {
		queryBody["search_after"] = cursor
	}

	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(queryBody); err != nil {
		return SearchResponse{}, fmt.Errorf("error encoding query: %w", err)
	}

	res, err := e.client.Search(
		e.client.Search.WithContext(ctx),
		e.client.Search.WithIndex(req.IndexNames...),
		e.client.Search.WithBody(&buf),
		e.client.Search.WithTrackTotalHits(trackTotalHits),
	)
	if err != nil {
		return SearchResponse{}, fmt.Errorf("elasticsearch search failed: %w", err)
	}
	defer res.Body.Close()

	if res.IsError() {
		bodyBytes, _ := io.ReadAll(res.Body)
		return SearchResponse{}, fmt.Errorf("elasticsearch error response: %s", string(bodyBytes))
	}

	var esResp SearchResponse
	if err := json.NewDecoder(res.Body).Decode(&esResp); err != nil {
		return SearchResponse{}, fmt.Errorf("elasticsearch failed to parse response: %w", err)
	}
	return esResp, nil
}

// sortValuesEqual reports whether two sort cursors are identical.
// ES guarantees that successive requests with `search_after: <cursor>`
// advance strictly past the cursor, so an unchanged cursor between
// iterations means the index is exhausted.
func sortValuesEqual(a, b []interface{}) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

// mapMemoryMessageToESDoc converts a logical memory message into the
// Elasticsearch storage document, mirroring Python
// memory/utils/es_conn.py:map_message_to_es_fields. Elasticsearch has no
// server-side rag tokenizer, so tokenized_content_ltks is computed here
// before write; on Infinity the equivalent insert path stores content
// verbatim and lets Infinity tokenize on save.
func mapMemoryMessageToESDoc(message map[string]interface{}) map[string]interface{} {
	doc := make(map[string]interface{}, len(message)+2)
	for k, v := range message {
		switch k {
		case "message_type":
			doc["message_type_kwd"] = v
		case "status":
			doc["status_int"] = memoryMessageStatusInt(v)
		case "content":
			content, _ := v.(string)
			doc["content_ltks"] = content
			doc["tokenized_content_ltks"] = tokenizeMemoryMessageContent(content)
		default:
			doc[k] = v
		}
	}
	return doc
}

// tokenizeMemoryMessageContent runs the rag tokenizer pipeline
// (tokenize + fine-grained tokenize) over message content. On tokenizer
// failure it degrades to the untokenized text so the document stays
// searchable under the whitespace analyzer.
func tokenizeMemoryMessageContent(content string) string {
	tokenized, err := tokenizer.Tokenize(content)
	if err != nil {
		common.Warn("memory message tokenize failed, storing raw content", zap.Error(err))
		return content
	}
	fine, err := tokenizer.FineGrainedTokenize(tokenized)
	if err != nil {
		common.Warn("memory message fine-grained tokenize failed, storing coarse tokens", zap.Error(err))
		return tokenized
	}
	return fine
}

func memoryMessageStatusInt(value interface{}) int {
	if memoryMessageStatusBool(value) {
		return 1
	}
	return 0
}

func mapMemoryMessageESField(field string, useTokenizedContent bool) string {
	name := field
	boost := ""
	if base, suffix, ok := strings.Cut(field, "^"); ok {
		name = base
		boost = "^" + suffix
	}

	switch name {
	case "message_type":
		name = "message_type_kwd"
	case "status":
		name = "status_int"
	case "content":
		if useTokenizedContent {
			name = "tokenized_content_ltks"
		} else {
			name = "content_ltks"
		}
	}
	return name + boost
}

func mapMemoryMessageESFields(fields []string, useTokenizedContent bool) []string {
	if len(fields) == 0 {
		return fields
	}
	mapped := make([]string, 0, len(fields))
	seen := make(map[string]struct{}, len(fields))
	for _, field := range fields {
		mappedField := mapMemoryMessageESField(field, useTokenizedContent)
		if _, ok := seen[mappedField]; ok {
			continue
		}
		seen[mappedField] = struct{}{}
		mapped = append(mapped, mappedField)
	}
	return mapped
}

func normalizeMemoryMessageChunks(chunks []map[string]interface{}) {
	for _, chunk := range chunks {
		for key, val := range chunk {
			if memoryMessageVectorFieldRE.MatchString(key) {
				chunk["content_embed"] = val
				delete(chunk, key)
			}
		}
		if val, ok := chunk["message_type_kwd"]; ok {
			chunk["message_type"] = val
			delete(chunk, "message_type_kwd")
		}
		if val, ok := chunk["status_int"]; ok {
			chunk["status"] = memoryMessageStatusBool(val)
			delete(chunk, "status_int")
		}
		if val, ok := chunk["content_ltks"]; ok {
			chunk["content"] = val
			delete(chunk, "content_ltks")
		}
	}
}

func memoryMessageStatusBool(value interface{}) bool {
	switch v := value.(type) {
	case bool:
		return v
	case int:
		return v != 0
	case int64:
		return v != 0
	case float64:
		return v != 0
	case json.Number:
		n, err := v.Int64()
		return err == nil && n != 0
	case string:
		return v != "" && v != "0" && !strings.EqualFold(v, "false")
	default:
		return false
	}
}

// buildBoolQueryFromCondition builds an ES bool query from condition map.
// Skill indexes use status, regular chunk indexes use kb_id, and memory
// message indexes use memory_id plus message-specific storage fields.
func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string, isSkillIndex, isMemoryIndex bool) map[string]interface{} {
	var mustClauses []interface{}
	var mustNotClauses []interface{}
	var filterClauses []interface{}
	var shouldClauses []interface{}

	// Memory message indexes use memory_id, regular chunk indexes use kb_id.
	if kbIDs != nil && len(kbIDs) > 0 {
		fieldName := "kb_id"
		if isMemoryIndex {
			fieldName = "memory_id"
		}
		filterClauses = append(filterClauses, map[string]interface{}{
			"terms": map[string]interface{}{fieldName: kbIDs},
		})
	}

	// For skill index, add status = "1" filter by default (active skills)
	if isSkillIndex {
		filterClauses = append(filterClauses, map[string]interface{}{
			"term": map[string]interface{}{
				"status": "1",
			},
		})
	}

	if filter == nil {
		filter = make(map[string]interface{})
	}

	for k, v := range filter {
		if isMemoryIndex {
			k = mapMemoryMessageESField(k, false)
		}
		// For skill index, handle 'status' field instead of 'available_int'
		if isSkillIndex && k == "status" {
			if v == nil || v == "" {
				continue
			}
			if listVal, ok := v.([]interface{}); ok && len(listVal) > 0 {
				filterClauses = append(filterClauses, map[string]interface{}{
					"terms": map[string]interface{}{"status": listVal},
				})
			} else if strVal, ok := v.(string); ok && strVal != "" {
				filterClauses = append(filterClauses, map[string]interface{}{
					"term": map[string]interface{}{"status": strVal},
				})
			}
			continue
		}
		if k == "available_int" {
			var numVal float64
			switch val := v.(type) {
			case float64:
				numVal = val
			case int:
				numVal = float64(val)
			case int64:
				numVal = float64(val)
			default:
				continue
			}
			if numVal == 0 {
				filterClauses = append(filterClauses, map[string]interface{}{
					"range": map[string]interface{}{"available_int": map[string]interface{}{"lt": 1}},
				})
			} else {
				filterClauses = append(filterClauses, map[string]interface{}{
					"bool": map[string]interface{}{
						"must_not": []map[string]interface{}{
							{"range": map[string]interface{}{"available_int": map[string]interface{}{"lt": 1}}},
						},
					},
				})
			}
			continue
		}
		if k == "must_not" {
			if condition, ok := v.(map[string]interface{}); ok {
				if field, ok := condition["exists"].(string); ok && field != "" {
					mustNotClauses = append(mustNotClauses, map[string]interface{}{"exists": map[string]interface{}{"field": field}})
				}
			}
			continue
		}
		if k == "knowledge_graph_kwd" {
			var graphTypeClauses []interface{}
			for _, field := range []string{"type_kwd", "knowledge_graph_kwd"} {
				switch val := v.(type) {
				case []interface{}:
					if len(val) > 0 {
						graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"terms": map[string]interface{}{field: val}})
					}
				case []string:
					if len(val) > 0 {
						graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"terms": map[string]interface{}{field: val}})
					}
				case string:
					if val != "" {
						graphTypeClauses = append(graphTypeClauses, map[string]interface{}{"term": map[string]interface{}{field: val}})
					}
				}
			}
			if len(graphTypeClauses) > 0 {
				filterClauses = append(filterClauses, map[string]interface{}{"bool": map[string]interface{}{
					"should": graphTypeClauses, "minimum_should_match": 1,
				}})
			}
			continue
		}
		if k == "id" {
			if v == nil || v == "" {
				continue
			}
			if listVal, ok := v.([]interface{}); ok && len(listVal) > 0 {
				shouldClauses = append(shouldClauses,
					map[string]interface{}{"terms": map[string]interface{}{"id": listVal}},
					map[string]interface{}{"terms": map[string]interface{}{"_id": listVal}},
				)
			} else if strListVal, ok := v.([]string); ok && len(strListVal) > 0 {
				// A typed []string MUST be handled here: the generic loop
				// below skips the "id" key, so without this branch a caller
				// passing map[string]interface{}{"id": []string{...}} builds a
				// query with NO id filter and the scoped read silently
				// degenerates into "fetch up to Limit chunks of the document"
				// (observed via list_chunks: an 11-chunk window returned 3.8MB
				// because the whole 4171-chunk document was fetched instead).
				shouldClauses = append(shouldClauses,
					map[string]interface{}{"terms": map[string]interface{}{"id": strListVal}},
					map[string]interface{}{"terms": map[string]interface{}{"_id": strListVal}},
				)
			} else if strVal, ok := v.(string); ok && strVal != "" {
				shouldClauses = append(shouldClauses,
					map[string]interface{}{"term": map[string]interface{}{"id": strVal}},
					map[string]interface{}{"term": map[string]interface{}{"_id": strVal}},
				)
			} else if intVal, ok := v.(int); ok && intVal != 0 {
				shouldClauses = append(shouldClauses,
					map[string]interface{}{"term": map[string]interface{}{"id": intVal}},
					map[string]interface{}{"term": map[string]interface{}{"_id": intVal}},
				)
			}
			continue
		}
		if v == nil || v == "" {
			continue
		}
		if isMemoryIndex && k == "session_id" {
			if strVal, ok := v.(string); ok && strVal != "" {
				filterClauses = append(filterClauses, map[string]interface{}{
					"query_string": map[string]interface{}{
						"query":            fmt.Sprintf("*%s*", strVal),
						"fields":           []string{"session_id"},
						"analyze_wildcard": true,
					},
				})
				continue
			}
		}
		if listVal, ok := v.([]interface{}); ok {
			filterClauses = append(filterClauses, map[string]interface{}{
				"terms": map[string]interface{}{k: listVal},
			})
		} else if strListVal, ok := v.([]string); ok {
			filterClauses = append(filterClauses, map[string]interface{}{
				"terms": map[string]interface{}{k: strListVal},
			})
		} else if strVal, ok := v.(string); ok && strVal != "" {
			filterClauses = append(filterClauses, map[string]interface{}{
				"term": map[string]interface{}{k: strVal},
			})
		} else if intVal, ok := v.(int); ok {
			filterClauses = append(filterClauses, map[string]interface{}{
				"term": map[string]interface{}{k: intVal},
			})
		} else if floatVal, ok := v.(float64); ok {
			filterClauses = append(filterClauses, map[string]interface{}{
				"term": map[string]interface{}{k: floatVal},
			})
		}
	}

	// Build the bool query
	boolQuery := make(map[string]interface{})
	if len(mustClauses) > 0 {
		boolQuery["must"] = mustClauses
	}
	if len(mustNotClauses) > 0 {
		boolQuery["must_not"] = mustNotClauses
	}
	if len(filterClauses) > 0 {
		boolQuery["filter"] = filterClauses
	}
	if len(shouldClauses) > 0 {
		boolQuery["should"] = shouldClauses
		boolQuery["minimum_should_match"] = 1
	}

	if len(boolQuery) == 0 {
		return nil
	}

	return map[string]interface{}{"bool": boolQuery}
}

// buildQueryStringQuery builds a query_string query from MatchTextExpr
// When isSkillIndex is true, uses skill-specific fields (name_tks, tags_tks, etc.)
// Otherwise uses document fields (title_tks, content_ltks, etc.)
//
// The MatchingText is lowercased: the *_tks/*_ltks fields it targets are
// tokenized with the whitespace analyzer and RAGFlow's ingest pipeline stores
// pre-lowercased tokens there, but the whitespace search analyzer does NOT
// lowercase the query. Capitalized terms (any proper noun: "Ross Feldner",
// "New Age Graphics") therefore silently match nothing, and a keyword query
// consisting only of proper nouns returns zero hits — observed live when a
// perfect pivot query ("Isabel Wood co-lead Ross Feldner ...") dead-ended a
// benchmark question that was answerable from the corpus (q156).
func buildQueryStringQuery(matchText *types.MatchTextExpr, isSkillIndex, isMemoryIndex bool) map[string]interface{} {
	if matchText == nil {
		return nil
	}

	minimumShouldMatch := "0%"
	if matchText.ExtraOptions != nil {
		if msm, ok := matchText.ExtraOptions["minimum_should_match"].(float64); ok {
			minimumShouldMatch = common.FormatMinimumShouldMatchPercent(msm)
		}
	}

	fields := matchText.Fields
	if fields == nil || len(fields) == 0 {
		if isSkillIndex {
			fields = []string{"name_tks^10", "tags_tks^5", "description_tks^3", "content_tks^1"}
		} else if isMemoryIndex {
			fields = []string{"tokenized_content_ltks"}
		} else {
			fields = []string{"title_tks^10", "title_sm_tks^5", "important_kwd^30", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"}
		}
	}
	if isSkillIndex {
		fields = mapSkillSearchFields(fields)
	}
	if isMemoryIndex {
		fields = mapMemoryMessageESFields(fields, true)
	}

	boost := 1.0
	if matchText.ExtraOptions != nil {
		if b, ok := matchText.ExtraOptions["boost"].(float64); ok {
			boost = b
		}
	}

	return map[string]interface{}{
		"query_string": map[string]interface{}{
			"fields":               fields,
			"type":                 "best_fields",
			"query":                lowerCaseQueryText(matchText.MatchingText),
			"minimum_should_match": minimumShouldMatch,
			"boost":                boost,
		},
	}
}

func mapSkillSearchFields(fields []string) []string {
	mapped := make([]string, 0, len(fields))
	for _, field := range fields {
		name, boost, hasBoost := strings.Cut(field, "^")
		switch name {
		case "name":
			name = "name_tks"
		case "tags":
			name = "tags_tks"
		case "description":
			name = "description_tks"
		case "content":
			name = "content_tks"
		}
		if hasBoost {
			mapped = append(mapped, name+"^"+boost)
		} else {
			mapped = append(mapped, name)
		}
	}
	return mapped
}

// buildRankFeatureQuery builds rank_feature queries for learning to rank
func buildRankFeatureQuery(rankFeature map[string]float64) []map[string]interface{} {
	if rankFeature == nil || len(rankFeature) == 0 {
		return nil
	}

	// Sort keys for deterministic query order (Go map iteration is randomized)
	keys := make([]string, 0, len(rankFeature))
	for k := range rankFeature {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	var queries []map[string]interface{}
	for _, fld := range keys {
		if fld == common.PAGERANK_FLD {
			continue
		}
		sc := rankFeature[fld]
		tagField := fmt.Sprintf("%s.%s", common.TAG_FLD, fld)
		queries = append(queries, map[string]interface{}{
			"rank_feature": map[string]interface{}{
				"field":  tagField,
				"linear": map[string]interface{}{},
				"boost":  sc,
			},
		})
	}
	return queries
}

// GetChunk gets a chunk by ID using ES search API
func (e *Engine) GetChunk(ctx context.Context, baseName, chunkID string, datasetIDs []string) (interface{}, error) {
	if strings.HasPrefix(baseName, "memory_") {
		return e.getMemoryMessage(ctx, baseName, chunkID)
	}

	// Try search by doc_id field (which is stored in the document)
	for _, datasetID := range datasetIDs {
		source, found, err := e.searchChunkInDataset(ctx, baseName, chunkID, datasetID)
		if err != nil {
			return nil, err
		}
		if !found {
			continue
		}
		common.Info("GetChunk found hit", zap.String("baseName", baseName), zap.String("chunkID", chunkID))
		source["id"] = chunkID
		return source, nil
	}

	common.Info("GetChunk no hits found", zap.String("baseName", baseName), zap.String("chunkID", chunkID))
	return nil, nil
}

func (e *Engine) searchChunkInDataset(ctx context.Context, baseName, chunkID, datasetID string) (map[string]interface{}, bool, error) {
	searchReq := map[string]interface{}{
		"query": map[string]interface{}{
			"bool": map[string]interface{}{
				"must": []map[string]interface{}{
					{"term": map[string]interface{}{"id": chunkID}},
					{"term": map[string]interface{}{"kb_id": datasetID}},
				},
			},
		},
	}

	body, err := json.Marshal(searchReq)
	if err != nil {
		return nil, false, fmt.Errorf("failed to marshal search request: %w", err)
	}

	res, err := e.client.Search(
		e.client.Search.WithContext(ctx),
		e.client.Search.WithIndex(baseName),
		e.client.Search.WithBody(bytes.NewReader(body)),
	)

	defer closeESBody(res)

	if err != nil {
		return nil, false, fmt.Errorf("failed to search for chunk: %w", err)
	}

	if res.IsError() {
		return nil, false, fmt.Errorf("failed to search for chunk: %s", res.Status())
	}

	var searchResult map[string]interface{}
	if err = json.NewDecoder(res.Body).Decode(&searchResult); err != nil {
		return nil, false, fmt.Errorf("failed to parse search response: %w", err)
	}

	hits, ok := searchResult["hits"].(map[string]interface{})
	if !ok {
		return nil, false, nil
	}

	hitList, ok := hits["hits"].([]interface{})
	if !ok || len(hitList) == 0 {
		return nil, false, nil
	}

	firstHit, ok := hitList[0].(map[string]interface{})
	if !ok {
		return nil, false, nil
	}

	source, ok := firstHit["_source"].(map[string]interface{})
	if !ok {
		return nil, false, nil
	}
	return source, true, nil
}

func (e *Engine) getMemoryMessage(ctx context.Context, indexName, docID string) (interface{}, error) {
	req := esapi.GetRequest{
		Index:      indexName,
		DocumentID: docID,
	}
	res, err := req.Do(ctx, e.client)
	if err != nil {
		return nil, fmt.Errorf("failed to get memory message: %w", err)
	}
	defer res.Body.Close()

	if res.IsError() {
		if res.StatusCode == http.StatusNotFound {
			return nil, fmt.Errorf("%w: %s", types.ErrDocumentNotFound, docID)
		}
		return nil, fmt.Errorf("elasticsearch memory message get error: %s", res.Status())
	}

	var getResult struct {
		Found  bool                   `json:"found"`
		Source map[string]interface{} `json:"_source"`
	}
	if err = json.NewDecoder(res.Body).Decode(&getResult); err != nil {
		return nil, fmt.Errorf("failed to parse memory message get response: %w", err)
	}
	if !getResult.Found || getResult.Source == nil {
		return nil, nil
	}

	message := getResult.Source
	message["id"] = docID
	normalizeMemoryMessageChunks([]map[string]interface{}{message})
	return message, nil
}

// GetFields extracts the requested fields from ES search response chunks
//
// Unlike Infinity, Elasticsearch does NOT use convertSelectFields before querying.
// The original requested field names ARE the database column names:
//   - "content_with_weight" is stored and returned as "content_with_weight"
//   - No field name mapping is needed in GetFields
func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} {
	common.Info("GetFields called", zap.Int("chunkCount", len(chunks)), zap.Strings("fields", fields))
	result := make(map[string]map[string]interface{})

	if len(fields) == 0 || len(chunks) == 0 {
		return result
	}

	// Build field set for lookup
	fieldSet := make(map[string]bool)
	for _, f := range fields {
		fieldSet[f] = true
	}

	for _, chunk := range chunks {
		docID, ok := elasticsearchChunkID(chunk)
		if !ok {
			continue
		}
		if id, ok := chunk["id"].(string); !ok || id == "" {
			chunk["id"] = docID
		}

		m := make(map[string]interface{})
		for field := range fieldSet {
			val := chunk[field]

			if val == nil {
				continue
			}

			if listVal, ok := val.([]interface{}); ok {
				if len(listVal) == 1 {
					if _, isArray := listVal[0].([]interface{}); !isArray {
						val = listVal[0]
					}
				}
			}

			if _, ok = val.([]interface{}); ok {
				m[field] = val
				continue
			}

			if field == "available_int" {
				if _, ok = val.(int); ok {
					m[field] = val
					continue
				}
				if _, ok = val.(float64); ok {
					m[field] = val
					continue
				}
			}

			if _, ok = val.(string); !ok {
				val = fmt.Sprintf("%v", val)
			}
			m[field] = val
		}

		if len(m) > 0 {
			result[docID] = m
		}
	}

	common.Info("GetFields result", zap.Int("resultCount", len(result)), zap.Strings("keys", func() []string {
		keys := make([]string, 0, len(result))
		for k := range result {
			keys = append(keys, k)
		}
		return keys
	}()))
	return result
}

// GetAggregation aggregates chunk values by field name
// Input: [{"docnm_kwd": "docA"}, {"docnm_kwd": "docA"}, {"docnm_kwd": "docB"}]
// Returns: [{"key": "docA", "count": 2}, {"key": "docB", "count": 1}]
func (e *Engine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} {
	if len(chunks) == 0 || fieldName == "" {
		return []map[string]interface{}{}
	}

	tagCounts := make(map[string]int)
	for _, chunk := range chunks {
		value, ok := chunk[fieldName]
		if !ok || value == nil {
			continue
		}

		if valueStr, ok := value.(string); ok {
			if valueStr == "" {
				continue
			}
			separator := ","
			if fieldName == "tag_kwd" && strings.Contains(valueStr, "###") {
				separator = "###"
			}
			for tag := range strings.SplitSeq(valueStr, separator) {
				countElasticsearchAggregationTag(tagCounts, tag)
			}
			continue
		}

		if valueList, ok := value.([]interface{}); ok {
			for _, item := range valueList {
				if itemStr, ok := item.(string); ok {
					countElasticsearchAggregationTag(tagCounts, itemStr)
				}
			}
		}
	}

	if len(tagCounts) == 0 {
		return []map[string]interface{}{}
	}

	tags := make([]string, 0, len(tagCounts))
	for tag := range tagCounts {
		tags = append(tags, tag)
	}
	slices.SortFunc(tags, func(a, b string) int {
		if byCount := cmp.Compare(tagCounts[b], tagCounts[a]); byCount != 0 {
			return byCount
		}
		return cmp.Compare(a, b)
	})

	result := make([]map[string]interface{}, len(tags))
	for i, tag := range tags {
		result[i] = map[string]interface{}{"key": tag, "count": tagCounts[tag]}
	}

	return result
}

// GetChunkIDs extracts chunk IDs from ES search response chunks.
// Uses _id field (composite: {doc_id}_{kb_id}_{chunk_id}).
func (e *Engine) GetChunkIDs(chunks []map[string]interface{}) []string {
	ids := make([]string, 0, len(chunks))
	for _, chunk := range chunks {
		if id, ok := elasticsearchChunkID(chunk); ok {
			ids = append(ids, id)
		}
	}
	return ids
}

// GetHighlight returns highlighted text for matching keywords
func (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string {
	result := make(map[string]string)
	pattern := compileElasticsearchHighlightPattern(keywords)

	for _, chunk := range chunks {
		docID, ok := elasticsearchChunkID(chunk)
		if !ok {
			continue
		}

		txt, ok := chunk[fieldName].(string)
		if !ok {
			continue
		}
		if pattern != nil {
			txt = pattern.ReplaceAllStringFunc(txt, func(match string) string {
				return "<em>" + match + "</em>"
			})
		}
		result[docID] = txt
	}
	return result
}

func elasticsearchChunkID(chunk map[string]interface{}) (string, bool) {
	if id, ok := chunk["id"].(string); ok && id != "" {
		return id, true
	}
	if id, ok := chunk["_id"].(string); ok && id != "" {
		return id, true
	}
	return "", false
}

func countElasticsearchAggregationTag(counts map[string]int, tag string) {
	if tag = strings.TrimSpace(tag); tag != "" {
		counts[tag]++
	}
}

func compileElasticsearchHighlightPattern(keywords []string) *regexp.Regexp {
	nonEmpty := make([]string, 0, len(keywords))
	for _, kw := range keywords {
		if kw != "" {
			nonEmpty = append(nonEmpty, kw)
		}
	}
	if len(nonEmpty) == 0 {
		return nil
	}
	slices.SortStableFunc(nonEmpty, func(a, b string) int {
		return cmp.Compare(utf8.RuneCountInString(b), utf8.RuneCountInString(a))
	})
	parts := make([]string, len(nonEmpty))
	for i, keyword := range nonEmpty {
		parts[i] = regexp.QuoteMeta(keyword)
		if isLatinKeyword(keyword) {
			parts[i] += `\p{Latin}*`
		}
	}
	return regexp.MustCompile("(?i)" + strings.Join(parts, "|"))
}

func isLatinKeyword(keyword string) bool {
	for _, r := range keyword {
		if !unicode.In(r, unicode.Latin) {
			return false
		}
	}
	return keyword != ""
}

// DropChunkStore deletes a chunk index
func (e *Engine) DropChunkStore(ctx context.Context, baseName, datasetID string) error {
	return e.dropIndex(ctx, baseName)
}

// ChunkStoreExists checks if a chunk index exists
func (e *Engine) ChunkStoreExists(ctx context.Context, baseName, datasetID string) (bool, error) {
	return e.indexExists(ctx, baseName)
}

// KNNScores performs a second-pass KNN search to get clean cosine similarities for ES.
// This keeps chunk vectors in the index and asks ES to compute the cosine similarity.
func (e *Engine) KNNScores(ctx context.Context, chunks []map[string]interface{}, queryVector []float64, topK int) (map[string]interface{}, error) {
	if len(chunks) == 0 || len(queryVector) == 0 {
		return nil, nil
	}

	// Extract chunk IDs from first search results
	chunkIDs := make([]string, 0, len(chunks))
	for _, chunk := range chunks {
		if id, ok := chunk["_id"].(string); ok {
			chunkIDs = append(chunkIDs, id)
		}
	}
	if len(chunkIDs) == 0 {
		return nil, nil
	}

	common.Info("KNNScores starting", zap.Int("chunkCount", len(chunkIDs)), zap.Strings("chunkIDs", chunkIDs), zap.Int("vectorSize", len(queryVector)))

	// Build KNN-only query filtered by chunk IDs
	vectorSize := len(queryVector)
	k := len(chunkIDs)
	knnQuery := map[string]interface{}{
		"field":          fmt.Sprintf("q_%d_vec", vectorSize),
		"query_vector":   queryVector,
		"k":              k,
		"num_candidates": k * 2,
		"similarity":     0.0, // No threshold - get all
		"filter": map[string]interface{}{
			"terms": map[string]interface{}{"id": chunkIDs},
		},
	}

	queryBody := map[string]interface{}{
		"knn":     knnQuery,
		"size":    k,
		"_source": false, // Don't need source fields, only need _id and _score
	}

	body, err := json.Marshal(queryBody)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal KNN query: %w", err)
	}

	//common.Info("KNNScores query body", zap.String("body", string(body)))

	// Execute search - use first index name from chunks if available
	indexName := ""
	if len(chunks) > 0 {
		if idx, ok := chunks[0]["_index"].(string); ok {
			indexName = idx
		}
	}

	res, err := e.client.Search(
		e.client.Search.WithContext(ctx),
		e.client.Search.WithIndex(indexName),
		e.client.Search.WithBody(bytes.NewReader(body)),
	)
	if err != nil {
		return nil, fmt.Errorf("KNN scores search failed: %w", err)
	}
	defer res.Body.Close()

	if res.IsError() {
		bodyBytes, _ := io.ReadAll(res.Body)
		return nil, fmt.Errorf("KNN scores search returned error: %s, body: %s", res.Status(), string(bodyBytes))
	}

	var esResp SearchResponse
	if err := json.NewDecoder(res.Body).Decode(&esResp); err != nil {
		return nil, fmt.Errorf("failed to parse KNN scores response: %w", err)
	}

	common.Info("KNNScores ES response", zap.Int("hitCount", len(esResp.Hits.Hits)), zap.Any("firstHit", func() interface{} {
		if len(esResp.Hits.Hits) > 0 {
			return esResp.Hits.Hits[0]
		}
		return nil
	}()))

	// Return raw ES response
	// Caller will pass to GetScores to extract scores
	knnResult := make(map[string]interface{})
	knnResult["hits"] = map[string]interface{}{
		"hits": esResp.Hits.Hits,
	}
	return knnResult, nil
}

// GetScores extracts similarity scores from KNN search result
func (e *Engine) GetScores(knnResult map[string]interface{}) map[string]float64 {
	scores := make(map[string]float64)
	hits, ok := knnResult["hits"].(map[string]interface{})
	if !ok {
		return scores
	}
	hitsList, ok := hits["hits"]
	if !ok {
		return scores
	}

	switch v := hitsList.(type) {
	case []interface{}:
		for _, h := range v {
			if hit, ok := h.(map[string]interface{}); ok {
				if docID, ok := hit["_id"].(string); ok && docID != "" {
					if scoreVal := hit["_score"]; scoreVal != nil {
						if score, ok := scoreVal.(float64); ok {
							scores[docID] = score
						}
					}
				}
			}
		}
	case []map[string]interface{}:
		for _, hit := range v {
			if docID, ok := hit["_id"].(string); ok && docID != "" {
				if scoreVal := hit["_score"]; scoreVal != nil {
					if score, ok := scoreVal.(float64); ok {
						scores[docID] = score
					}
				}
			}
		}
	default:
		// Handle slice of structs via reflection
		rv := reflect.ValueOf(v)
		if rv.Kind() == reflect.Slice {
			for i := 0; i < rv.Len(); i++ {
				elem := rv.Index(i)
				idField := elem.FieldByName("ID")
				if !idField.IsValid() {
					idField = elem.FieldByName("Id")
				}
				if !idField.IsValid() || idField.Kind() != reflect.String {
					continue
				}
				docID := idField.String()
				if docID == "" {
					continue
				}
				scoreField := elem.FieldByName("Score")
				if !scoreField.IsValid() || scoreField.Kind() != reflect.Float64 {
					continue
				}
				scores[docID] = scoreField.Float()
			}
		}
	}

	return scores
}

// loadSkillMapping loads the skill index mapping from config file
func loadSkillMapping() (map[string]interface{}, error) {
	// Try multiple possible locations for the mapping file
	possiblePaths := []string{
		"conf/skill_es_mapping.json",
		"../conf/skill_es_mapping.json",
		"/app/conf/skill_es_mapping.json",
	}

	var data []byte
	var err error
	for _, path := range possiblePaths {
		data, err = os.ReadFile(path)
		if err == nil {
			break
		}
	}

	if err != nil {
		// Fallback to default skill mapping if file not found
		return getDefaultSkillMapping(), nil
	}

	var mapping map[string]interface{}
	if err := json.Unmarshal(data, &mapping); err != nil {
		return nil, fmt.Errorf("failed to parse skill mapping: %w", err)
	}

	return mapping, nil
}

func memoryMessageVectorField(vectorSize int) string {
	return fmt.Sprintf("q_%d_vec", vectorSize)
}

func memoryMessageVectorProperty(vectorSize int) map[string]interface{} {
	return map[string]interface{}{
		"type":       "dense_vector",
		"dims":       vectorSize,
		"index":      true,
		"similarity": "cosine",
	}
}

func parseMemoryMessageVectorSize(field string) (int, bool) {
	if !memoryMessageVectorFieldRE.MatchString(field) {
		return 0, false
	}
	sizeText := strings.TrimSuffix(strings.TrimPrefix(field, "q_"), "_vec")
	vectorSize, err := strconv.Atoi(sizeText)
	if err != nil || vectorSize <= 0 {
		return 0, false
	}
	return vectorSize, true
}

func (e *Engine) memoryMessageVectorMappingExists(ctx context.Context, indexName, fieldName string) (bool, error) {
	req := esapi.IndicesGetMappingRequest{
		Index: []string{indexName},
	}
	res, err := req.Do(ctx, e.client)
	defer closeESBody(res)
	if err != nil {
		return false, fmt.Errorf("failed to get memory vector mapping: %w", err)
	}

	if res.StatusCode == http.StatusNotFound {
		return false, nil
	}
	if res.IsError() {
		bodyBytes, _ := io.ReadAll(res.Body)
		reason := extractErrorReason(bodyBytes)
		if reason != "" {
			return false, fmt.Errorf("elasticsearch error getting memory vector mapping %s.%s: %s", indexName, fieldName, reason)
		}
		return false, fmt.Errorf("elasticsearch returned error getting memory vector mapping %s.%s: %s, body: %s", indexName, fieldName, res.Status(), string(bodyBytes))
	}

	var mappings map[string]interface{}
	if err := json.NewDecoder(res.Body).Decode(&mappings); err != nil {
		return false, fmt.Errorf("failed to decode memory vector mapping: %w", err)
	}

	indexMapping, ok := mappings[indexName].(map[string]interface{})
	if !ok {
		return false, nil
	}
	mapping, ok := indexMapping["mappings"].(map[string]interface{})
	if !ok {
		return false, nil
	}
	properties, ok := mapping["properties"].(map[string]interface{})
	if !ok {
		return false, nil
	}
	_, ok = properties[fieldName]
	return ok, nil
}

func (e *Engine) ensureMemoryMessageVectorMapping(ctx context.Context, indexName string, vectorSize int) error {
	if vectorSize <= 0 {
		return fmt.Errorf("memory vector size must be positive, got %d", vectorSize)
	}

	fieldName := memoryMessageVectorField(vectorSize)
	exists, err := e.memoryMessageVectorMappingExists(ctx, indexName, fieldName)
	if err != nil {
		return err
	}
	if exists {
		return nil
	}

	body := map[string]interface{}{
		"properties": map[string]interface{}{
			fieldName: memoryMessageVectorProperty(vectorSize),
		},
	}
	data, err := json.Marshal(body)
	if err != nil {
		return fmt.Errorf("failed to marshal memory vector mapping: %w", err)
	}

	req := esapi.IndicesPutMappingRequest{
		Index: []string{indexName},
		Body:  bytes.NewReader(data),
	}
	res, err := req.Do(ctx, e.client)
	defer closeESBody(res)
	if err != nil {
		return fmt.Errorf("failed to update memory vector mapping: %w", err)
	}

	if res.IsError() {
		bodyBytes, _ := io.ReadAll(res.Body)
		reason := extractErrorReason(bodyBytes)
		if reason != "" {
			return fmt.Errorf("elasticsearch error updating memory vector mapping %s.%s: %s", indexName, fieldName, reason)
		}
		return fmt.Errorf("elasticsearch returned error updating memory vector mapping %s.%s: %s, body: %s", indexName, fieldName, res.Status(), string(bodyBytes))
	}

	return nil
}

func (e *Engine) ensureMemoryMessageVectorMappingsForDocs(ctx context.Context, indexName string, chunks []map[string]interface{}) error {
	seen := map[int]struct{}{}
	for _, chunk := range chunks {
		for field := range chunk {
			vectorSize, ok := parseMemoryMessageVectorSize(field)
			if !ok {
				continue
			}
			if _, ok := seen[vectorSize]; ok {
				continue
			}
			if err := e.ensureMemoryMessageVectorMapping(ctx, indexName, vectorSize); err != nil {
				return err
			}
			seen[vectorSize] = struct{}{}
		}
	}
	return nil
}

func (e *Engine) ensureMemoryMessageSearchVectorMappings(ctx context.Context, indexNames []string, vectorFieldName string, fallbackVectorSize int) error {
	vectorSize, ok := parseMemoryMessageVectorSize(vectorFieldName)
	if !ok {
		vectorSize = fallbackVectorSize
	}
	if vectorSize <= 0 {
		return fmt.Errorf("memory vector size must be positive, got %d", vectorSize)
	}

	for _, indexName := range indexNames {
		if !strings.HasPrefix(indexName, "memory_") {
			continue
		}
		exists, err := e.indexExists(ctx, indexName)
		if err != nil {
			return fmt.Errorf("failed to check memory index existence: %w", err)
		}
		if !exists {
			continue
		}
		if err = e.ensureMemoryMessageVectorMapping(ctx, indexName, vectorSize); err != nil {
			return err
		}
	}
	return nil
}

func getMemoryMessageMapping(vectorSize int) map[string]interface{} {
	vectorField := memoryMessageVectorField(vectorSize)
	return map[string]interface{}{
		"settings": map[string]interface{}{
			"number_of_shards":   1,
			"number_of_replicas": 0,
		},
		"mappings": map[string]interface{}{
			"properties": map[string]interface{}{
				"id": map[string]interface{}{
					"type": "keyword",
				},
				"doc_id": map[string]interface{}{
					"type": "keyword",
				},
				"kb_id": map[string]interface{}{
					"type": "keyword",
				},
				"memory_id": map[string]interface{}{
					"type": "keyword",
				},
				"user_id": map[string]interface{}{
					"type": "keyword",
				},
				"agent_id": map[string]interface{}{
					"type": "keyword",
				},
				"session_id": map[string]interface{}{
					"type": "keyword",
				},
				"message_id": map[string]interface{}{
					"type": "long",
				},
				"source_id": map[string]interface{}{
					"type": "long",
				},
				"message_type_kwd": map[string]interface{}{
					"type": "keyword",
				},
				"status_int": map[string]interface{}{
					"type": "integer",
				},
				"content_ltks": map[string]interface{}{
					"type":     "text",
					"analyzer": "whitespace",
				},
				"tokenized_content_ltks": map[string]interface{}{
					"type":     "text",
					"analyzer": "whitespace",
				},
				"valid_at": map[string]interface{}{
					"type":   "date",
					"format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time||epoch_millis",
				},
				"invalid_at": map[string]interface{}{
					"type":   "date",
					"format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time||epoch_millis",
				},
				"forget_at": map[string]interface{}{
					"type":   "date",
					"format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time||epoch_millis",
				},
				vectorField: memoryMessageVectorProperty(vectorSize),
			},
		},
	}
}

// getDefaultSkillMapping returns the default skill index mapping
func getDefaultSkillMapping() map[string]interface{} {
	return map[string]interface{}{
		"settings": map[string]interface{}{
			"index": map[string]interface{}{
				"number_of_shards":   1,
				"number_of_replicas": 0,
				"refresh_interval":   "1000ms",
			},
		},
		"mappings": map[string]interface{}{
			"dynamic": false,
			"properties": map[string]interface{}{
				"skill_id": map[string]interface{}{
					"type":  "keyword",
					"store": true,
				},
				"name": map[string]interface{}{
					"type":  "text",
					"index": false,
					"store": true,
				},
				"name_tks": map[string]interface{}{
					"type":     "text",
					"analyzer": "whitespace",
					"store":    true,
				},
				"tags": map[string]interface{}{
					"type":  "text",
					"index": false,
					"store": true,
				},
				"tags_tks": map[string]interface{}{
					"type":     "text",
					"analyzer": "whitespace",
					"store":    true,
				},
				"description": map[string]interface{}{
					"type":  "text",
					"index": false,
					"store": true,
				},
				"description_tks": map[string]interface{}{
					"type":     "text",
					"analyzer": "whitespace",
					"store":    true,
				},
				"content": map[string]interface{}{
					"type":  "text",
					"index": false,
					"store": true,
				},
				"content_tks": map[string]interface{}{
					"type":     "text",
					"analyzer": "whitespace",
					"store":    true,
				},
				"q_3072_vec": map[string]interface{}{
					"type":       "dense_vector",
					"dims":       3072,
					"index":      true,
					"similarity": "cosine",
				},
				"q_2560_vec": map[string]interface{}{
					"type":       "dense_vector",
					"dims":       2560,
					"index":      true,
					"similarity": "cosine",
				},
				"q_1536_vec": map[string]interface{}{
					"type":       "dense_vector",
					"dims":       1536,
					"index":      true,
					"similarity": "cosine",
				},
				"q_1024_vec": map[string]interface{}{
					"type":       "dense_vector",
					"dims":       1024,
					"index":      true,
					"similarity": "cosine",
				},
				"q_768_vec": map[string]interface{}{
					"type":       "dense_vector",
					"dims":       768,
					"index":      true,
					"similarity": "cosine",
				},
				"q_512_vec": map[string]interface{}{
					"type":       "dense_vector",
					"dims":       512,
					"index":      true,
					"similarity": "cosine",
				},
				"q_256_vec": map[string]interface{}{
					"type":       "dense_vector",
					"dims":       256,
					"index":      true,
					"similarity": "cosine",
				},
				"version": map[string]interface{}{
					"type":  "keyword",
					"store": true,
				},
				"status": map[string]interface{}{
					"type":  "keyword",
					"store": true,
				},
				"create_time": map[string]interface{}{
					"type":  "long",
					"store": true,
				},
				"update_time": map[string]interface{}{
					"type":  "long",
					"store": true,
				},
			},
		},
	}
}

// convertESResponse converts ES SearchResponse to unified chunks format
// queryStringOperators holds the tokens Lucene's query_string parser only
// recognises as operators in upper case.
var queryStringOperators = map[string]bool{"AND": true, "OR": true, "NOT": true}

// lowerCaseQueryText folds the terms of a query_string expression but keeps its
// upper-case boolean operators. The *_tks/*_ltks fields are whitespace-analyzed
// and so case-sensitive, while Lucene only reads AND/OR/NOT in upper case:
//
//	"病毒 OR 勒索" -> content_ltks:病毒 content_ltks:勒索
//	"病毒 or 勒索" -> content_ltks:病毒 content_ltks:or content_ltks:勒索
//
// That folded "or" is an extra clause every minimum_should_match percentage is
// re-based on — a 94-hit search came back with 32. Quoted text is a phrase, never
// an operator, so it is folded too.
func lowerCaseQueryText(text string) string {
	var b strings.Builder
	b.Grow(len(text))
	inQuote := false
	escaped := false
	for i := 0; i < len(text); {
		c := text[i]
		if escaped {
			b.WriteByte(c)
			escaped = false
			i++
			continue
		}
		switch {
		case c == '\\':
			b.WriteByte(c)
			escaped = true
			i++
		case c == '"':
			inQuote = !inQuote
			b.WriteByte(c)
			i++
		case isASCIILetter(c):
			j := i
			for j < len(text) && isASCIILetter(text[j]) {
				j++
			}
			word := text[i:j]
			if !inQuote && queryStringOperators[word] {
				b.WriteString(word)
			} else {
				b.WriteString(strings.ToLower(word))
			}
			i = j
		default:
			b.WriteByte(c)
			i++
		}
	}
	return b.String()
}

func isASCIILetter(c byte) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

// queryVectorField matches the serialized query vector: 1024 floats that say
// nothing about why a search came back short, so they never reach the log.
var queryVectorField = regexp.MustCompile(`"query_vector":\s*\[[^\]]*\]`)

// elideQueryVector drops that vector from a body before it is logged.
func elideQueryVector(body []byte) string {
	return queryVectorField.ReplaceAllString(string(body), `"query_vector":"<elided>"`)
}

func convertESResponse(esResp *SearchResponse, vectorFieldName string) []map[string]interface{} {
	if esResp == nil || esResp.Hits.Hits == nil {
		return []map[string]interface{}{}
	}

	chunks := make([]map[string]interface{}, len(esResp.Hits.Hits))
	for i, hit := range esResp.Hits.Hits {
		chunks[i] = hit.Source
		if chunks[i] == nil {
			chunks[i] = make(map[string]interface{})
		}
		chunks[i]["_score"] = hit.Score
		chunks[i]["_id"] = hit.ID
		chunks[i]["id"] = hit.ID // shim id to _id, matching Python es_conn.py behavior
		chunks[i]["_index"] = hit.Index
		if len(hit.Highlight) > 0 {
			chunks[i]["highlight"] = hit.Highlight
		}
	}
	return chunks
}

// parseOrderByExpr parses the OrderBy expression into ES sort format
func parseOrderByExpr(orderBy *types.OrderByExpr) []map[string]interface{} {
	if orderBy == nil || len(orderBy.Fields) == 0 {
		return nil
	}

	var result []map[string]interface{}
	for _, field := range orderBy.Fields {
		direction := "asc"
		if field.Type == types.SortDesc {
			direction = "desc"
		}

		// Skip id field (cannot order by text field)
		if field.Field == "id" {
			continue
		}

		// Special handling for page_num_int and top_int
		if field.Field == "page_num_int" || field.Field == "top_int" {
			result = append(result, map[string]interface{}{
				field.Field: map[string]interface{}{
					"order":         direction,
					"unmapped_type": "float",
					"mode":          "avg",
					"numeric_type":  "double",
				},
			})
		} else if strings.HasSuffix(field.Field, "_int") || strings.HasSuffix(field.Field, "_flt") {
			// Fields ending with _int or _flt
			result = append(result, map[string]interface{}{
				field.Field: map[string]interface{}{
					"order":         direction,
					"unmapped_type": "float",
				},
			})
		} else if field.Field == "_score" || field.Field == "score" {
			result = append(result, map[string]interface{}{
				"_score": direction,
			})
		} else {
			// Default: unmapped_type = keyword
			result = append(result, map[string]interface{}{
				field.Field: map[string]interface{}{
					"order":         direction,
					"unmapped_type": "keyword",
				},
			})
		}
	}

	return result
}

// calculateScores calculates _score for chunks
func calculateScores(chunks []map[string]interface{}, scoreColumn, pagerankField string) []map[string]interface{} {
	for i := range chunks {
		score := 0.0
		if scoreVal, ok := chunks[i][scoreColumn]; ok {
			if f, ok := toFloat64(scoreVal); ok {
				score += f
			}
		}
		if pagerankField != "" {
			if prVal, ok := chunks[i][pagerankField]; ok {
				if f, ok := toFloat64(prVal); ok {
					score += f
				}
			}
		}
		chunks[i]["_score"] = score
	}
	return chunks
}

// toFloat64 converts a value to float64
func toFloat64(v interface{}) (float64, bool) {
	switch val := v.(type) {
	case float64:
		return val, true
	case float32:
		return float64(val), true
	case int:
		return float64(val), true
	case int64:
		return float64(val), true
	}
	return 0, false
}

// sortByScore sorts chunks by _score descending and limits
func sortByScore(chunks []map[string]interface{}, limit int) []map[string]interface{} {
	if len(chunks) == 0 {
		return chunks
	}

	// Stable: the reference does not re-sort at all, so ties keep the order ES
	// returned them in.
	sort.SliceStable(chunks, func(i, j int) bool {
		scoreI := getChunkScore(chunks[i])
		scoreJ := getChunkScore(chunks[j])
		return scoreI > scoreJ
	})

	// Limit
	if len(chunks) > limit && limit > 0 {
		chunks = chunks[:limit]
	}

	return chunks
}

// getChunkScore extracts the score from a chunk
func getChunkScore(chunk map[string]interface{}) float64 {
	if v, ok := chunk["_score"].(float64); ok {
		return v
	}
	if v, ok := chunk["SCORE"].(float64); ok {
		return v
	}
	if v, ok := chunk["SIMILARITY"].(float64); ok {
		return v
	}
	return 0.0
}