fix: support contextual keywords as field names (#53968)

Fields named `iso` or `interval` can be created, but filters such as
`iso > 1` fail because the lexer emits a keyword token where the parser
expects an identifier.

Accept 20 contextual keyword families through a shared `fieldName` rule
in expression field positions while preserving their function, option,
and timestamp syntax. Update the visitor and regenerate the parser with
ANTLR 4.13.2.

Reject `LIKE`, `AND`, `OR`, `NOT`, and `IN` as field names in every
casing, and retain the existing case-insensitive `NULL` policy. Validate
struct-array parent names on both Create and Add paths, alongside child
names. Classify `ErrFieldInvalidName` (1701) as `InputError` at its
definition so ordinary names, reserved names, and RootCoord's
add-struct-field validator report the same classification. Remove the
redundant Proxy error markers and validate each struct parent name once
while preserving the existing validation order, codes, reasons,
identity, and non-retryability.

Compatibility: mixed-case names such as `And`, `In`, and `Like`
previously lexed as ordinary identifiers and could be created and
filtered. New Create/Add requests reject these names. Existing
collections are not revalidated, but backup restoration or cross-cluster
schema recreation containing these names will require renaming the
affected fields. This tightening is intentional; contextual keyword
field names remain supported.

Regression coverage includes contextual keywords and their dedicated
syntax, field identity/casing, SLL/LL parsing, core keyword rejection,
ordinary and struct-array Create/Add paths, reserved field names, and
InputError status/metric round trips. RootCoord's name validator now
also has classification and status round-trip coverage.

Validation:

- Current review follow-up: all tests in `pkg/util/merr`,
`pkg/util/requestutil`, and `pkg/common` passed with `-tags dynamic,test
-gcflags='all=-N -l' -count=1`; `git diff --check` passed.
- Current focused Proxy/RootCoord tests were blocked before execution by
older local native libraries missing required APIs. The development host
was inaccessible under the current network restrictions; native CI
validation is pending.
- Before this follow-up, the unchanged parser/rewriter implementation
passed 1,182 tests/subtests, focused Proxy regressions passed 248
tests/subtests with race detection and coverage, and
`merr`/`requestutil` guards passed 143 tests/subtests with race
detection and coverage.
- Generated parser output was reproduced with ANTLR 4.13.2.
- A previous full `make -o build-cpp-with-unittest test-go` attempt
timed out in `TestProxy/create_collection` while waiting for streaming
assignments and metadata-cache initialization. Later groups were not
reached; no fresh C++ build was performed.

issue: #53925

Fixes #53925

---------

Signed-off-by: xiaofanluan <[email protected]>
Co-authored-by: xiaofanluan <[email protected]>
This commit is contained in:
James
2026-10-11 17:54:18 +08:00
committed by GitHub
co-authored by xiaofanluan
parent f0f197ec04
commit f7727377fd
15 changed files with 2255 additions and 661 deletions
+53 -19
View File
@@ -1,19 +1,19 @@
grammar Plan;
expr:
Identifier (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? op2=(LT | LE | GT | GE | EQ | NE) ISO compare_string=StringLiteral # TimestamptzCompareForward
| ISO compare_string=StringLiteral op2=(LT | LE | GT | GE | EQ | NE) Identifier (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? # TimestamptzCompareReverse
fieldName (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? op2=(LT | LE | GT | GE | EQ | NE) ISO compare_string=StringLiteral # TimestamptzCompareForward
| ISO compare_string=StringLiteral op2=(LT | LE | GT | GE | EQ | NE) fieldName (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? # TimestamptzCompareReverse
| IntegerConstant # Integer
| FloatingConstant # Floating
| BooleanConstant # Boolean
| StringLiteral # String
| RawStringLiteral # RawString
| (Identifier|Meta) # Identifier
| (fieldName|Meta) # Identifier
| JSONIdentifier # JSONIdentifier
| StructFieldIdentifier # StructField
| StructIndexFieldIdentifier # StructIndexField
| StructSubFieldIdentifier # StructSubField
| LBRACE Identifier RBRACE # TemplateVariable
| LBRACE fieldName RBRACE # TemplateVariable
| '(' expr ')' # Parens
| '[' expr (',' expr)* ','? ']' # Array
| EmptyArray # EmptyArray
@@ -21,13 +21,13 @@ expr:
| expr LIKE expr # Like
| expr REGEXMATCH expr # RegexMatch
| expr REGEXNOTMATCH expr # RegexNotMatch
| TEXTMATCH'('Identifier',' expr (',' textMatchOption)? ')' # TextMatch
| TEXTMATCHFUZZY'('Identifier',' expr ',' Identifier ASSIGN IntegerConstant ')' # TextMatchFuzzy
| PHRASEMATCH'('Identifier',' expr (',' expr)? ')' # PhraseMatch
| TEXTMATCH'('fieldName',' expr (',' textMatchOption)? ')' # TextMatch
| TEXTMATCHFUZZY'('fieldName',' expr ',' Identifier ASSIGN IntegerConstant ')' # TextMatchFuzzy
| PHRASEMATCH'('fieldName',' expr (',' expr)? ')' # PhraseMatch
| RANDOMSAMPLE'(' expr ')' # RandomSample
| ElementFilter'('Identifier',' expr')' # ElementFilter
| op=(MATCH_ALL | MATCH_ANY) '(' Identifier ',' expr ')' # MatchSimple
| op=(MATCH_LEAST | MATCH_MOST | MATCH_EXACT) '(' Identifier ',' expr ',' THRESHOLD ASSIGN IntegerConstant ')' # MatchThreshold
| ElementFilter'('fieldName',' expr')' # ElementFilter
| op=(MATCH_ALL | MATCH_ANY) '(' fieldName ',' expr ')' # MatchSimple
| op=(MATCH_LEAST | MATCH_MOST | MATCH_EXACT) '(' fieldName ',' expr ',' THRESHOLD ASSIGN IntegerConstant ')' # MatchThreshold
| expr POW expr # Power
| op = (ADD | SUB | BNOT | NOT) expr # Unary
// | '(' typeName ')' expr # Cast
@@ -38,14 +38,14 @@ expr:
| (JSONContains | ArrayContains)'('expr',' expr')' # JSONContains
| (JSONContainsAll | ArrayContainsAll)'('expr',' expr')' # JSONContainsAll
| (JSONContainsAny | ArrayContainsAny)'('expr',' expr')' # JSONContainsAny
| op=(STEuqals | STTouches | STOverlaps | STCrosses | STContains | STIntersects | STWithin) '(' Identifier ',' expr ')' # SpatialBinary
| STDWithin'('Identifier',' expr',' expr')' # STDWithin
| STIsValid'('Identifier')' # STIsValid
| ArrayLength'('(Identifier | JSONIdentifier | StructFieldIdentifier | StructSubFieldIdentifier)')' # ArrayLength
| function=Identifier '(' field=(Identifier|Meta|JSONIdentifier|StructFieldIdentifier|StructIndexFieldIdentifier|StructSubFieldIdentifier) ',' expr ',' option=Identifier ASSIGN kind=Identifier ')' # MembershipMatchWithOption
| op=(STEuqals | STTouches | STOverlaps | STCrosses | STContains | STIntersects | STWithin) '(' fieldName ',' expr ')' # SpatialBinary
| STDWithin'('fieldName',' expr',' expr')' # STDWithin
| STIsValid'('fieldName')' # STIsValid
| ArrayLength'('(fieldName | JSONIdentifier | StructFieldIdentifier | StructSubFieldIdentifier)')' # ArrayLength
| function=Identifier '(' field=fieldReference ',' expr ',' option=Identifier ASSIGN kind=Identifier ')' # MembershipMatchWithOption
| Identifier '(' ( expr (',' expr )* ','? )? ')' # Call
| expr op1 = (LT | LE) (Identifier | JSONIdentifier | StructSubFieldIdentifier | StructIndexFieldIdentifier) op2 = (LT | LE) expr # Range
| expr op1 = (GT | GE) (Identifier | JSONIdentifier | StructSubFieldIdentifier | StructIndexFieldIdentifier) op2 = (GT | GE) expr # ReverseRange
| expr op1 = (LT | LE) (fieldName | JSONIdentifier | StructSubFieldIdentifier | StructIndexFieldIdentifier) op2 = (LT | LE) expr # Range
| expr op1 = (GT | GE) (fieldName | JSONIdentifier | StructSubFieldIdentifier | StructIndexFieldIdentifier) op2 = (GT | GE) expr # ReverseRange
| expr op = (LT | LE | GT | GE) expr # Relational
| expr op = (EQ | NE) expr # Equality
| expr BAND expr # BitAnd
@@ -53,8 +53,42 @@ expr:
| expr BOR expr # BitOr
| expr AND expr # LogicalAnd
| expr OR expr # LogicalOr
| (Identifier | JSONIdentifier) ISNULL # IsNull
| (Identifier | JSONIdentifier) ISNOTNULL # IsNotNull;
| (fieldName | JSONIdentifier) ISNULL # IsNull
| (fieldName | JSONIdentifier) ISNOTNULL # IsNotNull;
// These keywords retain their specialized syntax while also naming fields.
// Operators and constants remain reserved, and generic function names and
// option names continue to use Identifier rather than this rule.
fieldName:
Identifier
| ISO
| INTERVAL
| MINIMUM_SHOULD_MATCH
| THRESHOLD
| TEXTMATCHFUZZY
| MATCH_ALL
| MATCH_ANY
| MATCH_LEAST
| MATCH_MOST
| MATCH_EXACT
| ElementFilter
| STEuqals
| STTouches
| STOverlaps
| STCrosses
| STContains
| STIntersects
| STWithin
| STDWithin
| STIsValid;
fieldReference:
fieldName
| Meta
| JSONIdentifier
| StructFieldIdentifier
| StructIndexFieldIdentifier
| StructSubFieldIdentifier;
textMatchOption:
MINIMUM_SHOULD_MATCH ASSIGN IntegerConstant;
@@ -0,0 +1,469 @@
package planparserv2
import (
"fmt"
"strings"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/pkg/v3/proto/planpb"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
// These words have a dedicated token in the expression grammar, but may also
// identify fields. Keep both lexer spellings in the matrix: accepting only the
// mixed-case Identifier spelling would leave the original regression intact.
var contextualKeywordNames = []string{
"text_match_fuzzy", "element_filter",
"match_all", "match_any", "match_least", "match_most", "match_exact",
"st_equals", "st_touches", "st_overlaps", "st_crosses", "st_contains",
"st_intersects", "st_within", "st_dwithin", "st_isvalid",
"iso", "interval", "minimum_should_match", "threshold",
}
const contextualKeywordFieldID int64 = 2000
func contextualKeywordSpellings(name string) []string {
return []string{name, strings.ToUpper(name), strings.ToUpper(name[:1]) + name[1:]}
}
func contextualKeywordSchema(t *testing.T, name string, dataType schemapb.DataType, dynamic bool) *typeutil.SchemaHelper {
t.Helper()
schema := newTestSchema(dynamic)
field := &schemapb.FieldSchema{
FieldID: contextualKeywordFieldID, Name: name, DataType: dataType, Nullable: true,
}
if dataType == schemapb.DataType_Array {
field.ElementType = schemapb.DataType_Int64
}
schema.Fields = append(schema.Fields, field)
enableMatch(schema)
helper, err := typeutil.CreateSchemaHelper(schema)
require.NoError(t, err)
return helper
}
func requireContextualKeywordColumn(t *testing.T, column *planpb.ColumnInfo, name string) {
t.Helper()
require.NotNil(t, column, name)
require.Equal(t, contextualKeywordFieldID, column.GetFieldId(), name)
require.Empty(t, column.GetNestedPath(), name)
}
func TestContextualKeywordsScalarFields(t *testing.T) {
for _, dynamic := range []bool{false, true} {
for _, word := range contextualKeywordNames {
for _, name := range contextualKeywordSpellings(word) {
t.Run(fmt.Sprintf("dynamic_%t/%s", dynamic, name), func(t *testing.T) {
helper := contextualKeywordSchema(t, name, schemapb.DataType_Int64, dynamic)
parse := func(format string) *planpb.Expr {
t.Helper()
expr, err := ParseExpr(helper, fmt.Sprintf(format, name), nil)
require.NoError(t, err)
require.NotNil(t, expr)
return expr
}
for _, format := range []string{"%s == 400", "400 == %s"} {
unary := parse(format).GetUnaryRangeExpr()
require.NotNil(t, unary)
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
require.Equal(t, planpb.OpType_Equal, unary.GetOp())
require.Equal(t, int64(400), unary.GetValue().GetInt64Val())
}
term := parse("%s in [1, 400]").GetTermExpr()
require.NotNil(t, term)
requireContextualKeywordColumn(t, term.GetColumnInfo(), name)
require.Len(t, term.GetValues(), 2)
require.Equal(t, int64(400), term.GetValues()[1].GetInt64Val())
arith := parse("%s + 1 > 400").GetBinaryArithOpEvalRangeExpr()
require.NotNil(t, arith)
requireContextualKeywordColumn(t, arith.GetColumnInfo(), name)
require.Equal(t, planpb.ArithOpType_Add, arith.GetArithOp())
require.Equal(t, int64(1), arith.GetRightOperand().GetInt64Val())
require.Equal(t, planpb.OpType_GreaterThan, arith.GetOp())
require.Equal(t, int64(400), arith.GetValue().GetInt64Val())
for _, format := range []string{"5 < %s <= 400", "400 >= %s > 5"} {
rangeExpr := parse(format).GetBinaryRangeExpr()
require.NotNil(t, rangeExpr)
requireContextualKeywordColumn(t, rangeExpr.GetColumnInfo(), name)
require.Equal(t, int64(5), rangeExpr.GetLowerValue().GetInt64Val())
require.Equal(t, int64(400), rangeExpr.GetUpperValue().GetInt64Val())
require.False(t, rangeExpr.GetLowerInclusive())
require.True(t, rangeExpr.GetUpperInclusive())
}
for _, tc := range []struct {
format string
op planpb.NullExpr_NullOp
}{
{"%s is null", planpb.NullExpr_IsNull},
{"%s is not null", planpb.NullExpr_IsNotNull},
} {
nullExpr := parse(tc.format).GetNullExpr()
require.NotNil(t, nullExpr)
requireContextualKeywordColumn(t, nullExpr.GetColumnInfo(), name)
require.Equal(t, tc.op, nullExpr.GetOp())
require.True(t, nullExpr.GetColumnInfo().GetNullable())
}
// The same token must be usable in a template name, independently
// of its use as the column on the left.
expr, err := ParseExpr(helper, fmt.Sprintf("%s == {%s}", name, name), map[string]*schemapb.TemplateValue{
name: generateTemplateValue(schemapb.DataType_Int64, int64(400)),
})
require.NoError(t, err)
unary := expr.GetUnaryRangeExpr()
require.NotNil(t, unary)
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
require.Equal(t, int64(400), unary.GetValue().GetInt64Val())
call := parse("custom_predicate(%s, 1)").GetCallExpr()
require.NotNil(t, call)
require.Equal(t, "custom_predicate", call.GetFunctionName())
require.Len(t, call.GetFunctionParameters(), 2)
requireContextualKeywordColumn(t, call.GetFunctionParameters()[0].GetColumnExpr().GetInfo(), name)
})
}
}
}
}
func TestContextualKeywordsCaseSensitiveLookup(t *testing.T) {
for _, word := range contextualKeywordNames {
t.Run(word, func(t *testing.T) {
upper := strings.ToUpper(word)
helper := contextualKeywordSchema(t, word, schemapb.DataType_Int64, true)
expr, err := ParseExpr(helper, word+" == 400", nil)
require.NoError(t, err)
requireContextualKeywordColumn(t, expr.GetUnaryRangeExpr().GetColumnInfo(), word)
// A differently-cased name stays a different dynamic JSON key. It
// must not resolve to the lower-case declaration by normalization.
expr, err = ParseExpr(helper, upper+" == 400", nil)
require.NoError(t, err)
column := expr.GetUnaryRangeExpr().GetColumnInfo()
require.Equal(t, int64(dynamicFieldID), column.GetFieldId())
require.Equal(t, []string{upper}, column.GetNestedPath())
helper = contextualKeywordSchema(t, word, schemapb.DataType_Int64, false)
_, err = ParseExpr(helper, upper+" == 400", nil)
require.Error(t, err)
schema := newTestSchema(true)
schema.Fields = append(schema.Fields,
&schemapb.FieldSchema{FieldID: contextualKeywordFieldID, Name: word, DataType: schemapb.DataType_Int64},
&schemapb.FieldSchema{FieldID: contextualKeywordFieldID + 1, Name: upper, DataType: schemapb.DataType_Int64},
)
helper, err = typeutil.CreateSchemaHelper(schema)
require.NoError(t, err)
expr, err = ParseExpr(helper, upper+" == 400", nil)
require.NoError(t, err)
require.Equal(t, contextualKeywordFieldID+1, expr.GetUnaryRangeExpr().GetColumnInfo().GetFieldId())
require.Empty(t, expr.GetUnaryRangeExpr().GetColumnInfo().GetNestedPath())
})
}
}
func TestContextualKeywordsNestedFields(t *testing.T) {
for _, word := range contextualKeywordNames {
for _, name := range contextualKeywordSpellings(word) {
t.Run(name, func(t *testing.T) {
helper := contextualKeywordSchema(t, name, schemapb.DataType_JSON, true)
expr, err := ParseExpr(helper, fmt.Sprintf(`%s["value"] == 400`, name), nil)
require.NoError(t, err)
column := expr.GetUnaryRangeExpr().GetColumnInfo()
require.Equal(t, contextualKeywordFieldID, column.GetFieldId())
require.Equal(t, []string{"value"}, column.GetNestedPath())
helper = contextualKeywordSchema(t, name, schemapb.DataType_Array, true)
expr, err = ParseExpr(helper, fmt.Sprintf("%s[0] == 400", name), nil)
require.NoError(t, err)
column = expr.GetUnaryRangeExpr().GetColumnInfo()
require.Equal(t, contextualKeywordFieldID, column.GetFieldId())
require.Equal(t, []string{"0"}, column.GetNestedPath())
// Parent and sub-field names both use contextual tokens, while
// element predicates retain their dedicated scope and plan node.
schema := newTestSchema(true)
schema.StructArrayFields = append(schema.StructArrayFields, &schemapb.StructArrayFieldSchema{
FieldID: contextualKeywordFieldID, Name: name, Nullable: true,
Fields: []*schemapb.FieldSchema{{
FieldID: contextualKeywordFieldID + 1, Name: fmt.Sprintf("%s[%s]", name, name),
DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int64,
}},
})
helper, err = typeutil.CreateSchemaHelper(schema)
require.NoError(t, err)
expr, err = ParseExpr(helper, fmt.Sprintf("element_filter(%s, $[%s] > 5)", name, name), nil)
require.NoError(t, err)
element := expr.GetElementFilterExpr()
require.NotNil(t, element)
require.Equal(t, name, element.GetStructName())
require.Equal(t, contextualKeywordFieldID+1, element.GetElementExpr().GetUnaryRangeExpr().GetColumnInfo().GetFieldId())
if word == "element_filter" {
expr, err = ParseExpr(helper, fmt.Sprintf("ELEMENT_FILTER(%s, $[%s] > 5)", name, name), nil)
require.NoError(t, err)
require.NotNil(t, expr.GetElementFilterExpr())
require.Equal(t, name, expr.GetElementFilterExpr().GetStructName())
}
expr, err = ParseExpr(helper, fmt.Sprintf("%s is null", name), nil)
require.NoError(t, err)
require.NotNil(t, expr.GetNullExpr())
require.Equal(t, contextualKeywordFieldID+1, expr.GetNullExpr().GetColumnInfo().GetFieldId())
ret := handleExpr(helper, fmt.Sprintf("array_length(%s[%s])", name, name))
require.NoError(t, getError(ret))
length := getExpr(ret).expr.GetBinaryArithExpr()
require.NotNil(t, length)
require.Equal(t, planpb.ArithOpType_ArrayLength, length.GetOp())
require.Equal(t, contextualKeywordFieldID+1, length.GetLeft().GetColumnExpr().GetInfo().GetFieldId())
})
}
}
}
func TestContextualKeywordsTextMatchPlans(t *testing.T) {
for _, name := range []string{"text_match_fuzzy", "TEXT_MATCH_FUZZY"} {
t.Run(name, func(t *testing.T) {
helper := contextualKeywordSchema(t, name, schemapb.DataType_VarChar, true)
expr, err := ParseExpr(helper, fmt.Sprintf(`%s(%s, "hello", max_edit_distance=1)`, name, name), nil)
require.NoError(t, err)
require.Nil(t, expr.GetCallExpr())
unary := expr.GetUnaryRangeExpr()
require.NotNil(t, unary)
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
require.Equal(t, planpb.OpType_TextMatchFuzzy, unary.GetOp())
require.Equal(t, "hello", unary.GetValue().GetStringVal())
require.Len(t, unary.GetExtraValues(), 1)
require.Equal(t, int64(1), unary.GetExtraValues()[0].GetInt64Val())
expr, err = ParseExpr(helper, fmt.Sprintf(`starts_with(%s, "hello")`, name), nil)
require.NoError(t, err)
require.NotNil(t, expr.GetCallExpr())
require.Equal(t, "starts_with", expr.GetCallExpr().GetFunctionName())
requireContextualKeywordColumn(t, expr.GetCallExpr().GetFunctionParameters()[0].GetColumnExpr().GetInfo(), name)
})
}
for _, name := range []string{"minimum_should_match", "MINIMUM_SHOULD_MATCH"} {
helper := contextualKeywordSchema(t, name, schemapb.DataType_VarChar, true)
expr, err := ParseExpr(helper, fmt.Sprintf(`text_match(%s, "hello", %s=2)`, name, name), nil)
require.NoError(t, err)
unary := expr.GetUnaryRangeExpr()
require.NotNil(t, unary)
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
require.Equal(t, planpb.OpType_TextMatch, unary.GetOp())
require.Len(t, unary.GetExtraValues(), 1)
require.Equal(t, int64(2), unary.GetExtraValues()[0].GetInt64Val())
}
}
func TestContextualKeywordsSpatialPlans(t *testing.T) {
for _, tc := range []struct {
name string
op planpb.GISFunctionFilterExpr_GISOp
}{
{"st_equals", planpb.GISFunctionFilterExpr_Equals},
{"st_touches", planpb.GISFunctionFilterExpr_Touches},
{"st_overlaps", planpb.GISFunctionFilterExpr_Overlaps},
{"st_crosses", planpb.GISFunctionFilterExpr_Crosses},
{"st_contains", planpb.GISFunctionFilterExpr_Contains},
{"st_intersects", planpb.GISFunctionFilterExpr_Intersects},
{"st_within", planpb.GISFunctionFilterExpr_Within},
{"st_dwithin", planpb.GISFunctionFilterExpr_DWithin},
{"st_isvalid", planpb.GISFunctionFilterExpr_STIsValid},
} {
for _, name := range []string{tc.name, strings.ToUpper(tc.name)} {
t.Run(name, func(t *testing.T) {
helper := contextualKeywordSchema(t, name, schemapb.DataType_Geometry, true)
query := fmt.Sprintf(`%s(%s, "POINT(0 0)")`, name, name)
switch tc.op {
case planpb.GISFunctionFilterExpr_DWithin:
query = fmt.Sprintf(`%s(%s, "POINT(0 0)", 5)`, name, name)
case planpb.GISFunctionFilterExpr_STIsValid:
query = fmt.Sprintf("%s(%s)", name, name)
}
expr, err := ParseExpr(helper, query, nil)
require.NoError(t, err)
require.Nil(t, expr.GetCallExpr())
spatial := expr.GetGisfunctionFilterExpr()
require.NotNil(t, spatial)
requireContextualKeywordColumn(t, spatial.GetColumnInfo(), name)
require.Equal(t, tc.op, spatial.GetOp())
if tc.op != planpb.GISFunctionFilterExpr_STIsValid {
require.Equal(t, "POINT(0 0)", spatial.GetWktString())
}
if tc.op == planpb.GISFunctionFilterExpr_DWithin {
require.Equal(t, float64(5), spatial.GetDistance())
}
})
}
}
}
func TestContextualKeywordsMatchPlans(t *testing.T) {
for _, tc := range []struct {
name string
matchType planpb.MatchType
count int64
}{
{"match_all", planpb.MatchType_MatchAll, 0},
{"match_any", planpb.MatchType_MatchAny, 0},
{"match_least", planpb.MatchType_MatchLeast, 2},
{"match_most", planpb.MatchType_MatchMost, 2},
{"match_exact", planpb.MatchType_MatchExact, 2},
} {
for _, name := range []string{tc.name, strings.ToUpper(tc.name)} {
t.Run(name, func(t *testing.T) {
schema := newTestSchema(true)
schema.StructArrayFields = append(schema.StructArrayFields, &schemapb.StructArrayFieldSchema{
FieldID: contextualKeywordFieldID, Name: name,
Fields: []*schemapb.FieldSchema{{
FieldID: contextualKeywordFieldID + 1, Name: name + "[threshold]",
DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int64,
}},
})
helper, err := typeutil.CreateSchemaHelper(schema)
require.NoError(t, err)
query := fmt.Sprintf("%s(%s, $[threshold] > 5)", name, name)
if tc.count != 0 {
option := "threshold"
if name == strings.ToUpper(name) {
option = "THRESHOLD"
}
query = fmt.Sprintf("%s(%s, $[threshold] > 5, %s=2)", name, name, option)
}
expr, err := ParseExpr(helper, query, nil)
require.NoError(t, err)
require.Nil(t, expr.GetCallExpr())
match := expr.GetMatchExpr()
require.NotNil(t, match)
require.Equal(t, name, match.GetStructName())
require.Equal(t, tc.matchType, match.GetMatchType())
require.Equal(t, tc.count, match.GetCount())
require.Equal(t, contextualKeywordFieldID+1, match.GetPredicate().GetUnaryRangeExpr().GetColumnInfo().GetFieldId())
})
}
}
}
func TestContextualKeywordsTimePlans(t *testing.T) {
const date = "'2025-01-01T00:00:00Z'"
wantMicros := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMicro()
for _, word := range []string{"iso", "interval", "minimum_should_match", "threshold"} {
for _, name := range []string{word, strings.ToUpper(word)} {
t.Run(name, func(t *testing.T) {
helper := contextualKeywordSchema(t, name, schemapb.DataType_Timestamptz, true)
for _, query := range []string{name + " > iso " + date, "ISO " + date + " < " + name} {
expr, err := ParseExpr(helper, query, nil)
require.NoError(t, err, query)
unary := expr.GetUnaryRangeExpr()
require.NotNil(t, unary, query)
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
require.Equal(t, planpb.OpType_GreaterThan, unary.GetOp())
require.Equal(t, wantMicros, unary.GetValue().GetInt64Val())
}
for _, tc := range []struct {
query string
arith planpb.ArithOpType
op planpb.OpType
span *planpb.Interval
}{
{name + " + interval 'P1D' > ISO " + date, planpb.ArithOpType_Add, planpb.OpType_GreaterThan, &planpb.Interval{Days: 1}},
{"iso " + date + " >= " + name + " - INTERVAL 'PT2H'", planpb.ArithOpType_Sub, planpb.OpType_LessEqual, &planpb.Interval{Hours: 2}},
} {
expr, err := ParseExpr(helper, tc.query, nil)
require.NoError(t, err, tc.query)
arith := expr.GetTimestamptzArithCompareExpr()
require.NotNil(t, arith)
requireContextualKeywordColumn(t, arith.GetTimestamptzColumn(), name)
require.Equal(t, tc.arith, arith.GetArithOp())
require.Equal(t, tc.op, arith.GetCompareOp())
require.Equal(t, wantMicros, arith.GetCompareValue().GetInt64Val())
require.Equal(t, tc.span.GetDays(), arith.GetInterval().GetDays())
require.Equal(t, tc.span.GetHours(), arith.GetInterval().GetHours())
}
})
}
}
}
func TestContextualKeywordsMalformedCalls(t *testing.T) {
schema := newTestSchema(true)
enableMatch(schema)
helper, err := typeutil.CreateSchemaHelper(schema)
require.NoError(t, err)
for _, word := range contextualKeywordNames {
for _, name := range []string{word, strings.ToUpper(word)} {
// A keyword token followed by parentheses must never fall through
// to the generic Call visitor and evade its dedicated validation.
_, err := ParseExpr(helper, name+"()", nil)
require.Error(t, err, name)
}
}
for _, query := range []string{
`text_match_fuzzy(VarCharField, "hello", typo=1)`,
`text_match_fuzzy(VarCharField, "hello", max_edit_distance=3)`,
`text_match(VarCharField, "hello", threshold=1)`,
`match_least(struct_array, $[sub_int] > 0, count=1)`,
`match_least(struct_array, $[sub_int] > 0, threshold=0)`,
`match_all(struct_array, $[sub_int] > 0, threshold=1)`,
`element_filter(struct_array, $[missing] > 0)`,
`$[threshold] > 0`,
`st_isvalid(GeometryField, "POINT(0 0)")`,
`st_dwithin(GeometryField, "POINT(0 0)")`,
`st_equals(Int64Field, "POINT(0 0)")`,
`TimestamptzField > timezone '2025-01-01T00:00:00Z'`,
`timezone '2025-01-01T00:00:00Z' < TimestamptzField`,
`TimestamptzField + duration 'P1D' > ISO '2025-01-01T00:00:00Z'`,
`TimestamptzField > ISO 'invalid-date'`,
`ISO 'invalid-date' < TimestamptzField`,
`TimestamptzField + INTERVAL '1D' > ISO '2025-01-01T00:00:00Z'`,
`ISO '2025-01-01T00:00:00Z' < TimestamptzField + INTERVAL '1D'`,
} {
_, err := ParseExpr(helper, query, nil)
require.Error(t, err, query)
}
}
func TestContextualKeywordsTwoStageMatchesLL(t *testing.T) {
corpus := []string{
`text_match_fuzzy(text_match_fuzzy, "hello", max_edit_distance=1)`,
`text_match(minimum_should_match, "hello", minimum_should_match=2)`,
`st_equals(st_equals, "POINT(0 0)")`,
`ST_DWITHIN(ST_DWITHIN, "POINT(0 0)", 5)`,
`match_least(match_least, $[threshold] > 0, threshold=2)`,
`element_filter(element_filter, $[iso] > 0)`,
`iso > iso '2025-01-01T00:00:00Z'`,
`ISO '2025-01-01T00:00:00Z' < ISO`,
`interval + interval 'P1D' >= iso '2025-01-01T00:00:00Z'`,
`ISO '2025-01-01T00:00:00Z' <= INTERVAL - INTERVAL 'PT2H'`,
`text_match_fuzzy(VarCharField)`,
`match_least(struct_array, $[threshold] > 0, count=2)`,
`iso > timezone '2025-01-01T00:00:00Z'`,
}
for _, word := range contextualKeywordNames {
for _, name := range []string{word, strings.ToUpper(word)} {
corpus = append(corpus,
name+" == 400", "5 < "+name+" <= 400", "400 >= "+name+" > 5",
name+" is null", name+" == {"+name+"}", "custom_predicate("+name+", 1)",
)
}
}
for _, query := range corpus {
llTree, llErr := parseTree(query, false)
tsTree, tsErr := parseTree(query, true)
require.Equal(t, llErr == nil, tsErr == nil, "%s: LL=%v, two-stage=%v", query, llErr, tsErr)
if llErr == nil && tsErr == nil {
require.Equal(t, llTree, tsTree, query)
}
}
}
File diff suppressed because one or more lines are too long
@@ -219,6 +219,14 @@ func (v *BasePlanVisitor) VisitPower(ctx *PowerContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitFieldName(ctx *FieldNameContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitFieldReference(ctx *FieldReferenceContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitTextMatchOption(ctx *TextMatchOptionContext) interface{} {
return v.VisitChildren(ctx)
}
File diff suppressed because it is too large Load Diff
@@ -166,6 +166,12 @@ type PlanVisitor interface {
// Visit a parse tree produced by PlanParser#Power.
VisitPower(ctx *PowerContext) interface{}
// Visit a parse tree produced by PlanParser#fieldName.
VisitFieldName(ctx *FieldNameContext) interface{}
// Visit a parse tree produced by PlanParser#fieldReference.
VisitFieldReference(ctx *FieldReferenceContext) interface{}
// Visit a parse tree produced by PlanParser#textMatchOption.
VisitTextMatchOption(ctx *TextMatchOptionContext) interface{}
}
+32 -31
View File
@@ -966,7 +966,7 @@ func (v *ParserVisitor) parseTextMatchOperand(identifier string, queryExpr parse
}
func (v *ParserVisitor) VisitTextMatch(ctx *parser.TextMatchContext) interface{} {
identifier := ctx.Identifier().GetText()
identifier := ctx.FieldName().GetText()
columnInfo, value, placeholder, isTemplate, err := v.parseTextMatchOperand(
identifier, ctx.Expr(), "text match", "text_match query")
if err != nil {
@@ -1005,7 +1005,7 @@ func (v *ParserVisitor) VisitTextMatch(ctx *parser.TextMatchContext) interface{}
}
func (v *ParserVisitor) VisitTextMatchFuzzy(ctx *parser.TextMatchFuzzyContext) interface{} {
identifier := ctx.Identifier(0).GetText()
identifier := ctx.FieldName().GetText()
columnInfo, value, placeholder, isTemplate, err := v.parseTextMatchOperand(
identifier, ctx.Expr(), "text match fuzzy", "text_match_fuzzy query")
if err != nil {
@@ -1015,7 +1015,7 @@ func (v *ParserVisitor) VisitTextMatchFuzzy(ctx *parser.TextMatchFuzzyContext) i
// The option name is a soft keyword (a plain identifier) so that a scalar
// field literally named "max_edit_distance" is still usable elsewhere in a
// filter; only accept the expected option name here.
optionName := ctx.Identifier(1).GetText()
optionName := ctx.Identifier().GetText()
if !strings.EqualFold(optionName, "max_edit_distance") {
return merr.WrapErrParameterInvalidMsg(
"invalid option %q for text_match_fuzzy, expected max_edit_distance", optionName)
@@ -1069,7 +1069,7 @@ func (v *ParserVisitor) VisitTextMatchOption(ctx *parser.TextMatchOptionContext)
}
func (v *ParserVisitor) VisitPhraseMatch(ctx *parser.PhraseMatchContext) interface{} {
identifier := ctx.Identifier().GetText()
identifier := ctx.FieldName().GetText()
columnInfo, value, placeholder, isTemplate, err := v.parseTextMatchOperand(
identifier, ctx.Expr(0), "phrase match", "phrase_match query")
if err != nil {
@@ -1395,7 +1395,7 @@ func (v *ParserVisitor) getColumnInfoFromStructIndexField(identifier string) (*p
}, nil
}
func (v *ParserVisitor) getChildColumnInfo(identifier, child, structSubField, structIndexField antlr.TerminalNode) (*planpb.ColumnInfo, error) {
func (v *ParserVisitor) getChildColumnInfo(identifier parser.IFieldNameContext, child, structSubField, structIndexField antlr.TerminalNode) (*planpb.ColumnInfo, error) {
if identifier != nil {
childExpr, err := v.translateIdentifier(identifier.GetText())
if err != nil {
@@ -1440,7 +1440,7 @@ func (v *ParserVisitor) getStructArrayParentColumnInfo(fieldName string) (*planp
}, true, nil
}
func (v *ParserVisitor) getNullExprColumnInfo(identifier, child antlr.TerminalNode) (*planpb.ColumnInfo, error) {
func (v *ParserVisitor) getNullExprColumnInfo(identifier parser.IFieldNameContext, child antlr.TerminalNode) (*planpb.ColumnInfo, error) {
if identifier != nil {
// try struct first
if columnInfo, ok, err := v.getStructArrayParentColumnInfo(identifier.GetText()); ok || err != nil {
@@ -1506,21 +1506,22 @@ func (v *ParserVisitor) VisitMembershipMatchWithOption(ctx *parser.MembershipMat
var fieldText string
var fieldInfo *planpb.ColumnInfo
var err error
fieldText = ctx.GetField().GetText()
switch ctx.GetField().GetTokenType() {
case parser.PlanParserIdentifier, parser.PlanParserMeta:
field := ctx.GetField()
fieldText = field.GetText()
switch {
case field.FieldName() != nil || field.Meta() != nil:
var fieldExpr *ExprWithType
fieldExpr, err = v.translateIdentifier(fieldText)
if err == nil {
fieldInfo = toColumnInfo(fieldExpr)
}
case parser.PlanParserJSONIdentifier:
case field.JSONIdentifier() != nil:
fieldInfo, err = v.getColumnInfoFromJSONIdentifier(fieldText)
case parser.PlanParserStructFieldIdentifier:
case field.StructFieldIdentifier() != nil:
fieldInfo, err = v.getColumnInfoFromStructField(fieldText)
case parser.PlanParserStructIndexFieldIdentifier:
case field.StructIndexFieldIdentifier() != nil:
fieldInfo, err = v.getColumnInfoFromStructIndexField(fieldText)
case parser.PlanParserStructSubFieldIdentifier:
case field.StructSubFieldIdentifier() != nil:
fieldInfo, err = v.getColumnInfoFromStructSubField(fieldText)
}
if err != nil {
@@ -1546,7 +1547,7 @@ func (v *ParserVisitor) VisitMembershipMatchWithOption(ctx *parser.MembershipMat
// VisitRange translates expr to range plan.
func (v *ParserVisitor) VisitRange(ctx *parser.RangeContext) interface{} {
columnInfo, err := v.getChildColumnInfo(
ctx.Identifier(),
ctx.FieldName(),
ctx.JSONIdentifier(),
ctx.StructSubFieldIdentifier(),
ctx.StructIndexFieldIdentifier(),
@@ -1626,7 +1627,7 @@ func (v *ParserVisitor) VisitRange(ctx *parser.RangeContext) interface{} {
// VisitReverseRange parses the expression like "1 > a > 0".
func (v *ParserVisitor) VisitReverseRange(ctx *parser.ReverseRangeContext) interface{} {
columnInfo, err := v.getChildColumnInfo(
ctx.Identifier(),
ctx.FieldName(),
ctx.JSONIdentifier(),
ctx.StructSubFieldIdentifier(),
ctx.StructIndexFieldIdentifier(),
@@ -2605,7 +2606,7 @@ func (v *ParserVisitor) VisitEmptyArray(ctx *parser.EmptyArrayContext) interface
}
func (v *ParserVisitor) VisitIsNotNull(ctx *parser.IsNotNullContext) interface{} {
column, err := v.getNullExprColumnInfo(ctx.Identifier(), ctx.JSONIdentifier())
column, err := v.getNullExprColumnInfo(ctx.FieldName(), ctx.JSONIdentifier())
if err != nil {
return err
}
@@ -2648,7 +2649,7 @@ func (v *ParserVisitor) VisitIsNotNull(ctx *parser.IsNotNullContext) interface{}
}
func (v *ParserVisitor) VisitIsNull(ctx *parser.IsNullContext) interface{} {
column, err := v.getNullExprColumnInfo(ctx.Identifier(), ctx.JSONIdentifier())
column, err := v.getNullExprColumnInfo(ctx.FieldName(), ctx.JSONIdentifier())
if err != nil {
return err
}
@@ -2940,15 +2941,15 @@ func (v *ParserVisitor) VisitArrayLength(ctx *parser.ArrayLengthContext) interfa
Nullable: field.GetNullable(),
}
} else {
if ctx.Identifier() != nil {
if parentColumnInfo, ok, parentErr := v.getStructArrayParentColumnInfo(ctx.Identifier().GetText()); ok || parentErr != nil {
if ctx.FieldName() != nil {
if parentColumnInfo, ok, parentErr := v.getStructArrayParentColumnInfo(ctx.FieldName().GetText()); ok || parentErr != nil {
columnInfo = parentColumnInfo
err = parentErr
}
}
if columnInfo == nil && err == nil {
columnInfo, err = v.getChildColumnInfo(
ctx.Identifier(),
ctx.FieldName(),
ctx.JSONIdentifier(),
ctx.StructSubFieldIdentifier(),
nil,
@@ -2994,7 +2995,7 @@ func (v *ParserVisitor) VisitTemplateVariable(ctx *parser.TemplateVariableContex
Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{
Value: nil,
TemplateVariableName: ctx.Identifier().GetText(),
TemplateVariableName: ctx.FieldName().GetText(),
},
},
IsTemplate: true,
@@ -3003,7 +3004,7 @@ func (v *ParserVisitor) VisitTemplateVariable(ctx *parser.TemplateVariableContex
}
func (v *ParserVisitor) VisitSpatialBinary(ctx *parser.SpatialBinaryContext) interface{} {
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
childExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
if err != nil {
return err
}
@@ -3063,7 +3064,7 @@ func (v *ParserVisitor) VisitSpatialBinary(ctx *parser.SpatialBinaryContext) int
}
func (v *ParserVisitor) VisitSTIsValid(ctx *parser.STIsValidContext) interface{} {
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
childExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
if err != nil {
return err
}
@@ -3089,7 +3090,7 @@ func (v *ParserVisitor) VisitSTIsValid(ctx *parser.STIsValidContext) interface{}
func (v *ParserVisitor) VisitSTDWithin(ctx *parser.STDWithinContext) interface{} {
// Process the geometry field identifier
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
childExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
if err != nil {
return err
}
@@ -3173,8 +3174,8 @@ func (v *ParserVisitor) VisitSTDWithin(ctx *parser.STDWithinContext) interface{}
// 2. Slow Path: If an INTERVAL exists, it generates a TimestamptzArithCompareExpr
// for specialized arithmetic evaluation.
func (v *ParserVisitor) VisitTimestamptzCompareForward(ctx *parser.TimestamptzCompareForwardContext) interface{} {
colExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
identifier := ctx.Identifier().Accept(v)
colExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
identifier := ctx.FieldName().GetText()
if err != nil {
return merr.WrapErrParameterInvalidMsg("can not translate identifier: %s", identifier)
}
@@ -3247,8 +3248,8 @@ func (v *ParserVisitor) VisitTimestamptzCompareForward(ctx *parser.TimestamptzCo
// 3. Slow Path: For complex expressions involving INTERVAL, it produces a
// TimestamptzArithCompareExpr with the reversed operator.
func (v *ParserVisitor) VisitTimestamptzCompareReverse(ctx *parser.TimestamptzCompareReverseContext) interface{} {
colExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
identifier := ctx.Identifier().GetText()
colExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
identifier := ctx.FieldName().GetText()
if err != nil {
return merr.WrapErrParameterInvalidMsg("can not translate identifier: %s", identifier)
}
@@ -3364,7 +3365,7 @@ func (v *ParserVisitor) VisitElementFilter(ctx *parser.ElementFilterContext) int
}
// Get struct array field name (first parameter)
arrayFieldName := ctx.Identifier().GetText()
arrayFieldName := ctx.FieldName().GetText()
// Set current context for element expression parsing
v.currentStructArrayField = arrayFieldName
@@ -3527,7 +3528,7 @@ func (v *ParserVisitor) parseMatchExpr(structArrayFieldName string, exprCtx pars
// VisitMatchSimple handles MATCH_ALL and MATCH_ANY expressions
// Syntax: MATCH_ALL/MATCH_ANY(structArrayField, $[intField] == 1 && $[strField] == "aaa")
func (v *ParserVisitor) VisitMatchSimple(ctx *parser.MatchSimpleContext) interface{} {
structArrayFieldName := ctx.Identifier().GetText()
structArrayFieldName := ctx.FieldName().GetText()
var matchType planpb.MatchType
var opName string
switch ctx.GetOp().GetTokenType() {
@@ -3546,7 +3547,7 @@ func (v *ParserVisitor) VisitMatchSimple(ctx *parser.MatchSimpleContext) interfa
// VisitMatchThreshold handles MATCH_LEAST, MATCH_MOST, and MATCH_EXACT expressions
// Syntax: MATCH_LEAST/MATCH_MOST/MATCH_EXACT(structArrayField, $[intField] == 1, threshold=N)
func (v *ParserVisitor) VisitMatchThreshold(ctx *parser.MatchThresholdContext) interface{} {
structArrayFieldName := ctx.Identifier().GetText()
structArrayFieldName := ctx.FieldName().GetText()
countStr := ctx.IntegerConstant().GetText()
count, err := strconv.ParseInt(countStr, 10, 64)
+264
View File
@@ -0,0 +1,264 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package proxy
import (
"context"
"strings"
"testing"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/metrics"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/requestutil"
)
var contextualFieldKeywordsForTest = []string{
"text_match_fuzzy", "match_all", "match_any", "match_least", "match_most", "match_exact",
"iso", "interval", "minimum_should_match", "threshold", "element_filter",
"st_equals", "st_touches", "st_overlaps", "st_crosses", "st_contains",
"st_intersects", "st_within", "st_dwithin", "st_isvalid",
}
func fieldKeywordCaseVariantsForTest(keyword string) []string {
names := make([]string, 0, 1<<len(keyword))
for mask := 0; mask < 1<<len(keyword); mask++ {
name := []byte(keyword)
for i := range name {
if mask&(1<<i) != 0 {
name[i] -= 'a' - 'A'
}
}
names = append(names, string(name))
}
return names
}
func keywordValidationSchemaForTest() *schemapb.CollectionSchema {
return &schemapb.CollectionSchema{
Name: "keyword_validation",
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "pk", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{
FieldID: 101, Name: "embedding", DataType: schemapb.DataType_FloatVector,
TypeParams: []*commonpb.KeyValuePair{{Key: common.DimKey, Value: "4"}},
},
},
}
}
func keywordArrayFieldForTest(name string) *schemapb.FieldSchema {
return &schemapb.FieldSchema{
Name: name, DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int64,
TypeParams: []*commonpb.KeyValuePair{{Key: common.MaxCapacityKey, Value: "16"}},
}
}
func keywordStructFieldForTest(name string) *schemapb.StructArrayFieldSchema {
return &schemapb.StructArrayFieldSchema{
Name: name, Nullable: true,
Fields: []*schemapb.FieldSchema{keywordArrayFieldForTest("values")},
}
}
func createKeywordSchemaForTest(t *testing.T, schema *schemapb.CollectionSchema) error {
t.Helper()
bytes, err := proto.Marshal(schema)
require.NoError(t, err)
task := &createCollectionTask{
CreateCollectionRequest: &milvuspb.CreateCollectionRequest{
Base: &commonpb.MsgBase{}, CollectionName: schema.GetName(), Schema: bytes,
},
}
return task.PreExecute(context.Background())
}
func requireKeywordValidationForTest(t *testing.T, err error, reserved bool) {
t.Helper()
if reserved {
requireInputValidationErrorForTest(t, err, merr.ErrFieldInvalidName, 1701)
} else {
require.NoError(t, err)
}
}
func requireInputValidationErrorForTest(t *testing.T, err, expectedErr error, expectedCode int32) {
t.Helper()
require.ErrorIs(t, err, expectedErr)
require.Equal(t, expectedCode, merr.Code(err))
require.Equal(t, merr.InputError, merr.GetErrorType(err))
// Create/Add RPCs project the task error with merr.Status; REST then
// reconstructs it with merr.Error. Classification must survive both.
status := merr.Status(err)
require.Equal(t, expectedCode, status.GetCode())
require.Equal(t, err.Error(), status.GetReason())
require.False(t, status.GetRetriable())
require.Equal(t, "true", status.GetExtraInfo()[merr.InputErrorFlagKey])
label, cause := requestutil.ParseMetricLabel(status, nil)
require.Equal(t, metrics.FailLabel, label)
require.Equal(t, metrics.CauseUser, cause)
roundTrip := merr.Error(status)
require.ErrorIs(t, roundTrip, expectedErr)
require.Equal(t, expectedCode, merr.Code(roundTrip))
require.Equal(t, merr.InputError, merr.GetErrorType(roundTrip))
require.False(t, merr.IsRetryableErr(roundTrip))
}
func TestFieldNameKeywordPolicy(t *testing.T) {
check := func(t *testing.T, name string, reserved bool) {
schema := keywordValidationSchemaForTest()
schema.Fields = append(schema.Fields, &schemapb.FieldSchema{
Name: name, DataType: schemapb.DataType_Int64,
})
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, schema), reserved)
// The public add-field path must enforce the same policy as creation.
field, err := proto.Marshal(&schemapb.FieldSchema{
Name: name, DataType: schemapb.DataType_Int64, Nullable: true,
})
require.NoError(t, err)
add := &addCollectionFieldTask{
oldSchema: keywordValidationSchemaForTest(),
AddCollectionFieldRequest: &milvuspb.AddCollectionFieldRequest{Schema: field},
}
requireKeywordValidationForTest(t, add.PreExecute(context.Background()), reserved)
// Struct sub-fields already use the shared field-name validator.
requireKeywordValidationForTest(t, ValidateFieldsInStruct(
keywordArrayFieldForTest(name), keywordValidationSchemaForTest()), reserved)
// Check sub-field failures through actual create/add struct tasks too,
// before the names are transformed to profile[field].
structField := keywordStructFieldForTest("profile")
structField.Fields[0].Name = name
structSchema := keywordValidationSchemaForTest()
structSchema.StructArrayFields = []*schemapb.StructArrayFieldSchema{structField}
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, structSchema), reserved)
addStruct := &addCollectionStructFieldTask{
oldSchema: keywordValidationSchemaForTest(),
AddCollectionStructFieldRequest: &milvuspb.AddCollectionStructFieldRequest{
StructArrayFieldSchema: structField,
},
}
requireKeywordValidationForTest(t, addStruct.PreExecute(context.Background()), reserved)
}
for _, keyword := range []string{"like", "and", "or", "not", "in", "null"} {
for _, name := range fieldKeywordCaseVariantsForTest(keyword) {
t.Run(name, func(t *testing.T) { check(t, name, true) })
}
}
for _, keyword := range contextualFieldKeywordsForTest {
for _, name := range []string{keyword, strings.ToUpper(keyword)} {
t.Run(name, func(t *testing.T) { check(t, name, false) })
}
}
t.Run("ordinary_field", func(t *testing.T) { check(t, "ordinary_field", false) })
for _, name := range []string{"", "1field", "bad name", "a-b", strings.Repeat("a", 256)} {
t.Run("invalid_"+name, func(t *testing.T) { check(t, name, true) })
}
}
func TestStructArrayFieldKeywordPolicy(t *testing.T) {
check := func(t *testing.T, name string, reserved bool) {
schema := keywordValidationSchemaForTest()
schema.StructArrayFields = []*schemapb.StructArrayFieldSchema{keywordStructFieldForTest(name)}
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, schema), reserved)
add := &addCollectionStructFieldTask{
oldSchema: keywordValidationSchemaForTest(),
AddCollectionStructFieldRequest: &milvuspb.AddCollectionStructFieldRequest{
StructArrayFieldSchema: keywordStructFieldForTest(name),
},
}
requireKeywordValidationForTest(t, add.PreExecute(context.Background()), reserved)
}
for _, keyword := range []string{"like", "and", "or", "not", "in", "null"} {
for _, name := range fieldKeywordCaseVariantsForTest(keyword) {
t.Run(name, func(t *testing.T) { check(t, name, true) })
}
}
for _, keyword := range contextualFieldKeywordsForTest {
for _, name := range []string{keyword, strings.ToUpper(keyword)} {
t.Run(name, func(t *testing.T) { check(t, name, false) })
}
}
t.Run("ordinary_struct", func(t *testing.T) { check(t, "profile", false) })
for _, name := range []string{"", "1field", "bad name", "$meta", "a-b", strings.Repeat("a", 256)} {
t.Run("invalid_"+name, func(t *testing.T) { check(t, name, true) })
}
t.Run("parent_name_is_checked_before_children", func(t *testing.T) {
err := ValidateStructArrayField(&schemapb.StructArrayFieldSchema{Name: "AND"}, keywordValidationSchemaForTest())
requireKeywordValidationForTest(t, err, true)
})
}
func TestFieldNameErrorClassification(t *testing.T) {
// Every invalid field name originates from user-supplied schema input.
// Factory and validation paths must agree on the wire and metric cause.
factoryErr := merr.WrapErrFieldNameInvalid("Like", "Invalid field name: Like. Like is keyword in milvus.")
requireKeywordValidationForTest(t, factoryErr, true)
validationErr := validateFieldName("Like")
requireKeywordValidationForTest(t, validationErr, true)
require.Equal(t, factoryErr.Error(), validationErr.Error())
require.Equal(t, merr.Status(factoryErr).GetErrorCode(), merr.Status(validationErr).GetErrorCode())
for _, name := range []string{common.RowIDFieldName, common.TimeStampFieldName, common.VirtualPKFieldName} {
t.Run("reserved_"+name, func(t *testing.T) {
for _, structField := range []bool{false, true} {
schema := keywordValidationSchemaForTest()
if structField {
schema.StructArrayFields = []*schemapb.StructArrayFieldSchema{keywordStructFieldForTest(name)}
} else {
schema.Fields = append(schema.Fields, &schemapb.FieldSchema{Name: name, DataType: schemapb.DataType_Int64})
}
requireKeywordValidationForTest(t, validateReservedFieldNames(schema), true)
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, schema), true)
}
})
}
t.Run("meta_keeps_existing_api_error_codes", func(t *testing.T) {
// Creation and direct name validation reject '$' with 1701. AddField
// already rejects system fields earlier with 1100; preserve that code.
requireKeywordValidationForTest(t, validateFieldName(common.MetaFieldName), true)
schema := keywordValidationSchemaForTest()
schema.Fields = append(schema.Fields, &schemapb.FieldSchema{
Name: common.MetaFieldName, DataType: schemapb.DataType_Int64,
})
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, schema), true)
field, err := proto.Marshal(&schemapb.FieldSchema{
Name: common.MetaFieldName, DataType: schemapb.DataType_Int64, Nullable: true,
})
require.NoError(t, err)
add := &addCollectionFieldTask{
oldSchema: keywordValidationSchemaForTest(),
AddCollectionFieldRequest: &milvuspb.AddCollectionFieldRequest{Schema: field},
}
requireInputValidationErrorForTest(t, add.PreExecute(context.Background()), merr.ErrParameterInvalid, 1100)
})
}
+1 -1
View File
@@ -772,7 +772,7 @@ func validateAddStructFieldRequest(schema *schemapb.CollectionSchema, structFiel
if funcutil.SliceContain([]string{common.RowIDFieldName, common.TimeStampFieldName, common.MetaFieldName, common.NamespaceFieldName, common.VirtualPKFieldName}, structFieldSchema.GetName()) {
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", structFieldSchema.GetName())
}
if err := ValidateStructArrayField(structFieldSchema, schema); err != nil {
if err := validateStructArrayField(structFieldSchema, schema); err != nil {
return err
}
if err := validateAddStructSubFieldProperties(structFieldSchema); err != nil {
+10
View File
@@ -324,6 +324,7 @@ func validatePartitionTag(partitionTag string, strictCheck bool) error {
return nil
}
// validateFieldName validates names from user schema requests.
func validateFieldName(fieldName string) error {
fieldName = strings.TrimSpace(fieldName)
@@ -822,6 +823,15 @@ func validateStructArrayFieldMaxCapacity(structArrayField *schemapb.StructArrayF
// ValidateStructArrayField validates the struct array field schema.
// When the struct is nullable, sub-field schemas are mutated in-place to set Nullable=true.
func ValidateStructArrayField(structArrayField *schemapb.StructArrayFieldSchema, schema *schemapb.CollectionSchema) error {
if err := validateFieldName(structArrayField.GetName()); err != nil {
return err
}
return validateStructArrayField(structArrayField, schema)
}
// validateStructArrayField validates the remaining schema after the parent name
// has been checked. Add-field validation checks the name before reserved fields.
func validateStructArrayField(structArrayField *schemapb.StructArrayFieldSchema, schema *schemapb.CollectionSchema) error {
if len(structArrayField.Fields) == 0 {
return merr.WrapErrParameterInvalidMsg("struct array field %s has no sub-fields", structArrayField.Name)
}
@@ -18,6 +18,7 @@ package rootcoord
import (
"context"
"strings"
"testing"
"github.com/stretchr/testify/require"
@@ -32,6 +33,24 @@ import (
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
func TestValidateAddedStructFieldNameErrorClassification(t *testing.T) {
for _, name := range []string{"", strings.Repeat("a", Params.ProxyCfg.MaxNameLength.GetAsInt()+1), "1field", "bad name", "AND"} {
t.Run(name, func(t *testing.T) {
err := validateAddedStructFieldName(name)
require.ErrorIs(t, err, merr.ErrFieldInvalidName)
require.Equal(t, merr.InputError, merr.GetErrorType(err))
status := merr.Status(err)
require.Equal(t, int32(1701), status.GetCode())
require.Equal(t, err.Error(), status.GetReason())
require.Equal(t, "true", status.GetExtraInfo()[merr.InputErrorFlagKey])
require.False(t, status.GetRetriable())
roundTrip := merr.Error(status)
require.ErrorIs(t, roundTrip, merr.ErrFieldInvalidName)
require.Equal(t, merr.InputError, merr.GetErrorType(roundTrip))
})
}
}
func TestDDLCallbacksAlterCollectionAddStructField(t *testing.T) {
core := initStreamingSystemAndCore(t)
+10 -6
View File
@@ -40,14 +40,18 @@ var FieldNameKeywords = map[string]struct{}{
}
// IsFieldNameKeyword reports whether fieldName is a reserved word that cannot be
// used as a field name. `null` is matched case-insensitively — any casing of NULL
// is rejected — to stay consistent with the expression parser, which rejects a
// bare NULL literal regardless of casing (issue #50882). It is therefore handled
// here rather than enumerated in FieldNameKeywords; the other keywords match the
// exact casings listed there.
// used as a field name. Core operators (like, and, or, not, in) and null are
// reserved in every casing. Contextual keywords such as iso and interval stay
// usable as field names; other reserved words retain the exact casings listed
// in FieldNameKeywords.
func IsFieldNameKeyword(fieldName string) bool {
if _, ok := FieldNameKeywords[fieldName]; ok {
return true
}
return strings.EqualFold(fieldName, "null")
switch strings.ToLower(fieldName) {
case "like", "and", "or", "not", "in", "null":
return true
default:
return false
}
}
+49
View File
@@ -1,6 +1,7 @@
package common
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -11,3 +12,51 @@ func TestIsFieldNameKeywordRejectsNullCaseInsensitive(t *testing.T) {
assert.True(t, IsFieldNameKeyword(name), name)
}
}
func TestIsFieldNameKeywordCoreOperators(t *testing.T) {
for _, keyword := range []string{"like", "and", "or", "not", "in", "null"} {
// Enumerate every ASCII casing so title case and mixed case cannot
// bypass the policy while lowercase and uppercase are blocked.
for mask := 0; mask < 1<<len(keyword); mask++ {
name := []byte(keyword)
for i := range name {
if mask&(1<<i) != 0 {
name[i] -= 'a' - 'A'
}
}
t.Run(string(name), func(t *testing.T) {
assert.True(t, IsFieldNameKeyword(string(name)))
})
}
}
}
func TestIsFieldNameKeywordContextualWords(t *testing.T) {
for _, keyword := range []string{
"text_match_fuzzy", "match_all", "match_any", "match_least", "match_most", "match_exact",
"iso", "interval", "minimum_should_match", "threshold", "element_filter",
"st_equals", "st_touches", "st_overlaps", "st_crosses", "st_contains",
"st_intersects", "st_within", "st_dwithin", "st_isvalid",
} {
for _, name := range []string{keyword, strings.ToUpper(keyword)} {
t.Run(name, func(t *testing.T) {
assert.False(t, IsFieldNameKeyword(name))
})
}
}
}
func TestIsFieldNameKeywordExistingPolicy(t *testing.T) {
for name := range FieldNameKeywords {
assert.True(t, IsFieldNameKeyword(name), name)
}
// Unrelated operators and function names keep their existing exact-case
// policy. Soft option names and names merely containing a keyword stay valid.
for _, name := range []string{
"Exists", "Text_Match", "Phrase_Match", "Random_Sample", "Array_Length", "Json_Contains",
"max_edit_distance", "membership_match", "type", "bloom", "roaring",
"null_value", "NOTES", "like_count", "in_stock",
} {
assert.False(t, IsFieldNameKeyword(name), name)
}
}
@@ -117,6 +117,7 @@ func TestSentinelErrorTypeClassification(t *testing.T) {
"IncorrectParameterFormat": ErrIncorrectParameterFormat,
"MissingRequiredParameters": ErrMissingRequiredParameters,
"InvalidInsertData": ErrInvalidInsertData,
"FieldInvalidName": ErrFieldInvalidName,
}
for name, err := range inputSentinels {
assert.Equal(t, InputError, GetErrorType(err), "%s should be InputError", name)
@@ -153,6 +154,7 @@ func TestSentinelErrorTypeClassification(t *testing.T) {
801, 802, // Database: num limit / invalid name
1100, 1101, 1102, // Parameter: invalid / missing / too large
1400, 1401, 1402, // Privilege
1701, // FieldInvalidName
1800, 1801, 1802, 1804, // Auth / parameter format / insert data
2100, // ImportFailed
2201, // QueryPlan
+1 -1
View File
@@ -223,7 +223,7 @@ var (
// field related
ErrFieldNotFound = newMilvusError("field not found", 1700, false) // SystemError by default; proxy boundaries stamp InputError where the field name is user-supplied (e.g. group_by/anns field), while result-assembly lookups stay system.
ErrFieldInvalidName = newMilvusError("field name invalid", 1701, false)
ErrFieldInvalidName = newMilvusError("field name invalid", 1701, false, WithErrorType(InputError))
// high-level restful api related
ErrNeedAuthenticate = newMilvusError("user hasn't authenticated", 1800, false, WithErrorType(InputError))