mirror of
https://github.com/milvus-io/milvus.git
synced 2026-10-11 22:49:53 +00:00
fix: support contextual keywords as field names (#53968)
Fields named `iso` or `interval` can be created, but filters such as `iso > 1` fail because the lexer emits a keyword token where the parser expects an identifier. Accept 20 contextual keyword families through a shared `fieldName` rule in expression field positions while preserving their function, option, and timestamp syntax. Update the visitor and regenerate the parser with ANTLR 4.13.2. Reject `LIKE`, `AND`, `OR`, `NOT`, and `IN` as field names in every casing, and retain the existing case-insensitive `NULL` policy. Validate struct-array parent names on both Create and Add paths, alongside child names. Classify `ErrFieldInvalidName` (1701) as `InputError` at its definition so ordinary names, reserved names, and RootCoord's add-struct-field validator report the same classification. Remove the redundant Proxy error markers and validate each struct parent name once while preserving the existing validation order, codes, reasons, identity, and non-retryability. Compatibility: mixed-case names such as `And`, `In`, and `Like` previously lexed as ordinary identifiers and could be created and filtered. New Create/Add requests reject these names. Existing collections are not revalidated, but backup restoration or cross-cluster schema recreation containing these names will require renaming the affected fields. This tightening is intentional; contextual keyword field names remain supported. Regression coverage includes contextual keywords and their dedicated syntax, field identity/casing, SLL/LL parsing, core keyword rejection, ordinary and struct-array Create/Add paths, reserved field names, and InputError status/metric round trips. RootCoord's name validator now also has classification and status round-trip coverage. Validation: - Current review follow-up: all tests in `pkg/util/merr`, `pkg/util/requestutil`, and `pkg/common` passed with `-tags dynamic,test -gcflags='all=-N -l' -count=1`; `git diff --check` passed. - Current focused Proxy/RootCoord tests were blocked before execution by older local native libraries missing required APIs. The development host was inaccessible under the current network restrictions; native CI validation is pending. - Before this follow-up, the unchanged parser/rewriter implementation passed 1,182 tests/subtests, focused Proxy regressions passed 248 tests/subtests with race detection and coverage, and `merr`/`requestutil` guards passed 143 tests/subtests with race detection and coverage. - Generated parser output was reproduced with ANTLR 4.13.2. - A previous full `make -o build-cpp-with-unittest test-go` attempt timed out in `TestProxy/create_collection` while waiting for streaming assignments and metadata-cache initialization. Later groups were not reached; no fresh C++ build was performed. issue: #53925 Fixes #53925 --------- Signed-off-by: xiaofanluan <[email protected]> Co-authored-by: xiaofanluan <[email protected]>
This commit is contained in:
@@ -1,19 +1,19 @@
|
||||
grammar Plan;
|
||||
|
||||
expr:
|
||||
Identifier (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? op2=(LT | LE | GT | GE | EQ | NE) ISO compare_string=StringLiteral # TimestamptzCompareForward
|
||||
| ISO compare_string=StringLiteral op2=(LT | LE | GT | GE | EQ | NE) Identifier (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? # TimestamptzCompareReverse
|
||||
fieldName (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? op2=(LT | LE | GT | GE | EQ | NE) ISO compare_string=StringLiteral # TimestamptzCompareForward
|
||||
| ISO compare_string=StringLiteral op2=(LT | LE | GT | GE | EQ | NE) fieldName (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? # TimestamptzCompareReverse
|
||||
| IntegerConstant # Integer
|
||||
| FloatingConstant # Floating
|
||||
| BooleanConstant # Boolean
|
||||
| StringLiteral # String
|
||||
| RawStringLiteral # RawString
|
||||
| (Identifier|Meta) # Identifier
|
||||
| (fieldName|Meta) # Identifier
|
||||
| JSONIdentifier # JSONIdentifier
|
||||
| StructFieldIdentifier # StructField
|
||||
| StructIndexFieldIdentifier # StructIndexField
|
||||
| StructSubFieldIdentifier # StructSubField
|
||||
| LBRACE Identifier RBRACE # TemplateVariable
|
||||
| LBRACE fieldName RBRACE # TemplateVariable
|
||||
| '(' expr ')' # Parens
|
||||
| '[' expr (',' expr)* ','? ']' # Array
|
||||
| EmptyArray # EmptyArray
|
||||
@@ -21,13 +21,13 @@ expr:
|
||||
| expr LIKE expr # Like
|
||||
| expr REGEXMATCH expr # RegexMatch
|
||||
| expr REGEXNOTMATCH expr # RegexNotMatch
|
||||
| TEXTMATCH'('Identifier',' expr (',' textMatchOption)? ')' # TextMatch
|
||||
| TEXTMATCHFUZZY'('Identifier',' expr ',' Identifier ASSIGN IntegerConstant ')' # TextMatchFuzzy
|
||||
| PHRASEMATCH'('Identifier',' expr (',' expr)? ')' # PhraseMatch
|
||||
| TEXTMATCH'('fieldName',' expr (',' textMatchOption)? ')' # TextMatch
|
||||
| TEXTMATCHFUZZY'('fieldName',' expr ',' Identifier ASSIGN IntegerConstant ')' # TextMatchFuzzy
|
||||
| PHRASEMATCH'('fieldName',' expr (',' expr)? ')' # PhraseMatch
|
||||
| RANDOMSAMPLE'(' expr ')' # RandomSample
|
||||
| ElementFilter'('Identifier',' expr')' # ElementFilter
|
||||
| op=(MATCH_ALL | MATCH_ANY) '(' Identifier ',' expr ')' # MatchSimple
|
||||
| op=(MATCH_LEAST | MATCH_MOST | MATCH_EXACT) '(' Identifier ',' expr ',' THRESHOLD ASSIGN IntegerConstant ')' # MatchThreshold
|
||||
| ElementFilter'('fieldName',' expr')' # ElementFilter
|
||||
| op=(MATCH_ALL | MATCH_ANY) '(' fieldName ',' expr ')' # MatchSimple
|
||||
| op=(MATCH_LEAST | MATCH_MOST | MATCH_EXACT) '(' fieldName ',' expr ',' THRESHOLD ASSIGN IntegerConstant ')' # MatchThreshold
|
||||
| expr POW expr # Power
|
||||
| op = (ADD | SUB | BNOT | NOT) expr # Unary
|
||||
// | '(' typeName ')' expr # Cast
|
||||
@@ -38,14 +38,14 @@ expr:
|
||||
| (JSONContains | ArrayContains)'('expr',' expr')' # JSONContains
|
||||
| (JSONContainsAll | ArrayContainsAll)'('expr',' expr')' # JSONContainsAll
|
||||
| (JSONContainsAny | ArrayContainsAny)'('expr',' expr')' # JSONContainsAny
|
||||
| op=(STEuqals | STTouches | STOverlaps | STCrosses | STContains | STIntersects | STWithin) '(' Identifier ',' expr ')' # SpatialBinary
|
||||
| STDWithin'('Identifier',' expr',' expr')' # STDWithin
|
||||
| STIsValid'('Identifier')' # STIsValid
|
||||
| ArrayLength'('(Identifier | JSONIdentifier | StructFieldIdentifier | StructSubFieldIdentifier)')' # ArrayLength
|
||||
| function=Identifier '(' field=(Identifier|Meta|JSONIdentifier|StructFieldIdentifier|StructIndexFieldIdentifier|StructSubFieldIdentifier) ',' expr ',' option=Identifier ASSIGN kind=Identifier ')' # MembershipMatchWithOption
|
||||
| op=(STEuqals | STTouches | STOverlaps | STCrosses | STContains | STIntersects | STWithin) '(' fieldName ',' expr ')' # SpatialBinary
|
||||
| STDWithin'('fieldName',' expr',' expr')' # STDWithin
|
||||
| STIsValid'('fieldName')' # STIsValid
|
||||
| ArrayLength'('(fieldName | JSONIdentifier | StructFieldIdentifier | StructSubFieldIdentifier)')' # ArrayLength
|
||||
| function=Identifier '(' field=fieldReference ',' expr ',' option=Identifier ASSIGN kind=Identifier ')' # MembershipMatchWithOption
|
||||
| Identifier '(' ( expr (',' expr )* ','? )? ')' # Call
|
||||
| expr op1 = (LT | LE) (Identifier | JSONIdentifier | StructSubFieldIdentifier | StructIndexFieldIdentifier) op2 = (LT | LE) expr # Range
|
||||
| expr op1 = (GT | GE) (Identifier | JSONIdentifier | StructSubFieldIdentifier | StructIndexFieldIdentifier) op2 = (GT | GE) expr # ReverseRange
|
||||
| expr op1 = (LT | LE) (fieldName | JSONIdentifier | StructSubFieldIdentifier | StructIndexFieldIdentifier) op2 = (LT | LE) expr # Range
|
||||
| expr op1 = (GT | GE) (fieldName | JSONIdentifier | StructSubFieldIdentifier | StructIndexFieldIdentifier) op2 = (GT | GE) expr # ReverseRange
|
||||
| expr op = (LT | LE | GT | GE) expr # Relational
|
||||
| expr op = (EQ | NE) expr # Equality
|
||||
| expr BAND expr # BitAnd
|
||||
@@ -53,8 +53,42 @@ expr:
|
||||
| expr BOR expr # BitOr
|
||||
| expr AND expr # LogicalAnd
|
||||
| expr OR expr # LogicalOr
|
||||
| (Identifier | JSONIdentifier) ISNULL # IsNull
|
||||
| (Identifier | JSONIdentifier) ISNOTNULL # IsNotNull;
|
||||
| (fieldName | JSONIdentifier) ISNULL # IsNull
|
||||
| (fieldName | JSONIdentifier) ISNOTNULL # IsNotNull;
|
||||
|
||||
// These keywords retain their specialized syntax while also naming fields.
|
||||
// Operators and constants remain reserved, and generic function names and
|
||||
// option names continue to use Identifier rather than this rule.
|
||||
fieldName:
|
||||
Identifier
|
||||
| ISO
|
||||
| INTERVAL
|
||||
| MINIMUM_SHOULD_MATCH
|
||||
| THRESHOLD
|
||||
| TEXTMATCHFUZZY
|
||||
| MATCH_ALL
|
||||
| MATCH_ANY
|
||||
| MATCH_LEAST
|
||||
| MATCH_MOST
|
||||
| MATCH_EXACT
|
||||
| ElementFilter
|
||||
| STEuqals
|
||||
| STTouches
|
||||
| STOverlaps
|
||||
| STCrosses
|
||||
| STContains
|
||||
| STIntersects
|
||||
| STWithin
|
||||
| STDWithin
|
||||
| STIsValid;
|
||||
|
||||
fieldReference:
|
||||
fieldName
|
||||
| Meta
|
||||
| JSONIdentifier
|
||||
| StructFieldIdentifier
|
||||
| StructIndexFieldIdentifier
|
||||
| StructSubFieldIdentifier;
|
||||
|
||||
textMatchOption:
|
||||
MINIMUM_SHOULD_MATCH ASSIGN IntegerConstant;
|
||||
|
||||
@@ -0,0 +1,469 @@
|
||||
package planparserv2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/planpb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
// These words have a dedicated token in the expression grammar, but may also
|
||||
// identify fields. Keep both lexer spellings in the matrix: accepting only the
|
||||
// mixed-case Identifier spelling would leave the original regression intact.
|
||||
var contextualKeywordNames = []string{
|
||||
"text_match_fuzzy", "element_filter",
|
||||
"match_all", "match_any", "match_least", "match_most", "match_exact",
|
||||
"st_equals", "st_touches", "st_overlaps", "st_crosses", "st_contains",
|
||||
"st_intersects", "st_within", "st_dwithin", "st_isvalid",
|
||||
"iso", "interval", "minimum_should_match", "threshold",
|
||||
}
|
||||
|
||||
const contextualKeywordFieldID int64 = 2000
|
||||
|
||||
func contextualKeywordSpellings(name string) []string {
|
||||
return []string{name, strings.ToUpper(name), strings.ToUpper(name[:1]) + name[1:]}
|
||||
}
|
||||
|
||||
func contextualKeywordSchema(t *testing.T, name string, dataType schemapb.DataType, dynamic bool) *typeutil.SchemaHelper {
|
||||
t.Helper()
|
||||
schema := newTestSchema(dynamic)
|
||||
field := &schemapb.FieldSchema{
|
||||
FieldID: contextualKeywordFieldID, Name: name, DataType: dataType, Nullable: true,
|
||||
}
|
||||
if dataType == schemapb.DataType_Array {
|
||||
field.ElementType = schemapb.DataType_Int64
|
||||
}
|
||||
schema.Fields = append(schema.Fields, field)
|
||||
enableMatch(schema)
|
||||
helper, err := typeutil.CreateSchemaHelper(schema)
|
||||
require.NoError(t, err)
|
||||
return helper
|
||||
}
|
||||
|
||||
func requireContextualKeywordColumn(t *testing.T, column *planpb.ColumnInfo, name string) {
|
||||
t.Helper()
|
||||
require.NotNil(t, column, name)
|
||||
require.Equal(t, contextualKeywordFieldID, column.GetFieldId(), name)
|
||||
require.Empty(t, column.GetNestedPath(), name)
|
||||
}
|
||||
|
||||
func TestContextualKeywordsScalarFields(t *testing.T) {
|
||||
for _, dynamic := range []bool{false, true} {
|
||||
for _, word := range contextualKeywordNames {
|
||||
for _, name := range contextualKeywordSpellings(word) {
|
||||
t.Run(fmt.Sprintf("dynamic_%t/%s", dynamic, name), func(t *testing.T) {
|
||||
helper := contextualKeywordSchema(t, name, schemapb.DataType_Int64, dynamic)
|
||||
parse := func(format string) *planpb.Expr {
|
||||
t.Helper()
|
||||
expr, err := ParseExpr(helper, fmt.Sprintf(format, name), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, expr)
|
||||
return expr
|
||||
}
|
||||
|
||||
for _, format := range []string{"%s == 400", "400 == %s"} {
|
||||
unary := parse(format).GetUnaryRangeExpr()
|
||||
require.NotNil(t, unary)
|
||||
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
|
||||
require.Equal(t, planpb.OpType_Equal, unary.GetOp())
|
||||
require.Equal(t, int64(400), unary.GetValue().GetInt64Val())
|
||||
}
|
||||
|
||||
term := parse("%s in [1, 400]").GetTermExpr()
|
||||
require.NotNil(t, term)
|
||||
requireContextualKeywordColumn(t, term.GetColumnInfo(), name)
|
||||
require.Len(t, term.GetValues(), 2)
|
||||
require.Equal(t, int64(400), term.GetValues()[1].GetInt64Val())
|
||||
|
||||
arith := parse("%s + 1 > 400").GetBinaryArithOpEvalRangeExpr()
|
||||
require.NotNil(t, arith)
|
||||
requireContextualKeywordColumn(t, arith.GetColumnInfo(), name)
|
||||
require.Equal(t, planpb.ArithOpType_Add, arith.GetArithOp())
|
||||
require.Equal(t, int64(1), arith.GetRightOperand().GetInt64Val())
|
||||
require.Equal(t, planpb.OpType_GreaterThan, arith.GetOp())
|
||||
require.Equal(t, int64(400), arith.GetValue().GetInt64Val())
|
||||
|
||||
for _, format := range []string{"5 < %s <= 400", "400 >= %s > 5"} {
|
||||
rangeExpr := parse(format).GetBinaryRangeExpr()
|
||||
require.NotNil(t, rangeExpr)
|
||||
requireContextualKeywordColumn(t, rangeExpr.GetColumnInfo(), name)
|
||||
require.Equal(t, int64(5), rangeExpr.GetLowerValue().GetInt64Val())
|
||||
require.Equal(t, int64(400), rangeExpr.GetUpperValue().GetInt64Val())
|
||||
require.False(t, rangeExpr.GetLowerInclusive())
|
||||
require.True(t, rangeExpr.GetUpperInclusive())
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
format string
|
||||
op planpb.NullExpr_NullOp
|
||||
}{
|
||||
{"%s is null", planpb.NullExpr_IsNull},
|
||||
{"%s is not null", planpb.NullExpr_IsNotNull},
|
||||
} {
|
||||
nullExpr := parse(tc.format).GetNullExpr()
|
||||
require.NotNil(t, nullExpr)
|
||||
requireContextualKeywordColumn(t, nullExpr.GetColumnInfo(), name)
|
||||
require.Equal(t, tc.op, nullExpr.GetOp())
|
||||
require.True(t, nullExpr.GetColumnInfo().GetNullable())
|
||||
}
|
||||
|
||||
// The same token must be usable in a template name, independently
|
||||
// of its use as the column on the left.
|
||||
expr, err := ParseExpr(helper, fmt.Sprintf("%s == {%s}", name, name), map[string]*schemapb.TemplateValue{
|
||||
name: generateTemplateValue(schemapb.DataType_Int64, int64(400)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
unary := expr.GetUnaryRangeExpr()
|
||||
require.NotNil(t, unary)
|
||||
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
|
||||
require.Equal(t, int64(400), unary.GetValue().GetInt64Val())
|
||||
|
||||
call := parse("custom_predicate(%s, 1)").GetCallExpr()
|
||||
require.NotNil(t, call)
|
||||
require.Equal(t, "custom_predicate", call.GetFunctionName())
|
||||
require.Len(t, call.GetFunctionParameters(), 2)
|
||||
requireContextualKeywordColumn(t, call.GetFunctionParameters()[0].GetColumnExpr().GetInfo(), name)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualKeywordsCaseSensitiveLookup(t *testing.T) {
|
||||
for _, word := range contextualKeywordNames {
|
||||
t.Run(word, func(t *testing.T) {
|
||||
upper := strings.ToUpper(word)
|
||||
helper := contextualKeywordSchema(t, word, schemapb.DataType_Int64, true)
|
||||
expr, err := ParseExpr(helper, word+" == 400", nil)
|
||||
require.NoError(t, err)
|
||||
requireContextualKeywordColumn(t, expr.GetUnaryRangeExpr().GetColumnInfo(), word)
|
||||
|
||||
// A differently-cased name stays a different dynamic JSON key. It
|
||||
// must not resolve to the lower-case declaration by normalization.
|
||||
expr, err = ParseExpr(helper, upper+" == 400", nil)
|
||||
require.NoError(t, err)
|
||||
column := expr.GetUnaryRangeExpr().GetColumnInfo()
|
||||
require.Equal(t, int64(dynamicFieldID), column.GetFieldId())
|
||||
require.Equal(t, []string{upper}, column.GetNestedPath())
|
||||
|
||||
helper = contextualKeywordSchema(t, word, schemapb.DataType_Int64, false)
|
||||
_, err = ParseExpr(helper, upper+" == 400", nil)
|
||||
require.Error(t, err)
|
||||
|
||||
schema := newTestSchema(true)
|
||||
schema.Fields = append(schema.Fields,
|
||||
&schemapb.FieldSchema{FieldID: contextualKeywordFieldID, Name: word, DataType: schemapb.DataType_Int64},
|
||||
&schemapb.FieldSchema{FieldID: contextualKeywordFieldID + 1, Name: upper, DataType: schemapb.DataType_Int64},
|
||||
)
|
||||
helper, err = typeutil.CreateSchemaHelper(schema)
|
||||
require.NoError(t, err)
|
||||
expr, err = ParseExpr(helper, upper+" == 400", nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, contextualKeywordFieldID+1, expr.GetUnaryRangeExpr().GetColumnInfo().GetFieldId())
|
||||
require.Empty(t, expr.GetUnaryRangeExpr().GetColumnInfo().GetNestedPath())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualKeywordsNestedFields(t *testing.T) {
|
||||
for _, word := range contextualKeywordNames {
|
||||
for _, name := range contextualKeywordSpellings(word) {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
helper := contextualKeywordSchema(t, name, schemapb.DataType_JSON, true)
|
||||
expr, err := ParseExpr(helper, fmt.Sprintf(`%s["value"] == 400`, name), nil)
|
||||
require.NoError(t, err)
|
||||
column := expr.GetUnaryRangeExpr().GetColumnInfo()
|
||||
require.Equal(t, contextualKeywordFieldID, column.GetFieldId())
|
||||
require.Equal(t, []string{"value"}, column.GetNestedPath())
|
||||
|
||||
helper = contextualKeywordSchema(t, name, schemapb.DataType_Array, true)
|
||||
expr, err = ParseExpr(helper, fmt.Sprintf("%s[0] == 400", name), nil)
|
||||
require.NoError(t, err)
|
||||
column = expr.GetUnaryRangeExpr().GetColumnInfo()
|
||||
require.Equal(t, contextualKeywordFieldID, column.GetFieldId())
|
||||
require.Equal(t, []string{"0"}, column.GetNestedPath())
|
||||
|
||||
// Parent and sub-field names both use contextual tokens, while
|
||||
// element predicates retain their dedicated scope and plan node.
|
||||
schema := newTestSchema(true)
|
||||
schema.StructArrayFields = append(schema.StructArrayFields, &schemapb.StructArrayFieldSchema{
|
||||
FieldID: contextualKeywordFieldID, Name: name, Nullable: true,
|
||||
Fields: []*schemapb.FieldSchema{{
|
||||
FieldID: contextualKeywordFieldID + 1, Name: fmt.Sprintf("%s[%s]", name, name),
|
||||
DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int64,
|
||||
}},
|
||||
})
|
||||
helper, err = typeutil.CreateSchemaHelper(schema)
|
||||
require.NoError(t, err)
|
||||
expr, err = ParseExpr(helper, fmt.Sprintf("element_filter(%s, $[%s] > 5)", name, name), nil)
|
||||
require.NoError(t, err)
|
||||
element := expr.GetElementFilterExpr()
|
||||
require.NotNil(t, element)
|
||||
require.Equal(t, name, element.GetStructName())
|
||||
require.Equal(t, contextualKeywordFieldID+1, element.GetElementExpr().GetUnaryRangeExpr().GetColumnInfo().GetFieldId())
|
||||
if word == "element_filter" {
|
||||
expr, err = ParseExpr(helper, fmt.Sprintf("ELEMENT_FILTER(%s, $[%s] > 5)", name, name), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, expr.GetElementFilterExpr())
|
||||
require.Equal(t, name, expr.GetElementFilterExpr().GetStructName())
|
||||
}
|
||||
|
||||
expr, err = ParseExpr(helper, fmt.Sprintf("%s is null", name), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, expr.GetNullExpr())
|
||||
require.Equal(t, contextualKeywordFieldID+1, expr.GetNullExpr().GetColumnInfo().GetFieldId())
|
||||
|
||||
ret := handleExpr(helper, fmt.Sprintf("array_length(%s[%s])", name, name))
|
||||
require.NoError(t, getError(ret))
|
||||
length := getExpr(ret).expr.GetBinaryArithExpr()
|
||||
require.NotNil(t, length)
|
||||
require.Equal(t, planpb.ArithOpType_ArrayLength, length.GetOp())
|
||||
require.Equal(t, contextualKeywordFieldID+1, length.GetLeft().GetColumnExpr().GetInfo().GetFieldId())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualKeywordsTextMatchPlans(t *testing.T) {
|
||||
for _, name := range []string{"text_match_fuzzy", "TEXT_MATCH_FUZZY"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
helper := contextualKeywordSchema(t, name, schemapb.DataType_VarChar, true)
|
||||
expr, err := ParseExpr(helper, fmt.Sprintf(`%s(%s, "hello", max_edit_distance=1)`, name, name), nil)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, expr.GetCallExpr())
|
||||
unary := expr.GetUnaryRangeExpr()
|
||||
require.NotNil(t, unary)
|
||||
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
|
||||
require.Equal(t, planpb.OpType_TextMatchFuzzy, unary.GetOp())
|
||||
require.Equal(t, "hello", unary.GetValue().GetStringVal())
|
||||
require.Len(t, unary.GetExtraValues(), 1)
|
||||
require.Equal(t, int64(1), unary.GetExtraValues()[0].GetInt64Val())
|
||||
|
||||
expr, err = ParseExpr(helper, fmt.Sprintf(`starts_with(%s, "hello")`, name), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, expr.GetCallExpr())
|
||||
require.Equal(t, "starts_with", expr.GetCallExpr().GetFunctionName())
|
||||
requireContextualKeywordColumn(t, expr.GetCallExpr().GetFunctionParameters()[0].GetColumnExpr().GetInfo(), name)
|
||||
})
|
||||
}
|
||||
|
||||
for _, name := range []string{"minimum_should_match", "MINIMUM_SHOULD_MATCH"} {
|
||||
helper := contextualKeywordSchema(t, name, schemapb.DataType_VarChar, true)
|
||||
expr, err := ParseExpr(helper, fmt.Sprintf(`text_match(%s, "hello", %s=2)`, name, name), nil)
|
||||
require.NoError(t, err)
|
||||
unary := expr.GetUnaryRangeExpr()
|
||||
require.NotNil(t, unary)
|
||||
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
|
||||
require.Equal(t, planpb.OpType_TextMatch, unary.GetOp())
|
||||
require.Len(t, unary.GetExtraValues(), 1)
|
||||
require.Equal(t, int64(2), unary.GetExtraValues()[0].GetInt64Val())
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualKeywordsSpatialPlans(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
op planpb.GISFunctionFilterExpr_GISOp
|
||||
}{
|
||||
{"st_equals", planpb.GISFunctionFilterExpr_Equals},
|
||||
{"st_touches", planpb.GISFunctionFilterExpr_Touches},
|
||||
{"st_overlaps", planpb.GISFunctionFilterExpr_Overlaps},
|
||||
{"st_crosses", planpb.GISFunctionFilterExpr_Crosses},
|
||||
{"st_contains", planpb.GISFunctionFilterExpr_Contains},
|
||||
{"st_intersects", planpb.GISFunctionFilterExpr_Intersects},
|
||||
{"st_within", planpb.GISFunctionFilterExpr_Within},
|
||||
{"st_dwithin", planpb.GISFunctionFilterExpr_DWithin},
|
||||
{"st_isvalid", planpb.GISFunctionFilterExpr_STIsValid},
|
||||
} {
|
||||
for _, name := range []string{tc.name, strings.ToUpper(tc.name)} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
helper := contextualKeywordSchema(t, name, schemapb.DataType_Geometry, true)
|
||||
query := fmt.Sprintf(`%s(%s, "POINT(0 0)")`, name, name)
|
||||
switch tc.op {
|
||||
case planpb.GISFunctionFilterExpr_DWithin:
|
||||
query = fmt.Sprintf(`%s(%s, "POINT(0 0)", 5)`, name, name)
|
||||
case planpb.GISFunctionFilterExpr_STIsValid:
|
||||
query = fmt.Sprintf("%s(%s)", name, name)
|
||||
}
|
||||
expr, err := ParseExpr(helper, query, nil)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, expr.GetCallExpr())
|
||||
spatial := expr.GetGisfunctionFilterExpr()
|
||||
require.NotNil(t, spatial)
|
||||
requireContextualKeywordColumn(t, spatial.GetColumnInfo(), name)
|
||||
require.Equal(t, tc.op, spatial.GetOp())
|
||||
if tc.op != planpb.GISFunctionFilterExpr_STIsValid {
|
||||
require.Equal(t, "POINT(0 0)", spatial.GetWktString())
|
||||
}
|
||||
if tc.op == planpb.GISFunctionFilterExpr_DWithin {
|
||||
require.Equal(t, float64(5), spatial.GetDistance())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualKeywordsMatchPlans(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
matchType planpb.MatchType
|
||||
count int64
|
||||
}{
|
||||
{"match_all", planpb.MatchType_MatchAll, 0},
|
||||
{"match_any", planpb.MatchType_MatchAny, 0},
|
||||
{"match_least", planpb.MatchType_MatchLeast, 2},
|
||||
{"match_most", planpb.MatchType_MatchMost, 2},
|
||||
{"match_exact", planpb.MatchType_MatchExact, 2},
|
||||
} {
|
||||
for _, name := range []string{tc.name, strings.ToUpper(tc.name)} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
schema := newTestSchema(true)
|
||||
schema.StructArrayFields = append(schema.StructArrayFields, &schemapb.StructArrayFieldSchema{
|
||||
FieldID: contextualKeywordFieldID, Name: name,
|
||||
Fields: []*schemapb.FieldSchema{{
|
||||
FieldID: contextualKeywordFieldID + 1, Name: name + "[threshold]",
|
||||
DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int64,
|
||||
}},
|
||||
})
|
||||
helper, err := typeutil.CreateSchemaHelper(schema)
|
||||
require.NoError(t, err)
|
||||
query := fmt.Sprintf("%s(%s, $[threshold] > 5)", name, name)
|
||||
if tc.count != 0 {
|
||||
option := "threshold"
|
||||
if name == strings.ToUpper(name) {
|
||||
option = "THRESHOLD"
|
||||
}
|
||||
query = fmt.Sprintf("%s(%s, $[threshold] > 5, %s=2)", name, name, option)
|
||||
}
|
||||
expr, err := ParseExpr(helper, query, nil)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, expr.GetCallExpr())
|
||||
match := expr.GetMatchExpr()
|
||||
require.NotNil(t, match)
|
||||
require.Equal(t, name, match.GetStructName())
|
||||
require.Equal(t, tc.matchType, match.GetMatchType())
|
||||
require.Equal(t, tc.count, match.GetCount())
|
||||
require.Equal(t, contextualKeywordFieldID+1, match.GetPredicate().GetUnaryRangeExpr().GetColumnInfo().GetFieldId())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualKeywordsTimePlans(t *testing.T) {
|
||||
const date = "'2025-01-01T00:00:00Z'"
|
||||
wantMicros := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC).UnixMicro()
|
||||
for _, word := range []string{"iso", "interval", "minimum_should_match", "threshold"} {
|
||||
for _, name := range []string{word, strings.ToUpper(word)} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
helper := contextualKeywordSchema(t, name, schemapb.DataType_Timestamptz, true)
|
||||
for _, query := range []string{name + " > iso " + date, "ISO " + date + " < " + name} {
|
||||
expr, err := ParseExpr(helper, query, nil)
|
||||
require.NoError(t, err, query)
|
||||
unary := expr.GetUnaryRangeExpr()
|
||||
require.NotNil(t, unary, query)
|
||||
requireContextualKeywordColumn(t, unary.GetColumnInfo(), name)
|
||||
require.Equal(t, planpb.OpType_GreaterThan, unary.GetOp())
|
||||
require.Equal(t, wantMicros, unary.GetValue().GetInt64Val())
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
query string
|
||||
arith planpb.ArithOpType
|
||||
op planpb.OpType
|
||||
span *planpb.Interval
|
||||
}{
|
||||
{name + " + interval 'P1D' > ISO " + date, planpb.ArithOpType_Add, planpb.OpType_GreaterThan, &planpb.Interval{Days: 1}},
|
||||
{"iso " + date + " >= " + name + " - INTERVAL 'PT2H'", planpb.ArithOpType_Sub, planpb.OpType_LessEqual, &planpb.Interval{Hours: 2}},
|
||||
} {
|
||||
expr, err := ParseExpr(helper, tc.query, nil)
|
||||
require.NoError(t, err, tc.query)
|
||||
arith := expr.GetTimestamptzArithCompareExpr()
|
||||
require.NotNil(t, arith)
|
||||
requireContextualKeywordColumn(t, arith.GetTimestamptzColumn(), name)
|
||||
require.Equal(t, tc.arith, arith.GetArithOp())
|
||||
require.Equal(t, tc.op, arith.GetCompareOp())
|
||||
require.Equal(t, wantMicros, arith.GetCompareValue().GetInt64Val())
|
||||
require.Equal(t, tc.span.GetDays(), arith.GetInterval().GetDays())
|
||||
require.Equal(t, tc.span.GetHours(), arith.GetInterval().GetHours())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualKeywordsMalformedCalls(t *testing.T) {
|
||||
schema := newTestSchema(true)
|
||||
enableMatch(schema)
|
||||
helper, err := typeutil.CreateSchemaHelper(schema)
|
||||
require.NoError(t, err)
|
||||
for _, word := range contextualKeywordNames {
|
||||
for _, name := range []string{word, strings.ToUpper(word)} {
|
||||
// A keyword token followed by parentheses must never fall through
|
||||
// to the generic Call visitor and evade its dedicated validation.
|
||||
_, err := ParseExpr(helper, name+"()", nil)
|
||||
require.Error(t, err, name)
|
||||
}
|
||||
}
|
||||
for _, query := range []string{
|
||||
`text_match_fuzzy(VarCharField, "hello", typo=1)`,
|
||||
`text_match_fuzzy(VarCharField, "hello", max_edit_distance=3)`,
|
||||
`text_match(VarCharField, "hello", threshold=1)`,
|
||||
`match_least(struct_array, $[sub_int] > 0, count=1)`,
|
||||
`match_least(struct_array, $[sub_int] > 0, threshold=0)`,
|
||||
`match_all(struct_array, $[sub_int] > 0, threshold=1)`,
|
||||
`element_filter(struct_array, $[missing] > 0)`,
|
||||
`$[threshold] > 0`,
|
||||
`st_isvalid(GeometryField, "POINT(0 0)")`,
|
||||
`st_dwithin(GeometryField, "POINT(0 0)")`,
|
||||
`st_equals(Int64Field, "POINT(0 0)")`,
|
||||
`TimestamptzField > timezone '2025-01-01T00:00:00Z'`,
|
||||
`timezone '2025-01-01T00:00:00Z' < TimestamptzField`,
|
||||
`TimestamptzField + duration 'P1D' > ISO '2025-01-01T00:00:00Z'`,
|
||||
`TimestamptzField > ISO 'invalid-date'`,
|
||||
`ISO 'invalid-date' < TimestamptzField`,
|
||||
`TimestamptzField + INTERVAL '1D' > ISO '2025-01-01T00:00:00Z'`,
|
||||
`ISO '2025-01-01T00:00:00Z' < TimestamptzField + INTERVAL '1D'`,
|
||||
} {
|
||||
_, err := ParseExpr(helper, query, nil)
|
||||
require.Error(t, err, query)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContextualKeywordsTwoStageMatchesLL(t *testing.T) {
|
||||
corpus := []string{
|
||||
`text_match_fuzzy(text_match_fuzzy, "hello", max_edit_distance=1)`,
|
||||
`text_match(minimum_should_match, "hello", minimum_should_match=2)`,
|
||||
`st_equals(st_equals, "POINT(0 0)")`,
|
||||
`ST_DWITHIN(ST_DWITHIN, "POINT(0 0)", 5)`,
|
||||
`match_least(match_least, $[threshold] > 0, threshold=2)`,
|
||||
`element_filter(element_filter, $[iso] > 0)`,
|
||||
`iso > iso '2025-01-01T00:00:00Z'`,
|
||||
`ISO '2025-01-01T00:00:00Z' < ISO`,
|
||||
`interval + interval 'P1D' >= iso '2025-01-01T00:00:00Z'`,
|
||||
`ISO '2025-01-01T00:00:00Z' <= INTERVAL - INTERVAL 'PT2H'`,
|
||||
`text_match_fuzzy(VarCharField)`,
|
||||
`match_least(struct_array, $[threshold] > 0, count=2)`,
|
||||
`iso > timezone '2025-01-01T00:00:00Z'`,
|
||||
}
|
||||
for _, word := range contextualKeywordNames {
|
||||
for _, name := range []string{word, strings.ToUpper(word)} {
|
||||
corpus = append(corpus,
|
||||
name+" == 400", "5 < "+name+" <= 400", "400 >= "+name+" > 5",
|
||||
name+" is null", name+" == {"+name+"}", "custom_predicate("+name+", 1)",
|
||||
)
|
||||
}
|
||||
}
|
||||
for _, query := range corpus {
|
||||
llTree, llErr := parseTree(query, false)
|
||||
tsTree, tsErr := parseTree(query, true)
|
||||
require.Equal(t, llErr == nil, tsErr == nil, "%s: LL=%v, two-stage=%v", query, llErr, tsErr)
|
||||
if llErr == nil && tsErr == nil {
|
||||
require.Equal(t, llTree, tsTree, query)
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -219,6 +219,14 @@ func (v *BasePlanVisitor) VisitPower(ctx *PowerContext) interface{} {
|
||||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
||||
func (v *BasePlanVisitor) VisitFieldName(ctx *FieldNameContext) interface{} {
|
||||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
||||
func (v *BasePlanVisitor) VisitFieldReference(ctx *FieldReferenceContext) interface{} {
|
||||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
||||
func (v *BasePlanVisitor) VisitTextMatchOption(ctx *TextMatchOptionContext) interface{} {
|
||||
return v.VisitChildren(ctx)
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -166,6 +166,12 @@ type PlanVisitor interface {
|
||||
// Visit a parse tree produced by PlanParser#Power.
|
||||
VisitPower(ctx *PowerContext) interface{}
|
||||
|
||||
// Visit a parse tree produced by PlanParser#fieldName.
|
||||
VisitFieldName(ctx *FieldNameContext) interface{}
|
||||
|
||||
// Visit a parse tree produced by PlanParser#fieldReference.
|
||||
VisitFieldReference(ctx *FieldReferenceContext) interface{}
|
||||
|
||||
// Visit a parse tree produced by PlanParser#textMatchOption.
|
||||
VisitTextMatchOption(ctx *TextMatchOptionContext) interface{}
|
||||
}
|
||||
|
||||
@@ -966,7 +966,7 @@ func (v *ParserVisitor) parseTextMatchOperand(identifier string, queryExpr parse
|
||||
}
|
||||
|
||||
func (v *ParserVisitor) VisitTextMatch(ctx *parser.TextMatchContext) interface{} {
|
||||
identifier := ctx.Identifier().GetText()
|
||||
identifier := ctx.FieldName().GetText()
|
||||
columnInfo, value, placeholder, isTemplate, err := v.parseTextMatchOperand(
|
||||
identifier, ctx.Expr(), "text match", "text_match query")
|
||||
if err != nil {
|
||||
@@ -1005,7 +1005,7 @@ func (v *ParserVisitor) VisitTextMatch(ctx *parser.TextMatchContext) interface{}
|
||||
}
|
||||
|
||||
func (v *ParserVisitor) VisitTextMatchFuzzy(ctx *parser.TextMatchFuzzyContext) interface{} {
|
||||
identifier := ctx.Identifier(0).GetText()
|
||||
identifier := ctx.FieldName().GetText()
|
||||
columnInfo, value, placeholder, isTemplate, err := v.parseTextMatchOperand(
|
||||
identifier, ctx.Expr(), "text match fuzzy", "text_match_fuzzy query")
|
||||
if err != nil {
|
||||
@@ -1015,7 +1015,7 @@ func (v *ParserVisitor) VisitTextMatchFuzzy(ctx *parser.TextMatchFuzzyContext) i
|
||||
// The option name is a soft keyword (a plain identifier) so that a scalar
|
||||
// field literally named "max_edit_distance" is still usable elsewhere in a
|
||||
// filter; only accept the expected option name here.
|
||||
optionName := ctx.Identifier(1).GetText()
|
||||
optionName := ctx.Identifier().GetText()
|
||||
if !strings.EqualFold(optionName, "max_edit_distance") {
|
||||
return merr.WrapErrParameterInvalidMsg(
|
||||
"invalid option %q for text_match_fuzzy, expected max_edit_distance", optionName)
|
||||
@@ -1069,7 +1069,7 @@ func (v *ParserVisitor) VisitTextMatchOption(ctx *parser.TextMatchOptionContext)
|
||||
}
|
||||
|
||||
func (v *ParserVisitor) VisitPhraseMatch(ctx *parser.PhraseMatchContext) interface{} {
|
||||
identifier := ctx.Identifier().GetText()
|
||||
identifier := ctx.FieldName().GetText()
|
||||
columnInfo, value, placeholder, isTemplate, err := v.parseTextMatchOperand(
|
||||
identifier, ctx.Expr(0), "phrase match", "phrase_match query")
|
||||
if err != nil {
|
||||
@@ -1395,7 +1395,7 @@ func (v *ParserVisitor) getColumnInfoFromStructIndexField(identifier string) (*p
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (v *ParserVisitor) getChildColumnInfo(identifier, child, structSubField, structIndexField antlr.TerminalNode) (*planpb.ColumnInfo, error) {
|
||||
func (v *ParserVisitor) getChildColumnInfo(identifier parser.IFieldNameContext, child, structSubField, structIndexField antlr.TerminalNode) (*planpb.ColumnInfo, error) {
|
||||
if identifier != nil {
|
||||
childExpr, err := v.translateIdentifier(identifier.GetText())
|
||||
if err != nil {
|
||||
@@ -1440,7 +1440,7 @@ func (v *ParserVisitor) getStructArrayParentColumnInfo(fieldName string) (*planp
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func (v *ParserVisitor) getNullExprColumnInfo(identifier, child antlr.TerminalNode) (*planpb.ColumnInfo, error) {
|
||||
func (v *ParserVisitor) getNullExprColumnInfo(identifier parser.IFieldNameContext, child antlr.TerminalNode) (*planpb.ColumnInfo, error) {
|
||||
if identifier != nil {
|
||||
// try struct first
|
||||
if columnInfo, ok, err := v.getStructArrayParentColumnInfo(identifier.GetText()); ok || err != nil {
|
||||
@@ -1506,21 +1506,22 @@ func (v *ParserVisitor) VisitMembershipMatchWithOption(ctx *parser.MembershipMat
|
||||
var fieldText string
|
||||
var fieldInfo *planpb.ColumnInfo
|
||||
var err error
|
||||
fieldText = ctx.GetField().GetText()
|
||||
switch ctx.GetField().GetTokenType() {
|
||||
case parser.PlanParserIdentifier, parser.PlanParserMeta:
|
||||
field := ctx.GetField()
|
||||
fieldText = field.GetText()
|
||||
switch {
|
||||
case field.FieldName() != nil || field.Meta() != nil:
|
||||
var fieldExpr *ExprWithType
|
||||
fieldExpr, err = v.translateIdentifier(fieldText)
|
||||
if err == nil {
|
||||
fieldInfo = toColumnInfo(fieldExpr)
|
||||
}
|
||||
case parser.PlanParserJSONIdentifier:
|
||||
case field.JSONIdentifier() != nil:
|
||||
fieldInfo, err = v.getColumnInfoFromJSONIdentifier(fieldText)
|
||||
case parser.PlanParserStructFieldIdentifier:
|
||||
case field.StructFieldIdentifier() != nil:
|
||||
fieldInfo, err = v.getColumnInfoFromStructField(fieldText)
|
||||
case parser.PlanParserStructIndexFieldIdentifier:
|
||||
case field.StructIndexFieldIdentifier() != nil:
|
||||
fieldInfo, err = v.getColumnInfoFromStructIndexField(fieldText)
|
||||
case parser.PlanParserStructSubFieldIdentifier:
|
||||
case field.StructSubFieldIdentifier() != nil:
|
||||
fieldInfo, err = v.getColumnInfoFromStructSubField(fieldText)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -1546,7 +1547,7 @@ func (v *ParserVisitor) VisitMembershipMatchWithOption(ctx *parser.MembershipMat
|
||||
// VisitRange translates expr to range plan.
|
||||
func (v *ParserVisitor) VisitRange(ctx *parser.RangeContext) interface{} {
|
||||
columnInfo, err := v.getChildColumnInfo(
|
||||
ctx.Identifier(),
|
||||
ctx.FieldName(),
|
||||
ctx.JSONIdentifier(),
|
||||
ctx.StructSubFieldIdentifier(),
|
||||
ctx.StructIndexFieldIdentifier(),
|
||||
@@ -1626,7 +1627,7 @@ func (v *ParserVisitor) VisitRange(ctx *parser.RangeContext) interface{} {
|
||||
// VisitReverseRange parses the expression like "1 > a > 0".
|
||||
func (v *ParserVisitor) VisitReverseRange(ctx *parser.ReverseRangeContext) interface{} {
|
||||
columnInfo, err := v.getChildColumnInfo(
|
||||
ctx.Identifier(),
|
||||
ctx.FieldName(),
|
||||
ctx.JSONIdentifier(),
|
||||
ctx.StructSubFieldIdentifier(),
|
||||
ctx.StructIndexFieldIdentifier(),
|
||||
@@ -2605,7 +2606,7 @@ func (v *ParserVisitor) VisitEmptyArray(ctx *parser.EmptyArrayContext) interface
|
||||
}
|
||||
|
||||
func (v *ParserVisitor) VisitIsNotNull(ctx *parser.IsNotNullContext) interface{} {
|
||||
column, err := v.getNullExprColumnInfo(ctx.Identifier(), ctx.JSONIdentifier())
|
||||
column, err := v.getNullExprColumnInfo(ctx.FieldName(), ctx.JSONIdentifier())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2648,7 +2649,7 @@ func (v *ParserVisitor) VisitIsNotNull(ctx *parser.IsNotNullContext) interface{}
|
||||
}
|
||||
|
||||
func (v *ParserVisitor) VisitIsNull(ctx *parser.IsNullContext) interface{} {
|
||||
column, err := v.getNullExprColumnInfo(ctx.Identifier(), ctx.JSONIdentifier())
|
||||
column, err := v.getNullExprColumnInfo(ctx.FieldName(), ctx.JSONIdentifier())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2940,15 +2941,15 @@ func (v *ParserVisitor) VisitArrayLength(ctx *parser.ArrayLengthContext) interfa
|
||||
Nullable: field.GetNullable(),
|
||||
}
|
||||
} else {
|
||||
if ctx.Identifier() != nil {
|
||||
if parentColumnInfo, ok, parentErr := v.getStructArrayParentColumnInfo(ctx.Identifier().GetText()); ok || parentErr != nil {
|
||||
if ctx.FieldName() != nil {
|
||||
if parentColumnInfo, ok, parentErr := v.getStructArrayParentColumnInfo(ctx.FieldName().GetText()); ok || parentErr != nil {
|
||||
columnInfo = parentColumnInfo
|
||||
err = parentErr
|
||||
}
|
||||
}
|
||||
if columnInfo == nil && err == nil {
|
||||
columnInfo, err = v.getChildColumnInfo(
|
||||
ctx.Identifier(),
|
||||
ctx.FieldName(),
|
||||
ctx.JSONIdentifier(),
|
||||
ctx.StructSubFieldIdentifier(),
|
||||
nil,
|
||||
@@ -2994,7 +2995,7 @@ func (v *ParserVisitor) VisitTemplateVariable(ctx *parser.TemplateVariableContex
|
||||
Expr: &planpb.Expr_ValueExpr{
|
||||
ValueExpr: &planpb.ValueExpr{
|
||||
Value: nil,
|
||||
TemplateVariableName: ctx.Identifier().GetText(),
|
||||
TemplateVariableName: ctx.FieldName().GetText(),
|
||||
},
|
||||
},
|
||||
IsTemplate: true,
|
||||
@@ -3003,7 +3004,7 @@ func (v *ParserVisitor) VisitTemplateVariable(ctx *parser.TemplateVariableContex
|
||||
}
|
||||
|
||||
func (v *ParserVisitor) VisitSpatialBinary(ctx *parser.SpatialBinaryContext) interface{} {
|
||||
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
|
||||
childExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -3063,7 +3064,7 @@ func (v *ParserVisitor) VisitSpatialBinary(ctx *parser.SpatialBinaryContext) int
|
||||
}
|
||||
|
||||
func (v *ParserVisitor) VisitSTIsValid(ctx *parser.STIsValidContext) interface{} {
|
||||
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
|
||||
childExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -3089,7 +3090,7 @@ func (v *ParserVisitor) VisitSTIsValid(ctx *parser.STIsValidContext) interface{}
|
||||
|
||||
func (v *ParserVisitor) VisitSTDWithin(ctx *parser.STDWithinContext) interface{} {
|
||||
// Process the geometry field identifier
|
||||
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
|
||||
childExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -3173,8 +3174,8 @@ func (v *ParserVisitor) VisitSTDWithin(ctx *parser.STDWithinContext) interface{}
|
||||
// 2. Slow Path: If an INTERVAL exists, it generates a TimestamptzArithCompareExpr
|
||||
// for specialized arithmetic evaluation.
|
||||
func (v *ParserVisitor) VisitTimestamptzCompareForward(ctx *parser.TimestamptzCompareForwardContext) interface{} {
|
||||
colExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
|
||||
identifier := ctx.Identifier().Accept(v)
|
||||
colExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
|
||||
identifier := ctx.FieldName().GetText()
|
||||
if err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("can not translate identifier: %s", identifier)
|
||||
}
|
||||
@@ -3247,8 +3248,8 @@ func (v *ParserVisitor) VisitTimestamptzCompareForward(ctx *parser.TimestamptzCo
|
||||
// 3. Slow Path: For complex expressions involving INTERVAL, it produces a
|
||||
// TimestamptzArithCompareExpr with the reversed operator.
|
||||
func (v *ParserVisitor) VisitTimestamptzCompareReverse(ctx *parser.TimestamptzCompareReverseContext) interface{} {
|
||||
colExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
|
||||
identifier := ctx.Identifier().GetText()
|
||||
colExpr, err := v.translateIdentifier(ctx.FieldName().GetText())
|
||||
identifier := ctx.FieldName().GetText()
|
||||
if err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("can not translate identifier: %s", identifier)
|
||||
}
|
||||
@@ -3364,7 +3365,7 @@ func (v *ParserVisitor) VisitElementFilter(ctx *parser.ElementFilterContext) int
|
||||
}
|
||||
|
||||
// Get struct array field name (first parameter)
|
||||
arrayFieldName := ctx.Identifier().GetText()
|
||||
arrayFieldName := ctx.FieldName().GetText()
|
||||
|
||||
// Set current context for element expression parsing
|
||||
v.currentStructArrayField = arrayFieldName
|
||||
@@ -3527,7 +3528,7 @@ func (v *ParserVisitor) parseMatchExpr(structArrayFieldName string, exprCtx pars
|
||||
// VisitMatchSimple handles MATCH_ALL and MATCH_ANY expressions
|
||||
// Syntax: MATCH_ALL/MATCH_ANY(structArrayField, $[intField] == 1 && $[strField] == "aaa")
|
||||
func (v *ParserVisitor) VisitMatchSimple(ctx *parser.MatchSimpleContext) interface{} {
|
||||
structArrayFieldName := ctx.Identifier().GetText()
|
||||
structArrayFieldName := ctx.FieldName().GetText()
|
||||
var matchType planpb.MatchType
|
||||
var opName string
|
||||
switch ctx.GetOp().GetTokenType() {
|
||||
@@ -3546,7 +3547,7 @@ func (v *ParserVisitor) VisitMatchSimple(ctx *parser.MatchSimpleContext) interfa
|
||||
// VisitMatchThreshold handles MATCH_LEAST, MATCH_MOST, and MATCH_EXACT expressions
|
||||
// Syntax: MATCH_LEAST/MATCH_MOST/MATCH_EXACT(structArrayField, $[intField] == 1, threshold=N)
|
||||
func (v *ParserVisitor) VisitMatchThreshold(ctx *parser.MatchThresholdContext) interface{} {
|
||||
structArrayFieldName := ctx.Identifier().GetText()
|
||||
structArrayFieldName := ctx.FieldName().GetText()
|
||||
|
||||
countStr := ctx.IntegerConstant().GetText()
|
||||
count, err := strconv.ParseInt(countStr, 10, 64)
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/metrics"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/requestutil"
|
||||
)
|
||||
|
||||
var contextualFieldKeywordsForTest = []string{
|
||||
"text_match_fuzzy", "match_all", "match_any", "match_least", "match_most", "match_exact",
|
||||
"iso", "interval", "minimum_should_match", "threshold", "element_filter",
|
||||
"st_equals", "st_touches", "st_overlaps", "st_crosses", "st_contains",
|
||||
"st_intersects", "st_within", "st_dwithin", "st_isvalid",
|
||||
}
|
||||
|
||||
func fieldKeywordCaseVariantsForTest(keyword string) []string {
|
||||
names := make([]string, 0, 1<<len(keyword))
|
||||
for mask := 0; mask < 1<<len(keyword); mask++ {
|
||||
name := []byte(keyword)
|
||||
for i := range name {
|
||||
if mask&(1<<i) != 0 {
|
||||
name[i] -= 'a' - 'A'
|
||||
}
|
||||
}
|
||||
names = append(names, string(name))
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func keywordValidationSchemaForTest() *schemapb.CollectionSchema {
|
||||
return &schemapb.CollectionSchema{
|
||||
Name: "keyword_validation",
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 100, Name: "pk", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
|
||||
{
|
||||
FieldID: 101, Name: "embedding", DataType: schemapb.DataType_FloatVector,
|
||||
TypeParams: []*commonpb.KeyValuePair{{Key: common.DimKey, Value: "4"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func keywordArrayFieldForTest(name string) *schemapb.FieldSchema {
|
||||
return &schemapb.FieldSchema{
|
||||
Name: name, DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int64,
|
||||
TypeParams: []*commonpb.KeyValuePair{{Key: common.MaxCapacityKey, Value: "16"}},
|
||||
}
|
||||
}
|
||||
|
||||
func keywordStructFieldForTest(name string) *schemapb.StructArrayFieldSchema {
|
||||
return &schemapb.StructArrayFieldSchema{
|
||||
Name: name, Nullable: true,
|
||||
Fields: []*schemapb.FieldSchema{keywordArrayFieldForTest("values")},
|
||||
}
|
||||
}
|
||||
|
||||
func createKeywordSchemaForTest(t *testing.T, schema *schemapb.CollectionSchema) error {
|
||||
t.Helper()
|
||||
bytes, err := proto.Marshal(schema)
|
||||
require.NoError(t, err)
|
||||
task := &createCollectionTask{
|
||||
CreateCollectionRequest: &milvuspb.CreateCollectionRequest{
|
||||
Base: &commonpb.MsgBase{}, CollectionName: schema.GetName(), Schema: bytes,
|
||||
},
|
||||
}
|
||||
return task.PreExecute(context.Background())
|
||||
}
|
||||
|
||||
func requireKeywordValidationForTest(t *testing.T, err error, reserved bool) {
|
||||
t.Helper()
|
||||
if reserved {
|
||||
requireInputValidationErrorForTest(t, err, merr.ErrFieldInvalidName, 1701)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
|
||||
func requireInputValidationErrorForTest(t *testing.T, err, expectedErr error, expectedCode int32) {
|
||||
t.Helper()
|
||||
require.ErrorIs(t, err, expectedErr)
|
||||
require.Equal(t, expectedCode, merr.Code(err))
|
||||
require.Equal(t, merr.InputError, merr.GetErrorType(err))
|
||||
|
||||
// Create/Add RPCs project the task error with merr.Status; REST then
|
||||
// reconstructs it with merr.Error. Classification must survive both.
|
||||
status := merr.Status(err)
|
||||
require.Equal(t, expectedCode, status.GetCode())
|
||||
require.Equal(t, err.Error(), status.GetReason())
|
||||
require.False(t, status.GetRetriable())
|
||||
require.Equal(t, "true", status.GetExtraInfo()[merr.InputErrorFlagKey])
|
||||
label, cause := requestutil.ParseMetricLabel(status, nil)
|
||||
require.Equal(t, metrics.FailLabel, label)
|
||||
require.Equal(t, metrics.CauseUser, cause)
|
||||
|
||||
roundTrip := merr.Error(status)
|
||||
require.ErrorIs(t, roundTrip, expectedErr)
|
||||
require.Equal(t, expectedCode, merr.Code(roundTrip))
|
||||
require.Equal(t, merr.InputError, merr.GetErrorType(roundTrip))
|
||||
require.False(t, merr.IsRetryableErr(roundTrip))
|
||||
}
|
||||
|
||||
func TestFieldNameKeywordPolicy(t *testing.T) {
|
||||
check := func(t *testing.T, name string, reserved bool) {
|
||||
schema := keywordValidationSchemaForTest()
|
||||
schema.Fields = append(schema.Fields, &schemapb.FieldSchema{
|
||||
Name: name, DataType: schemapb.DataType_Int64,
|
||||
})
|
||||
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, schema), reserved)
|
||||
|
||||
// The public add-field path must enforce the same policy as creation.
|
||||
field, err := proto.Marshal(&schemapb.FieldSchema{
|
||||
Name: name, DataType: schemapb.DataType_Int64, Nullable: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
add := &addCollectionFieldTask{
|
||||
oldSchema: keywordValidationSchemaForTest(),
|
||||
AddCollectionFieldRequest: &milvuspb.AddCollectionFieldRequest{Schema: field},
|
||||
}
|
||||
requireKeywordValidationForTest(t, add.PreExecute(context.Background()), reserved)
|
||||
|
||||
// Struct sub-fields already use the shared field-name validator.
|
||||
requireKeywordValidationForTest(t, ValidateFieldsInStruct(
|
||||
keywordArrayFieldForTest(name), keywordValidationSchemaForTest()), reserved)
|
||||
|
||||
// Check sub-field failures through actual create/add struct tasks too,
|
||||
// before the names are transformed to profile[field].
|
||||
structField := keywordStructFieldForTest("profile")
|
||||
structField.Fields[0].Name = name
|
||||
structSchema := keywordValidationSchemaForTest()
|
||||
structSchema.StructArrayFields = []*schemapb.StructArrayFieldSchema{structField}
|
||||
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, structSchema), reserved)
|
||||
addStruct := &addCollectionStructFieldTask{
|
||||
oldSchema: keywordValidationSchemaForTest(),
|
||||
AddCollectionStructFieldRequest: &milvuspb.AddCollectionStructFieldRequest{
|
||||
StructArrayFieldSchema: structField,
|
||||
},
|
||||
}
|
||||
requireKeywordValidationForTest(t, addStruct.PreExecute(context.Background()), reserved)
|
||||
}
|
||||
|
||||
for _, keyword := range []string{"like", "and", "or", "not", "in", "null"} {
|
||||
for _, name := range fieldKeywordCaseVariantsForTest(keyword) {
|
||||
t.Run(name, func(t *testing.T) { check(t, name, true) })
|
||||
}
|
||||
}
|
||||
for _, keyword := range contextualFieldKeywordsForTest {
|
||||
for _, name := range []string{keyword, strings.ToUpper(keyword)} {
|
||||
t.Run(name, func(t *testing.T) { check(t, name, false) })
|
||||
}
|
||||
}
|
||||
t.Run("ordinary_field", func(t *testing.T) { check(t, "ordinary_field", false) })
|
||||
for _, name := range []string{"", "1field", "bad name", "a-b", strings.Repeat("a", 256)} {
|
||||
t.Run("invalid_"+name, func(t *testing.T) { check(t, name, true) })
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructArrayFieldKeywordPolicy(t *testing.T) {
|
||||
check := func(t *testing.T, name string, reserved bool) {
|
||||
schema := keywordValidationSchemaForTest()
|
||||
schema.StructArrayFields = []*schemapb.StructArrayFieldSchema{keywordStructFieldForTest(name)}
|
||||
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, schema), reserved)
|
||||
|
||||
add := &addCollectionStructFieldTask{
|
||||
oldSchema: keywordValidationSchemaForTest(),
|
||||
AddCollectionStructFieldRequest: &milvuspb.AddCollectionStructFieldRequest{
|
||||
StructArrayFieldSchema: keywordStructFieldForTest(name),
|
||||
},
|
||||
}
|
||||
requireKeywordValidationForTest(t, add.PreExecute(context.Background()), reserved)
|
||||
}
|
||||
|
||||
for _, keyword := range []string{"like", "and", "or", "not", "in", "null"} {
|
||||
for _, name := range fieldKeywordCaseVariantsForTest(keyword) {
|
||||
t.Run(name, func(t *testing.T) { check(t, name, true) })
|
||||
}
|
||||
}
|
||||
for _, keyword := range contextualFieldKeywordsForTest {
|
||||
for _, name := range []string{keyword, strings.ToUpper(keyword)} {
|
||||
t.Run(name, func(t *testing.T) { check(t, name, false) })
|
||||
}
|
||||
}
|
||||
t.Run("ordinary_struct", func(t *testing.T) { check(t, "profile", false) })
|
||||
for _, name := range []string{"", "1field", "bad name", "$meta", "a-b", strings.Repeat("a", 256)} {
|
||||
t.Run("invalid_"+name, func(t *testing.T) { check(t, name, true) })
|
||||
}
|
||||
t.Run("parent_name_is_checked_before_children", func(t *testing.T) {
|
||||
err := ValidateStructArrayField(&schemapb.StructArrayFieldSchema{Name: "AND"}, keywordValidationSchemaForTest())
|
||||
requireKeywordValidationForTest(t, err, true)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFieldNameErrorClassification(t *testing.T) {
|
||||
// Every invalid field name originates from user-supplied schema input.
|
||||
// Factory and validation paths must agree on the wire and metric cause.
|
||||
factoryErr := merr.WrapErrFieldNameInvalid("Like", "Invalid field name: Like. Like is keyword in milvus.")
|
||||
requireKeywordValidationForTest(t, factoryErr, true)
|
||||
validationErr := validateFieldName("Like")
|
||||
requireKeywordValidationForTest(t, validationErr, true)
|
||||
require.Equal(t, factoryErr.Error(), validationErr.Error())
|
||||
require.Equal(t, merr.Status(factoryErr).GetErrorCode(), merr.Status(validationErr).GetErrorCode())
|
||||
|
||||
for _, name := range []string{common.RowIDFieldName, common.TimeStampFieldName, common.VirtualPKFieldName} {
|
||||
t.Run("reserved_"+name, func(t *testing.T) {
|
||||
for _, structField := range []bool{false, true} {
|
||||
schema := keywordValidationSchemaForTest()
|
||||
if structField {
|
||||
schema.StructArrayFields = []*schemapb.StructArrayFieldSchema{keywordStructFieldForTest(name)}
|
||||
} else {
|
||||
schema.Fields = append(schema.Fields, &schemapb.FieldSchema{Name: name, DataType: schemapb.DataType_Int64})
|
||||
}
|
||||
requireKeywordValidationForTest(t, validateReservedFieldNames(schema), true)
|
||||
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, schema), true)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("meta_keeps_existing_api_error_codes", func(t *testing.T) {
|
||||
// Creation and direct name validation reject '$' with 1701. AddField
|
||||
// already rejects system fields earlier with 1100; preserve that code.
|
||||
requireKeywordValidationForTest(t, validateFieldName(common.MetaFieldName), true)
|
||||
schema := keywordValidationSchemaForTest()
|
||||
schema.Fields = append(schema.Fields, &schemapb.FieldSchema{
|
||||
Name: common.MetaFieldName, DataType: schemapb.DataType_Int64,
|
||||
})
|
||||
requireKeywordValidationForTest(t, createKeywordSchemaForTest(t, schema), true)
|
||||
|
||||
field, err := proto.Marshal(&schemapb.FieldSchema{
|
||||
Name: common.MetaFieldName, DataType: schemapb.DataType_Int64, Nullable: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
add := &addCollectionFieldTask{
|
||||
oldSchema: keywordValidationSchemaForTest(),
|
||||
AddCollectionFieldRequest: &milvuspb.AddCollectionFieldRequest{Schema: field},
|
||||
}
|
||||
requireInputValidationErrorForTest(t, add.PreExecute(context.Background()), merr.ErrParameterInvalid, 1100)
|
||||
})
|
||||
}
|
||||
@@ -772,7 +772,7 @@ func validateAddStructFieldRequest(schema *schemapb.CollectionSchema, structFiel
|
||||
if funcutil.SliceContain([]string{common.RowIDFieldName, common.TimeStampFieldName, common.MetaFieldName, common.NamespaceFieldName, common.VirtualPKFieldName}, structFieldSchema.GetName()) {
|
||||
return merr.WrapErrParameterInvalidMsg("not support to add system field, field name = %s", structFieldSchema.GetName())
|
||||
}
|
||||
if err := ValidateStructArrayField(structFieldSchema, schema); err != nil {
|
||||
if err := validateStructArrayField(structFieldSchema, schema); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAddStructSubFieldProperties(structFieldSchema); err != nil {
|
||||
|
||||
@@ -324,6 +324,7 @@ func validatePartitionTag(partitionTag string, strictCheck bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateFieldName validates names from user schema requests.
|
||||
func validateFieldName(fieldName string) error {
|
||||
fieldName = strings.TrimSpace(fieldName)
|
||||
|
||||
@@ -822,6 +823,15 @@ func validateStructArrayFieldMaxCapacity(structArrayField *schemapb.StructArrayF
|
||||
// ValidateStructArrayField validates the struct array field schema.
|
||||
// When the struct is nullable, sub-field schemas are mutated in-place to set Nullable=true.
|
||||
func ValidateStructArrayField(structArrayField *schemapb.StructArrayFieldSchema, schema *schemapb.CollectionSchema) error {
|
||||
if err := validateFieldName(structArrayField.GetName()); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateStructArrayField(structArrayField, schema)
|
||||
}
|
||||
|
||||
// validateStructArrayField validates the remaining schema after the parent name
|
||||
// has been checked. Add-field validation checks the name before reserved fields.
|
||||
func validateStructArrayField(structArrayField *schemapb.StructArrayFieldSchema, schema *schemapb.CollectionSchema) error {
|
||||
if len(structArrayField.Fields) == 0 {
|
||||
return merr.WrapErrParameterInvalidMsg("struct array field %s has no sub-fields", structArrayField.Name)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ package rootcoord
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -32,6 +33,24 @@ import (
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
func TestValidateAddedStructFieldNameErrorClassification(t *testing.T) {
|
||||
for _, name := range []string{"", strings.Repeat("a", Params.ProxyCfg.MaxNameLength.GetAsInt()+1), "1field", "bad name", "AND"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := validateAddedStructFieldName(name)
|
||||
require.ErrorIs(t, err, merr.ErrFieldInvalidName)
|
||||
require.Equal(t, merr.InputError, merr.GetErrorType(err))
|
||||
status := merr.Status(err)
|
||||
require.Equal(t, int32(1701), status.GetCode())
|
||||
require.Equal(t, err.Error(), status.GetReason())
|
||||
require.Equal(t, "true", status.GetExtraInfo()[merr.InputErrorFlagKey])
|
||||
require.False(t, status.GetRetriable())
|
||||
roundTrip := merr.Error(status)
|
||||
require.ErrorIs(t, roundTrip, merr.ErrFieldInvalidName)
|
||||
require.Equal(t, merr.InputError, merr.GetErrorType(roundTrip))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDDLCallbacksAlterCollectionAddStructField(t *testing.T) {
|
||||
core := initStreamingSystemAndCore(t)
|
||||
|
||||
|
||||
+10
-6
@@ -40,14 +40,18 @@ var FieldNameKeywords = map[string]struct{}{
|
||||
}
|
||||
|
||||
// IsFieldNameKeyword reports whether fieldName is a reserved word that cannot be
|
||||
// used as a field name. `null` is matched case-insensitively — any casing of NULL
|
||||
// is rejected — to stay consistent with the expression parser, which rejects a
|
||||
// bare NULL literal regardless of casing (issue #50882). It is therefore handled
|
||||
// here rather than enumerated in FieldNameKeywords; the other keywords match the
|
||||
// exact casings listed there.
|
||||
// used as a field name. Core operators (like, and, or, not, in) and null are
|
||||
// reserved in every casing. Contextual keywords such as iso and interval stay
|
||||
// usable as field names; other reserved words retain the exact casings listed
|
||||
// in FieldNameKeywords.
|
||||
func IsFieldNameKeyword(fieldName string) bool {
|
||||
if _, ok := FieldNameKeywords[fieldName]; ok {
|
||||
return true
|
||||
}
|
||||
return strings.EqualFold(fieldName, "null")
|
||||
switch strings.ToLower(fieldName) {
|
||||
case "like", "and", "or", "not", "in", "null":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -11,3 +12,51 @@ func TestIsFieldNameKeywordRejectsNullCaseInsensitive(t *testing.T) {
|
||||
assert.True(t, IsFieldNameKeyword(name), name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsFieldNameKeywordCoreOperators(t *testing.T) {
|
||||
for _, keyword := range []string{"like", "and", "or", "not", "in", "null"} {
|
||||
// Enumerate every ASCII casing so title case and mixed case cannot
|
||||
// bypass the policy while lowercase and uppercase are blocked.
|
||||
for mask := 0; mask < 1<<len(keyword); mask++ {
|
||||
name := []byte(keyword)
|
||||
for i := range name {
|
||||
if mask&(1<<i) != 0 {
|
||||
name[i] -= 'a' - 'A'
|
||||
}
|
||||
}
|
||||
t.Run(string(name), func(t *testing.T) {
|
||||
assert.True(t, IsFieldNameKeyword(string(name)))
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsFieldNameKeywordContextualWords(t *testing.T) {
|
||||
for _, keyword := range []string{
|
||||
"text_match_fuzzy", "match_all", "match_any", "match_least", "match_most", "match_exact",
|
||||
"iso", "interval", "minimum_should_match", "threshold", "element_filter",
|
||||
"st_equals", "st_touches", "st_overlaps", "st_crosses", "st_contains",
|
||||
"st_intersects", "st_within", "st_dwithin", "st_isvalid",
|
||||
} {
|
||||
for _, name := range []string{keyword, strings.ToUpper(keyword)} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
assert.False(t, IsFieldNameKeyword(name))
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsFieldNameKeywordExistingPolicy(t *testing.T) {
|
||||
for name := range FieldNameKeywords {
|
||||
assert.True(t, IsFieldNameKeyword(name), name)
|
||||
}
|
||||
// Unrelated operators and function names keep their existing exact-case
|
||||
// policy. Soft option names and names merely containing a keyword stay valid.
|
||||
for _, name := range []string{
|
||||
"Exists", "Text_Match", "Phrase_Match", "Random_Sample", "Array_Length", "Json_Contains",
|
||||
"max_edit_distance", "membership_match", "type", "bloom", "roaring",
|
||||
"null_value", "NOTES", "like_count", "in_stock",
|
||||
} {
|
||||
assert.False(t, IsFieldNameKeyword(name), name)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,6 +117,7 @@ func TestSentinelErrorTypeClassification(t *testing.T) {
|
||||
"IncorrectParameterFormat": ErrIncorrectParameterFormat,
|
||||
"MissingRequiredParameters": ErrMissingRequiredParameters,
|
||||
"InvalidInsertData": ErrInvalidInsertData,
|
||||
"FieldInvalidName": ErrFieldInvalidName,
|
||||
}
|
||||
for name, err := range inputSentinels {
|
||||
assert.Equal(t, InputError, GetErrorType(err), "%s should be InputError", name)
|
||||
@@ -153,6 +154,7 @@ func TestSentinelErrorTypeClassification(t *testing.T) {
|
||||
801, 802, // Database: num limit / invalid name
|
||||
1100, 1101, 1102, // Parameter: invalid / missing / too large
|
||||
1400, 1401, 1402, // Privilege
|
||||
1701, // FieldInvalidName
|
||||
1800, 1801, 1802, 1804, // Auth / parameter format / insert data
|
||||
2100, // ImportFailed
|
||||
2201, // QueryPlan
|
||||
|
||||
@@ -223,7 +223,7 @@ var (
|
||||
|
||||
// field related
|
||||
ErrFieldNotFound = newMilvusError("field not found", 1700, false) // SystemError by default; proxy boundaries stamp InputError where the field name is user-supplied (e.g. group_by/anns field), while result-assembly lookups stay system.
|
||||
ErrFieldInvalidName = newMilvusError("field name invalid", 1701, false)
|
||||
ErrFieldInvalidName = newMilvusError("field name invalid", 1701, false, WithErrorType(InputError))
|
||||
|
||||
// high-level restful api related
|
||||
ErrNeedAuthenticate = newMilvusError("user hasn't authenticated", 1800, false, WithErrorType(InputError))
|
||||
|
||||
Reference in New Issue
Block a user