mirror of
https://github.com/neuml/txtai.git
synced 2026-10-12 02:19:52 +00:00
Preserve wildcard compatibility and support explicit LIKE escapes
This commit is contained in:
@@ -81,6 +81,15 @@ Supports a `dict` with the parameters `cachelimit` and `cutoff`.
|
||||
|
||||
When `terms` is set to `True`, default parameters are used for the `cachelimit` and `cutoff`. Normally, these defaults are sufficient.
|
||||
|
||||
### Wildcard queries
|
||||
|
||||
Keyword queries containing `*` expand matching terms using SQL `LIKE`. Within these
|
||||
expressions, `%` matches any sequence and `_` matches one character. To match a literal
|
||||
percent sign, underscore or backslash, use `\%`, `\_` or `\\`, respectively.
|
||||
For example, `report\_*` matches `report_2026`, while `report_*` also matches `reportX2026`.
|
||||
Use a tokenizer that preserves these characters, such as `tokenizer: {whitespace: true}`,
|
||||
or pass pre-tokenized query terms. Queries without `*` retain exact-term matching.
|
||||
|
||||
## normalize
|
||||
```yaml
|
||||
normalize: boolean|str|dict
|
||||
|
||||
@@ -448,6 +448,7 @@ class Terms:
|
||||
Expands a list of terms using the following term expansion rules.
|
||||
|
||||
- Asterisks (*) for wildcard expressions
|
||||
- In wildcard expressions, SQL LIKE % and _ remain wildcards; backslash escapes literal characters
|
||||
|
||||
Args:
|
||||
terms: list of terms
|
||||
@@ -462,8 +463,7 @@ class Terms:
|
||||
if Terms.ASTERISK in term:
|
||||
# Require a prefix or suffix
|
||||
if term.replace(Terms.ASTERISK, "").strip():
|
||||
# Escape literal LIKE metacharacters before joining the wildcard segments.
|
||||
term = "%".join(part.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") for part in term.split(Terms.ASTERISK))
|
||||
term = term.replace(Terms.ASTERISK, "%")
|
||||
result = self.cursor.execute(Terms.WILDCARD_TERMS, [term])
|
||||
results.extend([t for t, in result])
|
||||
else:
|
||||
|
||||
@@ -164,19 +164,22 @@ class TestKeyword(unittest.TestCase):
|
||||
self.assertEqual(scoring.count(), 0)
|
||||
self.assertEqual(scoring.search("bear", 1), [])
|
||||
|
||||
def testWildcardLiteralCharacters(self):
|
||||
def testWildcardEscapedCharacters(self):
|
||||
"""
|
||||
Test that only asterisks act as wildcards in keyword queries
|
||||
Test explicit escaping while preserving SQL LIKE wildcard behavior
|
||||
"""
|
||||
|
||||
data = ["report_2026", "reportX2026", "rate%2026", "rateX2026", r"path\_2026", r"path\X2026", "path_2026", "plain2026", "plain2027"]
|
||||
queries = [
|
||||
("report_*", {0}),
|
||||
("rate%*", {2}),
|
||||
("*%2026", {2}),
|
||||
("*_2026", {0, 4, 6}),
|
||||
(r"path\_*", {4}),
|
||||
(r"path\*", {4, 5}),
|
||||
("report_*", {0, 1}),
|
||||
(r"report\_*", {0}),
|
||||
("rate%*", {2, 3}),
|
||||
(r"rate\%*", {2}),
|
||||
(r"*\%2026", {2}),
|
||||
("*_2026", set(range(8))),
|
||||
(r"*\_2026", {0, 4, 6}),
|
||||
(r"path\\\_*", {4}),
|
||||
(r"path\\*", {4, 5}),
|
||||
("plain*", {7, 8}),
|
||||
("pl**2026", {7}),
|
||||
("*2027", {8}),
|
||||
|
||||
Reference in New Issue
Block a user