Some more patching of dbwire

extra/dbwire/README.md

@@ -22,7 +22,7 @@ A wire protocol is shared across a whole family of products, so one client serve
 |-----------------|---------------------|---------|
 | `postgres.py`   | PostgreSQL v3       | PostgreSQL, CockroachDB, CrateDB, Redshift, Greenplum, Vertica |
 | `mysql.py`      | MySQL client/server | MySQL, MariaDB, TiDB, Aurora (MySQL), Percona |
-| `tds.py`        | TDS                 | Microsoft SQL Server, Sybase |
+| `tds.py`        | TDS 7.x             | Microsoft SQL Server |
 | `firebird.py`   | Firebird wire       | Firebird 3 / 4 / 5 |
 | `cubrid.py`     | CUBRID CAS          | CUBRID |
 | `clickhouse.py` | HTTP (TabSeparated) | ClickHouse and HTTP-compatible forks |
@@ -57,7 +57,9 @@ parameter binding, prepared statements, bulk load/`COPY`, or TLS. Notable per-pr
   `caching_sha2_password` authentication over a plaintext connection requires RSA/TLS and is not
   supported; use a `mysql_native_password` account for the dependency-free path.
 - **TDS** - cleartext login only. Servers that force encryption (for example Azure SQL Database)
-  require TLS and are not supported here; the native driver or SQLAlchemy tier covers those.
+  require TLS and are not supported here; the native driver or SQLAlchemy tier covers those. LOGIN7 is the
+  Microsoft dialect (TDS 7.x); Sybase ASE speaks TDS 5.0 (LOGINREC + its own token dialect), which is not
+  implemented yet - ASE `-d` goes through the pymssql/FreeTDS tier for now.
 - **Firebird** - SRP-256 (and SRP) authentication with ChaCha20 or RC4 wire encryption, as required by
   default on Firebird 3 and later. Legacy (pre-SRP) authentication is not implemented.
 - **CUBRID** - cleartext login over the CAS broker protocol. Large objects (BLOB/CLOB) are returned as

extra/dbwire/__init__.py

@@ -11,10 +11,12 @@ sqlmap's direct ('-d') connection when no native driver (and no SQLAlchemy) is i
 
 Design note: connectors speak a *wire protocol*, not a product, so a single client covers the whole
 compatible family - e.g. the PostgreSQL client also serves CockroachDB, CrateDB, Redshift and Greenplum;
-a MySQL client serves MariaDB/TiDB/Aurora; a TDS client serves MSSQL/Sybase. Each module exposes a small
-PEP 249 (DB-API 2.0) subset (connect(), Connection.cursor()/commit()/close(), Cursor.execute()/fetchall()).
+a MySQL client serves MariaDB/TiDB/Aurora. Each module exposes a small PEP 249 (DB-API 2.0) subset
+(connect(), Connection.cursor()/commit()/close(), Cursor.execute()/fetchall()).
 """
 
+import socket
+
 __version__ = "0.1"
 
 apilevel = "2.0"
@@ -71,6 +73,21 @@ def connection_lost(ex):
 
     return OperationalError("connection lost (%s)" % ex)
 
+def handshake_done(sock):
+    """
+    Drop the connect deadline, once the login exchange is over.
+
+    connect_timeout has to stay armed THROUGH the handshake, not just the TCP connect: a peer that
+    accepts the connection and then says nothing (a wrong port, a silent proxy, a dropping firewall) is
+    perfectly alive as far as keepalive() below is concerned, so an unbounded login read waits forever.
+    A query is the opposite case - see keepalive().
+    """
+
+    try:
+        sock.settimeout(None)
+    except Exception:
+        pass
+
 def keepalive(sock):
     """
     Ask the kernel to probe an idle connection, so a peer that dies without a FIN is eventually detected.
@@ -80,12 +97,29 @@ def keepalive(sock):
     failure being guarded against. Best-effort - the options are not portable everywhere.
     """
 
-    import socket as _socket
-
     try:
-        sock.setsockopt(_socket.SOL_SOCKET, _socket.SO_KEEPALIVE, 1)
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
         for name, value in (("TCP_KEEPIDLE", 60), ("TCP_KEEPINTVL", 10), ("TCP_KEEPCNT", 5)):
-            if hasattr(_socket, name):
-                sock.setsockopt(_socket.IPPROTO_TCP, getattr(_socket, name), value)
+            if hasattr(socket, name):
+                sock.setsockopt(socket.IPPROTO_TCP, getattr(socket, name), value)
     except Exception:
         pass
+
+def recvn(sock, n):
+    """
+    Read exactly n bytes off `sock`, or raise - every wire protocol here is framed, so a short read is a
+    desynchronized stream, not a smaller message.
+
+    Shared because it was five identical copies: a fix to the recv loop has to land once, not per module.
+    """
+
+    buf = b""
+    while len(buf) < n:
+        try:
+            chunk = sock.recv(n - len(buf))
+        except (socket.error, OSError) as ex:
+            raise connection_lost(ex)
+        if not chunk:
+            raise InterfaceError("connection closed by server")
+        buf += chunk
+    return buf

extra/dbwire/cubrid.py

@@ -24,7 +24,9 @@ from extra.dbwire import IntegrityError
 from extra.dbwire import NotSupportedError
 from extra.dbwire import OperationalError
 from extra.dbwire import connection_lost
+from extra.dbwire import handshake_done
 from extra.dbwire import keepalive
+from extra.dbwire import recvn
 from extra.dbwire import ProgrammingError
 
 _MAGIC = b"CUBRK"
@@ -261,23 +263,13 @@ class Connection(object):
         self._sock = None
 
     def _recvn(self, n):
-        buf = b""
-        while len(buf) < n:
-            try:
-                chunk = self._sock.recv(n - len(buf))
-            except (socket.error, OSError) as ex:
-                raise connection_lost(ex)
-            if not chunk:
-                raise InterfaceError("connection closed by server")
-            buf += chunk
-        return buf
+        return recvn(self._sock, n)
 
     def _open(self):
         # broker handshake (may redirect to a dedicated CAS worker port), then cleartext OPEN_DATABASE login
         try:
             sock = socket.create_connection((self._host, self._port), timeout=self._timeout)
             keepalive(sock)
-            sock.settimeout(None)
             sock.sendall(_MAGIC + struct.pack(">BB", _CLIENT_JDBC, _CAS_VERSION) + b"\x00\x00\x00")
             self._sock = sock
             (port,) = struct.unpack(">i", self._recvn(4))
@@ -287,21 +279,25 @@ class Connection(object):
                 self._safe_close()
                 sock = socket.create_connection((self._host, port), timeout=self._timeout)
                 keepalive(sock)
-                sock.settimeout(None)
                 self._sock = sock
+
+            login = self._fixed(self._database, 32) + self._fixed(self._user, 32) + self._fixed(self._password, 32)
+            login += b"\x00" * 532  # 512 extended-info + 20 reserved
+            self._sock.sendall(login)
+            reader = self._read_response()
+            reader.int()             # response_code (>=0; errors already raised in _read_response)
+            broker = reader.raw(8)
+            self._protocol_version = bytearray(broker)[4] & 0x3f
+            # enable auto-commit so each statement is independent (avoids the CAS keep-connection handshake dance)
+            self._call(_Writer(_FC_SET_DB_PARAMETER).arg_int(_PARAM_AUTO_COMMIT).arg_int(1))
         except (socket.error, socket.timeout) as ex:
             self._safe_close()
             raise OperationalError("could not connect to '%s:%s' (%s)" % (self._host, self._port, ex))
+        except Exception:   # a rejected login (or connection_lost() out of _recvn) is still ours to close
+            self._safe_close()
+            raise
 
-        login = self._fixed(self._database, 32) + self._fixed(self._user, 32) + self._fixed(self._password, 32)
-        login += b"\x00" * 532  # 512 extended-info + 20 reserved
-        self._sock.sendall(login)
-        reader = self._read_response()
-        reader.int()                 # response_code (>=0; errors already raised in _read_response)
-        broker = reader.raw(8)
-        self._protocol_version = bytearray(broker)[4] & 0x3f
-        # enable auto-commit so each statement is independent (avoids the CAS keep-connection handshake dance)
-        self._call(_Writer(_FC_SET_DB_PARAMETER).arg_int(_PARAM_AUTO_COMMIT).arg_int(1))
+        handshake_done(self._sock)
 
     @staticmethod
     def _fixed(value, length):
@@ -347,9 +343,9 @@ class Connection(object):
         text = message.lower()
         if any(k in text for k in ("unique", "duplicate", "foreign key", "constraint violat")):
             raise IntegrityError(message)
-        if any(k in text for k in ("syntax", "unknown class", "does not exist", "not found", "before ' '")):
-            raise ProgrammingError(message)
-        if any(k in text for k in ("cast", "conversion", "overflow", "truncat")):
+        if any(k in text for k in ("syntax", "unknown class", "does not exist", "not found", "before '")):
+            raise ProgrammingError(message)  # CUBRID points at the offending token as: before ' ,'y')'
+        if any(k in text for k in ("cast", "coerce", "conversion", "overflow", "truncat")):
             raise DataError(message)
         raise ProgrammingError(message)
 

extra/dbwire/firebird.py

@@ -27,7 +27,9 @@ from extra.dbwire import InterfaceError
 from extra.dbwire import NotSupportedError
 from extra.dbwire import OperationalError
 from extra.dbwire import connection_lost
+from extra.dbwire import handshake_done
 from extra.dbwire import keepalive
+from extra.dbwire import recvn
 
 # operation codes
 _op_connect = 1
@@ -296,16 +298,7 @@ class _Wire(object):
             raise connection_lost(ex)
 
     def _recv_raw(self, n):
-        buf = b""
-        while len(buf) < n:
-            try:
-                chunk = self._sock.recv(n - len(buf))
-            except (socket.error, OSError) as ex:
-                raise connection_lost(ex)
-            if not chunk:
-                raise InterfaceError("connection closed by server")
-            buf += chunk
-        return buf
+        return recvn(self._sock, n)
 
     def recv(self, n, align=False):
         # every length here comes off the wire (response buffers, status strings, per-value lengths): a
@@ -330,6 +323,38 @@ class _Wire(object):
         except Exception:
             pass
 
+def _parse_response(wire):
+    head = wire.recv(16)
+    handle = struct.unpack("!i", head[:4])[0]
+    object_id = head[4:12]
+    buf = wire.recv(struct.unpack("!i", head[12:16])[0], align=True)
+    _check_status(wire)
+    return handle, object_id, buf
+
+def _check_status(wire):
+    gds, message = set(), ""
+    n = wire.recv_int()
+    while n != _isc_arg_end:
+        if n == _isc_arg_gds:
+            gds_code = wire.recv_int()
+            if gds_code:
+                gds.add(gds_code)
+        elif n == _isc_arg_number:
+            message += " %d" % wire.recv_int()
+        elif n in (_isc_arg_string, _isc_arg_interpreted, _isc_arg_sql_state):
+            s = wire.recv(wire.recv_int(), align=True)
+            if n != _isc_arg_sql_state:
+                message += " " + s.decode("utf-8", "replace")
+        n = wire.recv_int()
+    if gds:
+        message = ("(remote) firebird error %s%s" % (sorted(gds), message)).strip()
+        if gds & _GDS_INTEGRITY:
+            raise IntegrityError(message)
+        if gds & _GDS_DATA:
+            raise DataError(message)
+        if _GDS_WARNING not in gds:
+            raise OperationalError(message)
+
 def _pack_int(v):
     return struct.pack("!i", v)
 
@@ -461,39 +486,7 @@ class Connection(object):
             op = self._wire.recv_int()
         if op != _op_response:
             raise OperationalError("unexpected Firebird operation %d" % op)
-        return self._parse_response()
-
-    def _parse_response(self):
-        head = self._wire.recv(16)
-        handle = struct.unpack("!i", head[:4])[0]
-        object_id = head[4:12]
-        buf = self._wire.recv(struct.unpack("!i", head[12:16])[0], align=True)
-        self._check_status()
-        return handle, object_id, buf
-
-    def _check_status(self):
-        gds, message = set(), ""
-        n = self._wire.recv_int()
-        while n != _isc_arg_end:
-            if n == _isc_arg_gds:
-                gds_code = self._wire.recv_int()
-                if gds_code:
-                    gds.add(gds_code)
-            elif n == _isc_arg_number:
-                message += " %d" % self._wire.recv_int()
-            elif n in (_isc_arg_string, _isc_arg_interpreted, _isc_arg_sql_state):
-                s = self._wire.recv(self._wire.recv_int(), align=True)
-                if n != _isc_arg_sql_state:
-                    message += " " + s.decode("utf-8", "replace")
-            n = self._wire.recv_int()
-        if gds:
-            message = ("(remote) firebird error %s%s" % (sorted(gds), message)).strip()
-            if gds & _GDS_INTEGRITY:
-                raise IntegrityError(message)
-            if gds & _GDS_DATA:
-                raise DataError(message)
-            if _GDS_WARNING not in gds:
-                raise OperationalError(message)
+        return _parse_response(self._wire)
 
     # ---- query ----
 
@@ -632,7 +625,7 @@ class Connection(object):
                 op = self._wire.recv_int()
             if op != _op_fetch_response:
                 if op == _op_response:
-                    self._parse_response()
+                    _parse_response(self._wire)
                 raise OperationalError("unexpected Firebird operation %d during fetch" % op)
             status = self._wire.recv_int()
             count = self._wire.recv_int()
@@ -765,7 +758,6 @@ def connect(host=None, port=3050, user=None, password=None, database=None, conne
     try:
         sock = socket.create_connection((host or "localhost", int(port or 3050)), timeout=connect_timeout)
         keepalive(sock)
-        sock.settimeout(None)
     except (socket.error, socket.timeout) as ex:
         raise OperationalError("could not connect to '%s:%s' (%s)" % (host, port, ex))
 
@@ -782,6 +774,7 @@ def connect(host=None, port=3050, user=None, password=None, database=None, conne
         _authenticate(wire, user, password, public_key, private_key)
         connection = Connection(wire, filename, user, password)
         _attach(connection, wire, user)
+        handshake_done(sock)
     except (DatabaseError, InterfaceError):
         wire.close()
         raise
@@ -805,7 +798,7 @@ def _authenticate(wire, user, password, public_key, private_key):
     if op == _op_reject:
         raise OperationalError("Firebird connection rejected")
     if op == _op_response:
-        Connection(wire, b"", user, password)._parse_response()  # will raise the server error
+        _parse_response(wire)  # will raise the server error
         raise OperationalError("Firebird connection rejected")
 
     wire.recv(12)  # accept block: protocol version / architecture / type (not needed once lazy-send is off)
@@ -858,7 +851,7 @@ def _read_response(wire, user, password):
         raise OperationalError("Firebird authentication failed")
     if op != _op_response:
         raise OperationalError("unexpected Firebird operation %d during login" % op)
-    return Connection(wire, b"", user, password)._parse_response()[2]
+    return _parse_response(wire)[2]
 
 def _attach(connection, wire, user):
     dpb = bytearray([_isc_dpb_version1])

extra/dbwire/monetdb.py

@@ -23,35 +23,25 @@ from extra.dbwire import InterfaceError
 from extra.dbwire import NotSupportedError
 from extra.dbwire import OperationalError
 from extra.dbwire import connection_lost
+from extra.dbwire import handshake_done
 from extra.dbwire import keepalive
+from extra.dbwire import recvn
 from extra.dbwire import ProgrammingError
 
 _MAX_BLOCK = 0xffff >> 1
 _MAX_MESSAGE_LENGTH = 0x40000000  # cap on a (re-assembled) response, to bound a hostile/corrupt stream
 
-def _recvn(sock, n):
-    buf = b""
-    while len(buf) < n:
-        try:
-            chunk = sock.recv(n - len(buf))
-        except (socket.error, OSError) as ex:
-            raise connection_lost(ex)
-        if not chunk:
-            raise InterfaceError("connection closed by server")
-        buf += chunk
-    return buf
-
 def _getblock(sock):
     # the block length is a 15-bit field, so bounding IT is pointless - a peer that never sets the last-flag
     # simply streams blocks forever. Bound the accumulated response instead (as the other wire modules do).
     chunks, total = [], 0
     while True:
-        (header,) = struct.unpack("<H", _recvn(sock, 2))
+        (header,) = struct.unpack("<H", recvn(sock, 2))
         length, last = header >> 1, header & 1
         total += length
         if total > _MAX_MESSAGE_LENGTH:
             raise InterfaceError("backend message too large (%d bytes)" % total)
-        chunks.append(_recvn(sock, length))
+        chunks.append(recvn(sock, length))
         if last:
             break
     return b"".join(chunks).decode("utf-8", "replace")
@@ -201,7 +191,6 @@ def connect(host=None, port=50000, user=None, password=None, database=None, conn
     try:
         sock = socket.create_connection((host, port), timeout=connect_timeout)
         keepalive(sock)
-        sock.settimeout(None)
     except (socket.error, socket.timeout) as ex:
         raise OperationalError("could not connect to '%s:%s' (%s)" % (host, port, ex))
 
@@ -218,7 +207,6 @@ def connect(host=None, port=50000, user=None, password=None, database=None, conn
                     host, port, database = m.group(1), int(m.group(2)), m.group(3) or database
                     sock = socket.create_connection((host, port), timeout=connect_timeout)
                     keepalive(sock)
-                    sock.settimeout(None)
                 continue  # merovingian proxy redirect: keep reading the next challenge on this socket
             if block[0] == "!":
                 raise OperationalError("(remote) %s" % block[1:].strip())
@@ -244,4 +232,9 @@ def connect(host=None, port=50000, user=None, password=None, database=None, conn
     except (socket.error, socket.timeout) as ex:
         connection.close()
         raise OperationalError("connection error: %s" % ex)
+    except Exception:   # e.g. connection_lost() out of _putblock/_getblock: the socket is still ours to close
+        connection.close()
+        raise
+
+    handshake_done(sock)
     return connection

extra/dbwire/mysql.py

@@ -23,7 +23,9 @@ from extra.dbwire import InterfaceError
 from extra.dbwire import NotSupportedError
 from extra.dbwire import OperationalError
 from extra.dbwire import connection_lost
+from extra.dbwire import handshake_done
 from extra.dbwire import keepalive
+from extra.dbwire import recvn
 from extra.dbwire import ProgrammingError
 
 # capability flags
@@ -62,31 +64,19 @@ def _cstring(data, off):
         return data[off:], len(data)
     return data[off:end], end + 1
 
-def _recvn(sock, n):
-    buf = b""
-    while len(buf) < n:
-        try:
-            chunk = sock.recv(n - len(buf))
-        except (socket.error, OSError) as ex:
-            raise connection_lost(ex)
-        if not chunk:
-            raise InterfaceError("connection closed by server")
-        buf += chunk
-    return buf
-
 def _read_packet(sock):
-    header = _recvn(sock, 4)
+    header = recvn(sock, 4)
     length = struct.unpack("<I", header[0:3] + b"\x00")[0]
     seq = _u8(header, 3)
-    payload = _recvn(sock, length)
+    payload = recvn(sock, length)
     total = length
     while length == 0xffffff:  # payload continues in the next packet
-        header = _recvn(sock, 4)
+        header = recvn(sock, 4)
         length = struct.unpack("<I", header[0:3] + b"\x00")[0]
         total += length
         if total > _MAX_MESSAGE_LENGTH:
             raise InterfaceError("backend message too large (%d bytes)" % total)
-        payload += _recvn(sock, length)
+        payload += recvn(sock, length)
     return seq, payload
 
 def _send_packet(sock, seq, payload):
@@ -295,7 +285,6 @@ def connect(host=None, port=3306, user=None, password=None, database=None, conne
     try:
         sock = socket.create_connection((host or "localhost", int(port or 3306)), timeout=connect_timeout)
         keepalive(sock)
-        sock.settimeout(None)
     except (socket.error, socket.timeout) as ex:
         raise OperationalError("could not connect to '%s:%s' (%s)" % (host, port, ex))
 
@@ -352,6 +341,7 @@ def connect(host=None, port=3306, user=None, password=None, database=None, conne
         _send_packet(sock, seq + 1, response)
 
         _finish_auth(sock, password or "", plugin or "mysql_native_password", salt)
+        handshake_done(sock)
     except (DatabaseError, InterfaceError):
         _safe_close(sock)
         raise
@@ -363,12 +353,16 @@ def connect(host=None, port=3306, user=None, password=None, database=None, conne
     # SET NAMES: reset collation_connection to the server's default (the fixed handshake collation 45 =
     # utf8mb4_general_ci otherwise clashes with MySQL 8's utf8mb4_0900_ai_ci columns -> 'illegal mix of
     # collations' 1271 in a UNION/CONCAT); results stay utf8mb4 so the utf-8 decode is unchanged. autocommit=1
-    # so DML persists even if the server default is autocommit=0. Both best-effort (one-time, at connect).
-    for setup in ("SET NAMES utf8mb4", "SET autocommit=1"):
-        try:
-            connection._query(setup)
-        except Exception:
-            pass
+    # so DML persists even if the server default is autocommit=0. Best-effort (one-time, at connect) - but the
+    # charset has to land on *something*, else rows decode as utf-8 that never was: pre-5.5.3 servers have no
+    # utf8mb4, and 'utf8' (3-byte) is the fallback every 4.1+ server does have.
+    for alternatives in (("SET NAMES utf8mb4", "SET NAMES utf8"), ("SET autocommit=1",)):
+        for setup in alternatives:
+            try:
+                connection._query(setup)
+                break
+            except Exception:
+                pass
     return connection
 
 def _safe_close(sock):

extra/dbwire/postgres.py

@@ -29,7 +29,9 @@ from extra.dbwire import InterfaceError
 from extra.dbwire import NotSupportedError
 from extra.dbwire import OperationalError
 from extra.dbwire import connection_lost
+from extra.dbwire import handshake_done
 from extra.dbwire import keepalive
+from extra.dbwire import recvn
 from extra.dbwire import ProgrammingError
 
 _PROTOCOL_VERSION = 196608  # 3.0
@@ -70,24 +72,12 @@ def _xor(a, b):
         return b"".join(chr(ord(x) ^ ord(y)) for x, y in zip(a, b))
     return bytes(x ^ y for x, y in zip(a, b))
 
-def _recvn(sock, n):
-    buf = b""
-    while len(buf) < n:
-        try:
-            chunk = sock.recv(n - len(buf))
-        except (socket.error, OSError) as ex:
-            raise connection_lost(ex)
-        if not chunk:
-            raise InterfaceError("connection closed by server")
-        buf += chunk
-    return buf
-
 def _read_message(sock):
-    mtype = _recvn(sock, 1)
-    (length,) = struct.unpack("!I", _recvn(sock, 4))
+    mtype = recvn(sock, 1)
+    (length,) = struct.unpack("!I", recvn(sock, 4))
     if length < 4 or length > _MAX_MESSAGE_LENGTH:
         raise InterfaceError("invalid backend message length (%d)" % length)
-    return mtype, _recvn(sock, length - 4)
+    return mtype, recvn(sock, length - 4)
 
 def _send(sock, mtype, payload):
     try:
@@ -313,7 +303,6 @@ def connect(host=None, port=5432, user=None, password=None, database=None, conne
     try:
         sock = socket.create_connection((host or "localhost", int(port or 5432)), timeout=connect_timeout)
         keepalive(sock)
-        sock.settimeout(None)
     except (socket.error, socket.timeout) as ex:
         raise OperationalError("could not connect to '%s:%s' (%s)" % (host, port, ex))
 
@@ -331,6 +320,7 @@ def connect(host=None, port=5432, user=None, password=None, database=None, conne
                 _raise_server_error_as_operational(payload)
             if mtype == b"Z":
                 break
+        handshake_done(sock)
     except Exception:  # any setup failure (DB-API or otherwise) must still close the socket
         try:
             sock.close()

extra/dbwire/tds.py

@@ -6,7 +6,10 @@ See the file 'LICENSE' for copying permission
 """
 
 """
-Minimal pure-python TDS (Tabular Data Stream) client for Microsoft SQL Server / Sybase (stdlib only).
+Minimal pure-python TDS (Tabular Data Stream) client for Microsoft SQL Server (stdlib only).
+
+LOGIN7 / TDS 7.4 is the Microsoft dialect. Sybase ASE speaks TDS 5.0 - same 8-byte packet framing, but a
+LOGINREC login and its own token/type dialect - which is not implemented yet, so ASE still needs pymssql.
 
 Cleartext login only (TDS pre-login encryption negotiated to NOT_SUP); a server that forces encryption
 would need TLS-in-TDS which is out of scope for the dependency-free client. Implements PRELOGIN, LOGIN7,
@@ -23,7 +26,9 @@ from extra.dbwire import InterfaceError
 from extra.dbwire import NotSupportedError
 from extra.dbwire import OperationalError
 from extra.dbwire import connection_lost
+from extra.dbwire import handshake_done
 from extra.dbwire import keepalive
+from extra.dbwire import recvn
 from extra.dbwire import ProgrammingError
 
 _MAX_MESSAGE_LENGTH = 0x40000000
@@ -38,18 +43,6 @@ _STATUS_EOM = 0x01
 def _u8(data, off):
     return struct.unpack("<B", data[off:off + 1])[0]
 
-def _recvn(sock, n):
-    buf = b""
-    while len(buf) < n:
-        try:
-            chunk = sock.recv(n - len(buf))
-        except (socket.error, OSError) as ex:
-            raise connection_lost(ex)
-        if not chunk:
-            raise InterfaceError("connection closed by server")
-        buf += chunk
-    return buf
-
 def _send_message(sock, mtype, data):
     # split into <= 4096-byte packets (8-byte header + <=4088 data); only the last carries the EOM status bit
     chunk_size = 4088
@@ -75,14 +68,14 @@ def _read_message(sock):
     # chunks in a list so reassembly stays linear rather than re-copying a growing immutable buffer.
     chunks, total = [], 0
     while True:
-        header = _recvn(sock, 8)
+        header = recvn(sock, 8)
         mtype, status, length = struct.unpack(">BBH", header[:4])
         if length < 8:
             raise InterfaceError("invalid TDS packet length (%d)" % length)
         total += length - 8
         if total > _MAX_MESSAGE_LENGTH:
             raise InterfaceError("TDS message exceeds the maximum allowed length (%d bytes)" % _MAX_MESSAGE_LENGTH)
-        chunks.append(_recvn(sock, length - 8))
+        chunks.append(recvn(sock, length - 8))
         if status & _STATUS_EOM:
             break
     return b"".join(chunks)
@@ -160,7 +153,10 @@ def _login7(sock, user, password, database, hostname="dbwire", appname="dbwire")
 
 # ---- token stream + type decoding --------------------------------------------------------------------
 
-def _read_us_varchar(data, off):
+def _read_b_varchar(data, off):
+    # B_VARCHAR: 1-byte length in UCS-2 *characters* (not bytes), then that many 16-bit units. COLMETADATA's
+    # ColName is a B_VARCHAR (MS-TDS 2.2.7.4) - identifiers cap at 128 chars and an unaliased expression
+    # column comes back with an empty name, so the one-byte count cannot overflow.
     (n,) = struct.unpack("<B", data[off:off + 1])
     return data[off + 1:off + 1 + n * 2].decode("utf-16-le", "replace"), off + 1 + n * 2
 
@@ -496,7 +492,7 @@ def _parse_tokens(sock, login=False):
                     (numparts,) = struct.unpack("<B", data[off:off + 1]); off += 1
                     for _ in range(numparts):
                         (plen,) = struct.unpack("<H", data[off:off + 2]); off += 2 + plen * 2
-                col.name, off = _read_us_varchar(data, off)
+                col.name, off = _read_b_varchar(data, off)
                 columns.append(col)
             description = [(c.name, c.type, None, None, None, None, None) for c in columns]
         elif token == 0xd1:  # ROW
@@ -611,13 +607,13 @@ def connect(host=None, port=1433, user=None, password=None, database=None, conne
     try:
         sock = socket.create_connection((host or "localhost", int(port or 1433)), timeout=connect_timeout)
         keepalive(sock)
-        sock.settimeout(None)
     except (socket.error, socket.timeout) as ex:
         raise OperationalError("could not connect to '%s:%s' (%s)" % (host, port, ex))
 
     try:
         _prelogin(sock)
         _login7(sock, user, password, database)
+        handshake_done(sock)
     except (DatabaseError, InterfaceError):
         try:
             sock.close()

lib/core/settings.py

@@ -20,7 +20,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.8.30"
+VERSION = "1.10.8.31"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

tests/test_dbwire.py

@@ -30,6 +30,7 @@ from _testutils import bootstrap
 bootstrap()
 
 import extra.dbwire as dbwire
+from extra.dbwire import clickhouse as _clickhouse
 from extra.dbwire import connection_lost
 from extra.dbwire import cubrid as _cubrid
 from extra.dbwire import firebird as _firebird
@@ -38,6 +39,7 @@ from extra.dbwire import monetdb as _monetdb
 from extra.dbwire import mysql as _mysql
 from extra.dbwire import postgres as _postgres
 from extra.dbwire import presto as _presto
+from extra.dbwire import recvn
 from extra.dbwire import tds as _tds
 
 
@@ -47,6 +49,7 @@ class FakeSocket(object):
     def __init__(self, inbound=b""):
         self.inbound = bytearray(inbound)
         self.sent = bytearray()
+        self.timeouts = []
         self.closed = False
 
     def feed(self, data):
@@ -62,8 +65,8 @@ class FakeSocket(object):
     def sendall(self, data):
         self.sent.extend(data)
 
-    def settimeout(self, _value):
-        pass
+    def settimeout(self, value):
+        self.timeouts.append(value)
 
     def setsockopt(self, *_args):
         pass
@@ -180,20 +183,23 @@ class PostgresScramTest(unittest.TestCase):
         self.assertRaises(dbwire.OperationalError, _postgres._authenticate, sock, "user", "secret")
 
 
-class MysqlCapabilityTest(unittest.TestCase):
-    def _handshake(self, server_caps):
-        payload = b"\x0a" + b"8.0.0-fake\x00" + struct.pack("<I", 1) + b"12345678" + b"\x00"
-        payload += struct.pack("<H", server_caps & 0xffff)
-        payload += b"\x21" + struct.pack("<H", 2)
-        payload += struct.pack("<H", (server_caps >> 16) & 0xffff)
-        payload += struct.pack("<B", 21) + (b"\x00" * 10) + b"123456789012\x00"
-        payload += b"mysql_native_password\x00"
-        return payload
+def _mysql_handshake(server_caps):
+    """The initial handshake packet of a server advertising `server_caps`."""
 
+    payload = b"\x0a" + b"8.0.0-fake\x00" + struct.pack("<I", 1) + b"12345678" + b"\x00"
+    payload += struct.pack("<H", server_caps & 0xffff)
+    payload += b"\x21" + struct.pack("<H", 2)
+    payload += struct.pack("<H", (server_caps >> 16) & 0xffff)
+    payload += struct.pack("<B", 21) + (b"\x00" * 10) + b"123456789012\x00"
+    payload += b"mysql_native_password\x00"
+    return payload
+
+
+class MysqlCapabilityTest(unittest.TestCase):
     def _connect_with(self, server_caps):
         """Drive the real handshake path with a fake server advertising `server_caps`."""
 
-        payload = self._handshake(server_caps)
+        payload = _mysql_handshake(server_caps)
         sock = FakeSocket(struct.pack("<I", len(payload))[:3] + b"\x00" + payload)
         self._last_sock = sock
         saved = socket.create_connection
@@ -228,6 +234,69 @@ class MysqlCapabilityTest(unittest.TestCase):
         self.assertFalse(flags & _mysql._CLIENT_PLUGIN_AUTH, "PLUGIN_AUTH was not advertised by the server")
 
 
+def _mysql_packets(data):
+    """Splits a recorded client byte stream into MySQL packet payloads."""
+
+    out, off = [], 0
+    while off + 4 <= len(data):
+        length = struct.unpack("<I", data[off:off + 3] + b"\x00")[0]
+        out.append(bytes(data[off + 4:off + 4 + length]))
+        off += 4 + length
+    return out
+
+
+class MysqlConnectTranscriptTest(unittest.TestCase):
+    """connect() has to keep the connect deadline armed until the login exchange is over, and the charset
+    has to end up set - a swallowed 'SET NAMES' failure decodes every later row as utf-8 that never was."""
+
+    _OK = b"\x00\x00\x00\x02\x00\x00\x00"
+    _ERR = b"\xff" + struct.pack("<H", 1115) + b"#42000" + b"Unknown character set: 'utf8mb4'"
+
+    def _packet(self, payload, seq=0):
+        return struct.pack("<I", len(payload))[:3] + struct.pack("<B", seq) + payload
+
+    def _connect(self, *replies):
+        caps = (_mysql._CLIENT_PROTOCOL_41 | _mysql._CLIENT_SECURE_CONNECTION | _mysql._CLIENT_PLUGIN_AUTH)
+        handshake = _mysql_handshake(caps)
+        inbound = self._packet(handshake) + b"".join(self._packet(_, 2) for _ in replies)
+        sock = FakeSocket(inbound)
+        saved = socket.create_connection
+        socket.create_connection = lambda *a, **k: sock
+        try:
+            connection = _mysql.connect(host="h", port=3306, user="u", password="p", database=None, connect_timeout=7)
+        finally:
+            socket.create_connection = saved
+        return connection, sock
+
+    def test_deadline_is_dropped_only_once_logged_in(self):
+        _connection, sock = self._connect(self._OK, self._OK, self._OK)
+        self.assertIsNone(sock.timeouts[-1], "connect deadline outlived the handshake")
+        self.assertNotIn(None, sock.timeouts[:-1], "connect deadline was dropped before the handshake")
+
+    def test_deadline_is_kept_when_the_server_goes_silent(self):
+        """A peer that accepts the connection and then says nothing is alive as far as keepalive() is
+        concerned - without a deadline the login read never returns."""
+
+        try:
+            self._connect()                 # nothing after the handshake packet
+            self.fail("a silent server was accepted")
+        except dbwire.Error:
+            pass
+
+    def test_charset_falls_back_when_utf8mb4_is_rejected(self):
+        """Pre-5.5.3 servers have no utf8mb4."""
+
+        _connection, sock = self._connect(self._OK, self._ERR, self._OK, self._OK)
+        queries = [_[1:] for _ in _mysql_packets(sock.sent) if _[:1] == b"\x03"]
+        self.assertEqual(queries[:2], [b"SET NAMES utf8mb4", b"SET NAMES utf8"])
+        self.assertIn(b"SET autocommit=1", queries)
+
+    def test_charset_fallback_is_not_sent_when_utf8mb4_is_accepted(self):
+        _connection, sock = self._connect(self._OK, self._OK, self._OK)
+        queries = [_[1:] for _ in _mysql_packets(sock.sent) if _[:1] == b"\x03"]
+        self.assertEqual(queries, [b"SET NAMES utf8mb4", b"SET autocommit=1"])
+
+
 class TdsFramingTest(unittest.TestCase):
     def _packet(self, body, eom=True):
         return struct.pack(">BBHHBB", 4, 1 if eom else 0, len(body) + 8, 0, 0, 0) + body
@@ -387,6 +456,61 @@ class BoundedReadTest(unittest.TestCase):
             _monetdb._MAX_MESSAGE_LENGTH = saved
 
 
+class DecoderTest(unittest.TestCase):
+    """Byte fixtures for the socket-free decoders, taken from what the real servers put on the wire."""
+
+    def test_tds_column_name_is_a_character_count(self):
+        """COLMETADATA ColName is B_VARCHAR - a length in UCS-2 characters (MS-TDS 2.2.7.4), so the count
+        byte is followed by twice as many bytes."""
+
+        data = struct.pack("<B", 3) + u"abc".encode("utf-16-le") + b"TRAILING"
+        self.assertEqual(_tds._read_b_varchar(data, 0), ("abc", 7))
+        self.assertEqual(_tds._read_b_varchar(struct.pack("<B", 0) + b"X", 0), ("", 1))
+
+    def test_clickhouse_tabseparated_escapes(self):
+        """ClickHouse escapes only \\t \\n \\r \\0 \\b \\f \\' \\\\ - a control byte such as 0x01 is written
+        raw, and a String holding invalid UTF-8 comes back as bytes rather than mojibake."""
+
+        self.assertIsNone(_clickhouse._unescape(b"\\N"))
+        self.assertEqual(_clickhouse._unescape(b"tab\\there"), b"tab\there")
+        self.assertEqual(_clickhouse._unescape(b"nl\\nx"), b"nl\nx")
+        self.assertEqual(_clickhouse._unescape(b"a\\\\b"), b"a\\b")
+        self.assertEqual(_clickhouse._unescape(b"\\0"), b"\x00")
+        self.assertEqual(_clickhouse._unescape(b"\x01\x02"), b"\x01\x02")
+        self.assertEqual(_clickhouse._decode_cell(b"\x00\xff\x80"), b"\x00\xff\x80")
+
+    def test_postgres_bytea_both_output_formats(self):
+        """bytea arrives as text in whatever bytea_output the server is set to. 'escape' renders a literal
+        backslash as '\\\\', so it can never collide with the '\\x' hex-format prefix."""
+
+        for wire, expected in ((b"\\x5c78414243", b"\\xABC"),   # hex format
+                               (b"\\\\xABC", b"\\xABC"),        # escape format, value starts with a backslash
+                               (b"\\x00ff80", b"\x00\xff\x80"),
+                               (b"\\000\\377\\200", b"\x00\xff\x80"),
+                               (b"\\\\", b"\\"),
+                               (b"\\\\001", b"\\001")):
+            self.assertEqual(_postgres._decode_bytea(wire), expected, wire)
+
+    def test_monetdb_quoted_values_keep_the_delimiter_out(self):
+        """mserver escapes tab/quote/newline/backslash INSIDE a quoted value, so the literal ',\\t' field
+        delimiter cannot occur in one; control bytes arrive as C octal escapes."""
+
+        line = u'[ 2,\t"a,\\tb",\t"q\\"uote",\t"ctl:\\001",\t"euro-\u20ac",\tNULL\t]'
+        description, rows = _monetdb._parse_result(u"%% id,\tv # name\n%s" % line)
+        self.assertEqual([_[0] for _ in description], ["id", "v"])
+        self.assertEqual(rows, [("2", "a,\tb", 'q"uote', "ctl:\x01", u"euro-\u20ac", None)])
+
+    def test_cubrid_error_classification(self):
+        """CUBRID points at the offending token as "before '<fragment>'"; a coercion failure is a DataError."""
+
+        for message, expected in (("(remote) Syntax: In line 1, column 1 before ' 1'", dbwire.ProgrammingError),
+                                  ("(remote) Unknown class \"public.t\".", dbwire.ProgrammingError),
+                                  ("(remote) Cannot coerce value of domain \"character\" to domain \"integer\".",
+                                   dbwire.DataError),
+                                  ("(remote) unique constraint violated", dbwire.IntegrityError)):
+            self.assertRaises(expected, _cubrid.Connection._raise, -494, message)
+
+
 class HelperTest(unittest.TestCase):
     def test_socket_failure_maps_into_the_dbapi_hierarchy(self):
         """Callers of a PEP 249 driver only catch Error and its subclasses."""
@@ -400,6 +524,15 @@ class HelperTest(unittest.TestCase):
         self.assertEqual(http_origin("[fe80::1]", 8123), "http://[fe80::1]:8123")
         self.assertEqual(http_origin(None, 8123), "http://localhost:8123")
 
+    def test_recvn_reads_exactly_n_bytes_across_chunks(self):
+        """Shared by every socket module: a framed protocol has no notion of a short read."""
+
+        sock = FakeSocket(b"ABCDEFGH")
+        original = sock.recv
+        sock.recv = lambda _count: original(3)          # dribble the stream 3 bytes at a time
+        self.assertEqual(recvn(sock, 8), b"ABCDEFGH")
+        self.assertRaises(dbwire.InterfaceError, recvn, FakeSocket(b"AB"), 4)
+
     def test_every_module_exposes_the_dbapi_surface(self):
         for name in ("postgres", "mysql", "tds", "firebird", "cubrid", "monetdb", "clickhouse", "presto"):
             module = __import__("extra.dbwire.%s" % name, fromlist=["connect"])