Patch for known NCHAR/UNION-query SQLi problems

README.md

@@ -4,7 +4,7 @@
 
 sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester, and a broad range of switches including database fingerprinting, over data fetching from the database, accessing the underlying file system, and executing commands on the operating system via out-of-band connections.
 
-**The sqlmap project is currently searching for sponsor(s).**
+**sqlmap is sponsored by [SpyderSec](https://spydersec.com/).**
 
 Screenshots
 ----

data/xml/queries.xml

@@ -131,8 +131,8 @@
             <blind query="SELECT tablename FROM pg_tables WHERE schemaname='%s' ORDER BY tablename OFFSET %d LIMIT 1" count="SELECT COUNT(tablename) FROM pg_tables WHERE schemaname='%s'"/>
         </tables>
         <columns>
-            <inband query="SELECT attname,typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s' ORDER BY attname" condition="attname"/>
-            <blind query="SELECT attname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s' ORDER BY attname" query2="SELECT typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relname='%s' AND a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND attname='%s' AND nspname='%s' ORDER BY attname" count="SELECT COUNT(attname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
+            <inband query="SELECT attname,typname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s' ORDER BY attname" condition="attname"/>
+            <blind query="SELECT attname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s' ORDER BY attname" query2="SELECT typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relname='%s' AND a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND attname='%s' AND nspname='%s' ORDER BY attname" count="SELECT COUNT(attname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
         </columns>
         <dump_table>
             <inband query="SELECT %s FROM %s.%s ORDER BY %s"/>
@@ -147,8 +147,8 @@
             <blind query="SELECT DISTINCT(schemaname) FROM pg_tables WHERE %s" query2="SELECT tablename FROM pg_tables WHERE schemaname='%s'" count="SELECT COUNT(DISTINCT(schemaname)) FROM pg_tables WHERE %s" count2="SELECT COUNT(tablename) FROM pg_tables WHERE schemaname='%s'" condition="tablename" condition2="schemaname"/>
         </search_table>
         <search_column>
-            <inband query="SELECT nspname,relname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" condition="attname" condition2="nspname" condition3="relname"/>
-            <blind query="SELECT DISTINCT(nspname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" query2="SELECT DISTINCT(relname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND nspname='%s'" count="SELECT COUNT(DISTINCT(nspname)) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" count2="SELECT COUNT(DISTINCT(relname)) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND nspname='%s'" condition="attname" condition2="nspname" condition3="relname"/>
+            <inband query="SELECT nspname,relname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" condition="attname" condition2="nspname" condition3="relname"/>
+            <blind query="SELECT DISTINCT(nspname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" query2="SELECT DISTINCT(relname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND nspname='%s'" count="SELECT COUNT(DISTINCT(nspname)) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" count2="SELECT COUNT(DISTINCT(relname)) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND nspname='%s'" condition="attname" condition2="nspname" condition3="relname"/>
         </search_column>
     </dbms>
 
@@ -1370,8 +1370,8 @@
             <blind query="SELECT table_name FROM information_schema.tables WHERE table_schema='%s' LIMIT 1 OFFSET %d" count="SELECT COUNT(table_name) FROM information_schema.tables WHERE table_schema='%s'"/>
         </tables>
         <columns>
-            <inband query="SELECT attname,typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
-            <blind query="SELECT attname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s'" query2="SELECT typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relname='%s' AND a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND attname='%s' AND nspname='%s'" count="SELECT COUNT(attname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
+            <inband query="SELECT attname,typname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
+            <blind query="SELECT attname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s'" query2="SELECT typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relname='%s' AND a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND attname='%s' AND nspname='%s'" count="SELECT COUNT(attname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
         </columns>
         <dump_table>
             <inband query="SELECT %s FROM %s.%s ORDER BY %s"/>
@@ -1386,8 +1386,8 @@
             <blind query="SELECT DISTINCT(table_schema) FROM information_schema.tables WHERE %s" query2="SELECT table_name FROM information_schema.tables WHERE table_schema='%s'" count="SELECT COUNT(DISTINCT(table_schema)) FROM information_schema.tables WHERE %s" count2="SELECT COUNT(table_name) FROM information_schema.tables WHERE table_schema='%s'" condition="table_name" condition2="table_schema"/>
         </search_table>
         <search_column>
-            <inband query="SELECT nspname,relname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" condition="attname" condition2="nspname" condition3="relname"/>
-            <blind query="SELECT DISTINCT(nspname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" query2="SELECT DISTINCT(relname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND nspname='%s'" count="SELECT COUNT(DISTINCT(nspname)) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" count2="SELECT COUNT(DISTINCT(relname)) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND nspname='%s'" condition="attname" condition2="nspname" condition3="relname"/>
+            <inband query="SELECT nspname,relname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" condition="attname" condition2="nspname" condition3="relname"/>
+            <blind query="SELECT DISTINCT(nspname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" query2="SELECT DISTINCT(relname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND nspname='%s'" count="SELECT COUNT(DISTINCT(nspname)) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" count2="SELECT COUNT(DISTINCT(relname)) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND nspname='%s'" condition="attname" condition2="nspname" condition3="relname"/>
         </search_column>
     </dbms>
 

lib/controller/checks.py

@@ -30,6 +30,7 @@ from lib.core.common import getSortedInjectionTests
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import intersect
+from lib.core.common import isDigit
 from lib.core.common import joinValue
 from lib.core.common import listToStrValue
 from lib.core.common import parseFilePaths
@@ -117,7 +118,7 @@ def checkSqlInjection(place, parameter, value):
     threadData = getCurrentThreadData()
 
     # Favoring non-string specific boundaries in case of digit-like parameter values
-    if value.isdigit():
+    if isDigit(value):
         kb.cache.intBoundaries = kb.cache.intBoundaries or sorted(copy.deepcopy(conf.boundaries), key=lambda boundary: any(_ in (boundary.prefix or "") or _ in (boundary.suffix or "") for _ in ('"', '\'')))
         boundaries = kb.cache.intBoundaries
     elif value.isalpha():
@@ -226,8 +227,8 @@ def checkSqlInjection(place, parameter, value):
             # Skip test if the user's wants to test only for a specific
             # technique
             if conf.technique and isinstance(conf.technique, list) and stype not in conf.technique:
-                debugMsg = "skipping test '%s' because the user " % title
-                debugMsg += "specified to test only for "
+                debugMsg = "skipping test '%s' because user " % title
+                debugMsg += "specified testing of only "
                 debugMsg += "%s techniques" % " & ".join(PAYLOAD.SQLINJECTION[_] for _ in conf.technique)
                 logger.debug(debugMsg)
                 continue
@@ -651,7 +652,7 @@ def checkSqlInjection(place, parameter, value):
                             except SqlmapConnectionException as ex:
                                 debugMsg = "problem occurred most likely because the "
                                 debugMsg += "server hasn't recovered as expected from the "
-                                debugMsg += "error-based payload used ('%s')" % getSafeExString(ex)
+                                debugMsg += "used error-based payload ('%s')" % getSafeExString(ex)
                                 logger.debug(debugMsg)
 
                         # In case of time-based blind or stacked queries
@@ -855,7 +856,9 @@ def checkSqlInjection(place, parameter, value):
             logger.warn(warnMsg)
 
         if not checkFalsePositives(injection):
-            kb.vulnHosts.remove(conf.hostname)
+            if conf.hostname in kb.vulnHosts:
+                kb.vulnHosts.remove(conf.hostname)
+
             if NOTE.FALSE_POSITIVE_OR_UNEXPLOITABLE not in injection.notes:
                 injection.notes.append(NOTE.FALSE_POSITIVE_OR_UNEXPLOITABLE)
 
@@ -876,8 +879,12 @@ def heuristicCheckDbms(injection):
     to identify with a simple DBMS specific boolean-based test what the DBMS
     may be
     """
+
     retVal = False
 
+    if conf.skipHeuristics:
+        return retVal
+
     pushValue(kb.injection)
     kb.injection = injection
 
@@ -1031,6 +1038,9 @@ def checkFilteredChars(injection):
     kb.injection = popValue()
 
 def heuristicCheckSqlInjection(place, parameter):
+    if conf.skipHeuristics:
+        return None
+
     if kb.heavilyDynamic:
         debugMsg = "heuristic check skipped because of heavy dynamicity"
         logger.debug(debugMsg)

lib/core/agent.py

@@ -471,6 +471,12 @@ class Agent(object):
         @rtype: C{str}
         """
 
+        match = re.search(r"(?i)(.+)( AS \w+)\Z", field)
+        if match:
+            field, suffix = match.groups()
+        else:
+            suffix = ""
+
         nulledCastedField = field
 
         if field and Backend.getIdentifiedDbms():
@@ -490,6 +496,12 @@ class Agent(object):
             if conf.hexConvert or kb.binaryField:
                 nulledCastedField = self.hexConvertField(nulledCastedField)
 
+        if suffix:
+            nulledCastedField += suffix
+
+        if not kb.nchar:
+            nulledCastedField = re.sub(r"( AS )N(CHAR|VARCHAR)", r"\g<1>\g<2>", nulledCastedField)
+
         return nulledCastedField
 
     def nullCastConcatFields(self, fields):
@@ -533,6 +545,7 @@ class Agent(object):
             nulledCastedFields = []
 
             for field in fieldsSplitted:
+                field = re.sub(r"(?i) AS \w+\Z", "", field)         # NOTE: fields such as "... AS type_name" have to be stripped from the alias part for this functionality to work
                 nulledCastedFields.append(self.nullAndCastField(field))
 
             delimiterStr = "%s'%s'%s" % (dbmsDelimiter, kb.chars.delimiter, dbmsDelimiter)
@@ -779,7 +792,6 @@ class Agent(object):
             elif fieldsNoSelect:
                 concatenatedQuery = "CONCAT(CONCAT('%s',%s),'%s')" % (kb.chars.start, concatenatedQuery, kb.chars.stop)
 
-
         return concatenatedQuery
 
     def forgeUnionQuery(self, query, position, count, comment, prefix, suffix, char, where, multipleUnions=None, limited=False, fromTable=None):

lib/core/common.py

@@ -971,8 +971,9 @@ def setColor(message, color=None, bold=False, level=None, istty=None):
                 if match:
                     retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
 
-                for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
-                    retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
+                if not any(_ in message for _ in ("Payload: ",)):
+                    for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
+                        retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
 
         message = message.strip()
 

lib/core/option.py

@@ -2071,6 +2071,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.mergeCookies = None
     kb.multipleCtrlC = False
     kb.negativeLogic = False
+    kb.nchar = True
     kb.nullConnection = None
     kb.oldMsf = None
     kb.orderByColumns = None

lib/core/optiondict.py

@@ -228,6 +228,7 @@ optDict = {
         "repair": "boolean",
         "saveConfig": "string",
         "scope": "string",
+        "skipHeuristics": "boolean",
         "skipWaf": "boolean",
         "testFilter": "string",
         "testSkip": "string",

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.4.11.0"
+VERSION = "1.4.12.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/update.py

@@ -14,6 +14,7 @@ import time
 import zipfile
 
 from lib.core.common import dataToStdout
+from lib.core.common import extractRegexResult
 from lib.core.common import getLatestRevision
 from lib.core.common import getSafeExString
 from lib.core.common import openFile
@@ -27,6 +28,7 @@ from lib.core.revision import getRevisionNumber
 from lib.core.settings import GIT_REPOSITORY
 from lib.core.settings import IS_WIN
 from lib.core.settings import VERSION
+from lib.core.settings import TYPE
 from lib.core.settings import ZIPBALL_PAGE
 from thirdparty.six.moves import urllib as _urllib
 
@@ -36,7 +38,34 @@ def update():
 
     success = False
 
-    if not os.path.exists(os.path.join(paths.SQLMAP_ROOT_PATH, ".git")):
+    if TYPE == "pip":
+        infoMsg = "updating sqlmap to the latest stable version from the "
+        infoMsg += "PyPI repository"
+        logger.info(infoMsg)
+
+        debugMsg = "sqlmap will try to update itself using 'pip' command"
+        logger.debug(debugMsg)
+
+        dataToStdout("\r[%s] [INFO] update in progress" % time.strftime("%X"))
+
+        output = ""
+        try:
+            process = subprocess.Popen("pip install -U sqlmap", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=paths.SQLMAP_ROOT_PATH)
+            pollProcess(process, True)
+            output, _ = process.communicate()
+            success = not process.returncode
+        except Exception as ex:
+            success = False
+            output = getSafeExString(ex)
+        finally:
+            output = getText(output)
+
+        if success:
+            logger.info("%s the latest revision '%s'" % ("already at" if "already up-to-date" in output else "updated to", extractRegexResult(r"\binstalled sqlmap-(?P<result>\d+\.\d+\.\d+)", output) or extractRegexResult(r"\((?P<result>\d+\.\d+\.\d+)\)", output)))
+        else:
+            logger.error("update could not be completed ('%s')" % re.sub(r"[^a-z0-9:/\\]+", " ", output).strip())
+
+    elif not os.path.exists(os.path.join(paths.SQLMAP_ROOT_PATH, ".git")):
         warnMsg = "not a git repository. It is recommended to clone the 'sqlmapproject/sqlmap' repository "
         warnMsg += "from GitHub (e.g. 'git clone --depth 1 %s sqlmap')" % GIT_REPOSITORY
         logger.warn(warnMsg)
@@ -95,6 +124,7 @@ def update():
                                 os.chmod(os.path.join(directory, "sqlmap.py"), attrs)
                             except OSError:
                                 logger.warning("could not set the file attributes of '%s'" % os.path.join(directory, "sqlmap.py"))
+
     else:
         infoMsg = "updating sqlmap to the latest development revision from the "
         infoMsg += "GitHub repository"

lib/parse/cmdline.py

@@ -700,6 +700,9 @@ def cmdLineParser(argv=None):
         general.add_argument("--scope", dest="scope",
             help="Regexp for filtering targets")
 
+        general.add_argument("--skip-heuristics", dest="skipHeuristics", action="store_true",
+            help="Skip heuristic detection of SQLi/XSS vulnerabilities")
+
         general.add_argument("--skip-waf", dest="skipWaf", action="store_true",
             help="Skip heuristic detection of WAF/IPS protection")
 
@@ -958,6 +961,10 @@ def cmdLineParser(argv=None):
                 argv[i] = ""
             elif argv[i] in DEPRECATED_OPTIONS:
                 argv[i] = ""
+            elif argv[i].startswith("--data-raw"):
+                argv[i] = argv[i].replace("--data-raw", "--data", 1)
+            elif argv[i].startswith("--drop-cookie"):
+                argv[i] = argv[i].replace("--drop-cookie", "--drop-set-cookie", 1)
             elif any(argv[i].startswith(_) for _ in ("--tamper", "--ignore-code", "--skip")):
                 key = re.search(r"\-?\-(\w+)\b", argv[i]).group(1)
                 index = auxIndexes.get(key, None)

lib/request/chunkedhandler.py

@@ -6,6 +6,7 @@ See the file 'LICENSE' for copying permission
 """
 
 from lib.core.data import conf
+from lib.core.enums import HTTP_HEADER
 from thirdparty.six.moves import urllib as _urllib
 
 class ChunkedHandler(_urllib.request.HTTPHandler):
@@ -20,20 +21,17 @@ class ChunkedHandler(_urllib.request.HTTPHandler):
 
         if request.data is not None:  # POST
             data = request.data
-            if not request.has_header("Content-type"):
-                request.add_unredirected_header(
-                    "Content-type",
-                    "application/x-www-form-urlencoded")
-            if not request.has_header("Content-length") and not conf.chunked:
-                request.add_unredirected_header(
-                    "Content-length", "%d" % len(data))
+            if not request.has_header(HTTP_HEADER.CONTENT_TYPE):
+                request.add_unredirected_header(HTTP_HEADER.CONTENT_TYPE, "application/x-www-form-urlencoded")
+            if not request.has_header(HTTP_HEADER.CONTENT_LENGTH) and not conf.chunked:
+                request.add_unredirected_header(HTTP_HEADER.CONTENT_LENGTH, "%d" % len(data))
 
         sel_host = host
         if request.has_proxy():
             sel_host = _urllib.parse.urlsplit(request.get_selector()).netloc
 
-        if not request.has_header("Host"):
-            request.add_unredirected_header("Host", sel_host)
+        if not request.has_header(HTTP_HEADER.HOST):
+            request.add_unredirected_header(HTTP_HEADER.HOST, sel_host)
         for name, value in self.parent.addheaders:
             name = name.capitalize()
             if not request.has_header(name):

lib/request/connect.py

@@ -222,7 +222,7 @@ class Connect(object):
                         try:
                             part = conn.read(MAX_CONNECTION_READ_SIZE)
                         except AssertionError:
-                            part = ""
+                            part = b""
 
                     if len(part) == MAX_CONNECTION_READ_SIZE:
                         warnMsg = "large response detected. This could take a while"

lib/request/direct.py

@@ -45,8 +45,9 @@ def direct(query, content=True):
                 break
 
     if select:
-        if not query.upper().startswith("SELECT "):
+        if re.search(r"(?i)\ASELECT ", query) is None:
             query = "SELECT %s" % query
+
         if conf.binaryFields:
             for field in conf.binaryFields:
                 field = field.strip()
@@ -58,7 +59,7 @@ def direct(query, content=True):
     output = hashDBRetrieve(query, True, True)
     start = time.time()
 
-    if not select and "EXEC " not in query.upper():
+    if not select and re.search(r"(?i)\bEXEC ", query) is None:
         timeout(func=conf.dbmsConnector.execute, args=(query,), duration=conf.timeout, default=None)
     elif not (output and ("%soutput" % conf.tablePrefix) not in query and ("%sfile" % conf.tablePrefix) not in query):
         output, state = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)

lib/request/dns.py

@@ -76,17 +76,20 @@ class DNSServer(object):
         self._check_localhost()
         self._requests = []
         self._lock = threading.Lock()
+
         try:
             self._socket = socket._orig_socket(socket.AF_INET, socket.SOCK_DGRAM)
         except AttributeError:
             self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
         self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         self._socket.bind(("", 53))
         self._running = False
         self._initialized = False
 
     def _check_localhost(self):
-        response = ""
+        response = b""
+
         try:
             s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
             s.connect(("", 53))
@@ -96,7 +99,7 @@ class DNSServer(object):
             pass
         finally:
             if response and b"google" in response:
-                raise socket.error("another DNS service already running on *:53")
+                raise socket.error("another DNS service already running on '0.0.0.0:53'")
 
     def pop(self, prefix=None, suffix=None):
         """

lib/request/inject.py

@@ -12,6 +12,7 @@ import time
 
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
+from lib.core.common import applyFunctionRecursively
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
 from lib.core.common import cleanQuery
@@ -286,7 +287,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
 
                 try:
                     try:
-                        for num in xrange(startLimit, stopLimit):
+                        for num in xrange(startLimit or 0, stopLimit or 0):
                             output = _goInferenceFields(expression, expressionFields, expressionFieldsList, payload, num=num, charsetType=charsetType, firstChar=firstChar, lastChar=lastChar, dump=dump)
                             outputs.append(output)
                     except OverflowError:
@@ -505,6 +506,26 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
         warnMsg += "or switch '--hex'" if hasattr(queries[Backend.getIdentifiedDbms()], "hex") else ""
         singleTimeWarnMessage(warnMsg)
 
+    # Dirty patch (MSSQL --binary-fields with 0x31003200...)
+    if Backend.isDbms(DBMS.MSSQL) and conf.binaryFields:
+        def _(value):
+            if isinstance(value, six.text_type):
+                if value.startswith(u"0x"):
+                    value = value[2:]
+                    if value and len(value) % 4 == 0:
+                        candidate = ""
+                        for i in xrange(len(value)):
+                            if i % 4 < 2:
+                                candidate += value[i]
+                            elif value[i] != '0':
+                                candidate = None
+                                break
+                        if candidate:
+                            value = candidate
+            return value
+
+        value = applyFunctionRecursively(value, _)
+
     # Dirty patch (safe-encoded unicode characters)
     if isinstance(value, six.text_type) and "\\x" in value:
         try:

lib/request/redirecthandler.py

@@ -13,6 +13,7 @@ from lib.core.common import getHostHeader
 from lib.core.common import getSafeExString
 from lib.core.common import logHTTPTraffic
 from lib.core.common import readInput
+from lib.core.convert import getBytes
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -64,8 +65,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                 self.redirect_request = self._redirect_request
 
     def _redirect_request(self, req, fp, code, msg, headers, newurl):
-        newurl = newurl.replace(' ', '%20')
-        return _urllib.request.Request(newurl, data=req.data, headers=req.headers, origin_req_host=req.get_origin_req_host())
+        return _urllib.request.Request(newurl.replace(' ', '%20'), data=req.data, headers=req.headers, origin_req_host=req.get_origin_req_host())
 
     def http_error_302(self, req, fp, code, msg, headers):
         start = time.time()
@@ -75,7 +75,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
         try:
             content = fp.read(MAX_CONNECTION_TOTAL_SIZE)
         except:  # e.g. IncompleteRead
-            content = ""
+            content = b""
         finally:
             if content:
                 try:  # try to write it back to the read buffer so we could reuse it in further steps
@@ -163,7 +163,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                             retVal = getSafeExString(ex)        # Note: pyflakes mistakenly marks 'ex' as undefined (NOTE: tested in both Python2 and Python3)
                         except:
                             retVal = ""
-                        return retVal
+                        return getBytes(retVal)
 
                     result.read = types.MethodType(_, result)
 

lib/techniques/blind/inference.py

@@ -123,9 +123,6 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             payload = payload.replace("SUBSTRING", "(SELECT%sSUBSTRING" % (match.group(1) if match.group(1) else " "), 1)
             expression = match.group(2).strip()
 
-
-#<inference query="(SELECT SUBSTRING((%s) FROM %d FOR 1) FROM %s)>'%c'"/>
-
     try:
         # Set kb.partRun in case "common prediction" feature (a.k.a. "good samaritan") is used or the engine is called from the API
         if conf.predictOutput:

lib/techniques/union/use.py

@@ -157,12 +157,20 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
                 warnMsg += "(probably due to its length and/or content): "
                 warnMsg += safecharencode(trimmed)
                 logger.warn(warnMsg)
+
             elif re.search(r"ORDER BY [^ ]+\Z", expression):
                 debugMsg = "retrying failed SQL query without the ORDER BY clause"
                 singleTimeDebugMessage(debugMsg)
 
                 expression = re.sub(r"\s*ORDER BY [^ ]+\Z", "", expression)
                 retVal = _oneShotUnionUse(expression, unpack, limited)
+
+            elif kb.nchar and re.search(r" AS N(CHAR|VARCHAR)", agent.nullAndCastField(expression)):
+                debugMsg = "turning off NATIONAL CHARACTER casting"  # NOTE: in some cases there are "known" incompatibilities between original columns and NCHAR (e.g. http://testphp.vulnweb.com/artists.php?artist=1)
+                singleTimeDebugMessage(debugMsg)
+
+                kb.nchar = False
+                retVal = _oneShotUnionUse(expression, unpack, limited)
     else:
         vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector
         kb.unionDuplicates = vector[7]
@@ -233,7 +241,7 @@ def unionUse(expression, unpack=True, dump=False):
         debugMsg += "it does not play well with UNION query SQL injection"
         singleTimeDebugMessage(debugMsg)
 
-    if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL) and expressionFields:
+    if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL) and expressionFields and not conf.binaryFields:
         match = re.search(r"SELECT\s*(.+?)\bFROM", expression, re.I)
         if match and not (Backend.isDbms(DBMS.ORACLE) and FROM_DUMMY_TABLE[DBMS.ORACLE] in expression):
             kb.jsonAggMode = True

lib/utils/hash.py

@@ -583,6 +583,41 @@ __functions__ = {
     HASH.SHA512_BASE64: sha512_generic_passwd,
 }
 
+def _finalize(retVal, results, processes, attack_info=None):
+    if _multiprocessing:
+        gc.enable()
+
+    # NOTE: https://github.com/sqlmapproject/sqlmap/issues/4367
+    # NOTE: https://dzone.com/articles/python-101-creating-multiple-processes
+    for process in processes:
+        try:
+            process.terminate()
+            process.join()
+        except (OSError, AttributeError):
+            pass
+
+    if retVal:
+        removals = set()
+
+        if conf.hashDB:
+            conf.hashDB.beginTransaction()
+
+        while not retVal.empty():
+            user, hash_, word = item = retVal.get(block=False)
+            results.append(item)
+            removals.add((user, hash_))
+            hashDBWrite(hash_, word)
+
+        for item in attack_info or []:
+            if (item[0][0], item[0][1]) in removals:
+                attack_info.remove(item)
+
+        if conf.hashDB:
+            conf.hashDB.endTransaction()
+
+        if hasattr(retVal, "close"):
+            retVal.close()
+
 def storeHashesToFile(attack_dict):
     if not attack_dict:
         return
@@ -1148,30 +1183,7 @@ def dictionaryAttack(attack_dict):
                     logger.warn(warnMsg)
 
                 finally:
-                    if _multiprocessing:
-                        gc.enable()
-
-                    # NOTE: https://github.com/sqlmapproject/sqlmap/issues/4367
-                    # NOTE: https://dzone.com/articles/python-101-creating-multiple-processes
-                    for process in processes:
-                        try:
-                            process.terminate()
-                            process.join()
-                        except (OSError, AttributeError):
-                            pass
-
-                    if retVal:
-                        if conf.hashDB:
-                            conf.hashDB.beginTransaction()
-
-                        while not retVal.empty():
-                            user, hash_, word = item = retVal.get(block=False)
-                            attack_info = [_ for _ in attack_info if _[0][0] != user or _[0][1] != hash_]
-                            hashDBWrite(hash_, word)
-                            results.append(item)
-
-                        if conf.hashDB:
-                            conf.hashDB.endTransaction()
+                    _finalize(retVal, results, processes, attack_info)
 
             clearConsoleLine()
 
@@ -1253,20 +1265,7 @@ def dictionaryAttack(attack_dict):
                                 pass
 
                     finally:
-                        if _multiprocessing:
-                            gc.enable()
-
-                        if retVal and conf.hashDB:
-                            if conf.hashDB:
-                                conf.hashDB.beginTransaction()
-
-                            while not retVal.empty():
-                                user, hash_, word = item = retVal.get(block=False)
-                                hashDBWrite(hash_, word)
-                                results.append(item)
-
-                            if conf.hashDB:
-                                conf.hashDB.endTransaction()
+                        _finalize(retVal, results, processes, attack_info)
 
                 clearConsoleLine()
 

plugins/dbms/hsqldb/fingerprint.py

@@ -145,12 +145,9 @@ class Fingerprint(GenericFingerprint):
         warnMsg = "on HSQLDB it is not possible to enumerate the hostname"
         logger.warn(warnMsg)
 
-
     def checkDbmsOs(self, detailed=False):
         if Backend.getOs():
             infoMsg = "the back-end DBMS operating system is %s" % Backend.getOs()
             logger.info(infoMsg)
         else:
             self.userChooseDbmsOs()
-
-

sqlmap.conf

@@ -787,6 +787,10 @@ repair = False
 # Example: (google|yahoo)
 scope = 
 
+# Skip heuristic detection of SQLi/XSS vulnerabilities.
+# Valid: True or False
+skipHeuristics = False
+
 # Skip heuristic detection of WAF/IPS protection.
 # Valid: True or False
 skipWaf = False

tamper/0eunion.py

@@ -29,4 +29,4 @@ def tamper(payload, **kwargs):
     '1e0UNION ALL SELECT'
     """
 
-    return re.sub(r"(\d+)\s+(UNION )", r"\g<1>e0\g<2>", payload, re.I) if payload else payload
+    return re.sub(r"(?i)(\d+)\s+(UNION )", r"\g<1>e0\g<2>", payload) if payload else payload

tamper/dunion.py

@@ -31,4 +31,4 @@ def tamper(payload, **kwargs):
     '1DUNION ALL SELECT'
     """
 
-    return re.sub(r"(\d+)\s+(UNION )", r"\g<1>D\g<2>", payload, re.I) if payload else payload
+    return re.sub(r"(?i)(\d+)\s+(UNION )", r"\g<1>D\g<2>", payload) if payload else payload

tamper/misunion.py

@@ -33,4 +33,4 @@ def tamper(payload, **kwargs):
     '1"-.1UNION ALL SELECT'
     """
 
-    return re.sub(r"\s+(UNION )", r"-.1\g<1>", payload, re.I) if payload else payload
+    return re.sub(r"(?i)\s+(UNION )", r"-.1\g<1>", payload) if payload else payload

tamper/schemasplit.py

@@ -28,4 +28,4 @@ def tamper(payload, **kwargs):
     'SELECT id FROM testdb 9.e.users'
     """
 
-    return re.sub(r"( FROM \w+)\.(\w+)", r"\g<1> 9.e.\g<2>", payload, re.I) if payload else payload
+    return re.sub(r"(?i)( FROM \w+)\.(\w+)", r"\g<1> 9.e.\g<2>", payload) if payload else payload