Fixes #4413

* Make asterisk work with --csrf-token option

* add --file-write support in HSQLDB

Co-authored-by: tree <[email protected]>

data/xml/queries.xml

@@ -198,11 +198,11 @@
             <blind query="SELECT TOP 1 name FROM master..sysdatabases WHERE name NOT IN (SELECT TOP %d name FROM master..sysdatabases ORDER BY name) ORDER BY name" count="SELECT LTRIM(STR(COUNT(name))) FROM master..sysdatabases"/>
         </dbs>
         <tables>
-            <inband query="SELECT %s..sysusers.name+'.'+%s..sysobjects.name FROM %s..sysobjects INNER JOIN %s..sysusers ON %s..sysobjects.uid=%s..sysusers.uid WHERE %s..sysobjects.xtype IN ('u','v')" query2="SELECT table_schema+'.'+table_name FROM information_schema.tables WHERE table_catalog='%s'" query3="SELECT name FROM %s..sysobjects WHERE xtype='U'"/>
+            <inband query="SELECT %s..sysusers.name+'.'+%s..sysobjects.name AS table_name FROM %s..sysobjects INNER JOIN %s..sysusers ON %s..sysobjects.uid=%s..sysusers.uid WHERE %s..sysobjects.xtype IN ('u','v')" query2="SELECT table_schema+'.'+table_name FROM information_schema.tables WHERE table_catalog='%s'" query3="SELECT name FROM %s..sysobjects WHERE xtype='U'"/>
             <blind query="SELECT TOP 1 %s..sysusers.name+'.'+%s..sysobjects.name FROM %s..sysobjects INNER JOIN %s..sysusers ON %s..sysobjects.uid=%s..sysusers.uid WHERE %s..sysobjects.xtype IN ('u','v') AND %s..sysusers.name+'.'+%s..sysobjects.name NOT IN (SELECT TOP %d %s..sysusers.name+'.'+%s..sysobjects.name FROM %s..sysobjects INNER JOIN %s..sysusers ON %s..sysobjects.uid=%s..sysusers.uid WHERE %s..sysobjects.xtype IN ('u','v') ORDER BY %s..sysusers.name+'.'+%s..sysobjects.name) ORDER BY %s..sysusers.name+'.'+%s..sysobjects.name" count="SELECT LTRIM(STR(COUNT(name))) FROM %s..sysobjects WHERE %s..sysobjects.xtype IN ('u','v')" query2="SELECT TOP 1 table_schema+'.'+table_name FROM information_schema.tables WHERE table_catalog='%s' AND table_schema+'.'+table_name NOT IN (SELECT TOP %d table_schema+'.'+table_name FROM information_schema.tables WHERE table_catalog='%s' ORDER BY table_schema+'.'+table_name) ORDER BY table_schema+'.'+table_name" count2="SELECT LTRIM(STR(COUNT(table_name))) FROM information_schema.tables WHERE table_catalog='%s'" query3="SELECT TOP 1 name FROM %s..sysobjects WHERE xtype='U' AND name NOT IN (SELECT TOP %d name FROM %s..sysobjects WHERE xtype='U' ORDER BY name) ORDER BY name" count3="SELECT COUNT(name) FROM %s..sysobjects WHERE xtype='U'"/>
         </tables>
         <columns>
-            <inband query="SELECT %s..syscolumns.name,TYPE_NAME(%s..syscolumns.xtype) FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s'" query2="SELECT COL_NAME(OBJECT_ID('%s.%s'),%d)" condition="[DB]..syscolumns.name"/>
+            <inband query="SELECT %s..syscolumns.name,TYPE_NAME(%s..syscolumns.xtype) AS type_name FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s'" query2="SELECT COL_NAME(OBJECT_ID('%s.%s'),%d)" condition="[DB]..syscolumns.name"/>
             <blind query="SELECT TOP 1 %s..syscolumns.name FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s' AND %s..syscolumns.name NOT IN (SELECT TOP %d %s..syscolumns.name FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s' ORDER BY %s..syscolumns.name) ORDER BY %s..syscolumns.name" query2="SELECT TYPE_NAME(%s..syscolumns.xtype) FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.name='%s' AND %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s'" query3="SELECT COL_NAME(OBJECT_ID('%s.%s'),%d)" count="SELECT LTRIM(STR(COUNT(name))) FROM %s..syscolumns WHERE id=(SELECT id FROM %s..sysobjects WHERE name='%s')" condition="[DB]..syscolumns.name"/>
         </columns>
         <dump_table>

extra/shutils/drei.sh

@@ -7,7 +7,7 @@
 
 export SQLMAP_DREI=1
 #for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done
-for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i | sed 's/Compiling/Checking/g'; done
+for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3 -m compileall $i | sed 's/Compiling/Checking/g'; done
 unset SQLMAP_DREI
 source `dirname "$0"`"/junk.sh"
 

lib/controller/controller.py

@@ -291,7 +291,7 @@ def start():
         logger.error(errMsg)
         return False
 
-    if kb.targets and len(kb.targets) > 1:
+    if kb.targets and isListLike(kb.targets) and len(kb.targets) > 1:
         infoMsg = "found a total of %d targets" % len(kb.targets)
         logger.info(infoMsg)
 
@@ -704,6 +704,12 @@ def start():
                     action()
 
         except KeyboardInterrupt:
+            if kb.lastCtrlCTime and (time.time() - kb.lastCtrlCTime < 1):
+                kb.multipleCtrlC = True
+                raise SqlmapUserQuitException("user aborted (Ctrl+C was pressed multiple times)")
+
+            kb.lastCtrlCTime = time.time()
+
             if conf.multipleTargets:
                 warnMsg = "user aborted in multiple target mode"
                 logger.warn(warnMsg)

lib/core/agent.py

@@ -1207,7 +1207,7 @@ class Agent(object):
 
     def whereQuery(self, query):
         if conf.dumpWhere and query:
-            if Backend.isDbms(DBMS.ORACLE) and re.search("qq ORDER BY \w+\)", query, re.I) is not None:
+            if Backend.isDbms(DBMS.ORACLE) and re.search(r"qq ORDER BY \w+\)", query, re.I) is not None:
                 prefix, suffix = re.sub(r"(?i)(qq)( ORDER BY \w+\))", r"\g<1> WHERE %s\g<2>" % conf.dumpWhere, query), ""
             else:
                 match = re.search(r" (LIMIT|ORDER).+", query, re.I)
@@ -1225,7 +1225,7 @@ class Agent(object):
                 prefix += " WHERE %s" % conf.dumpWhere
 
             query = prefix
-            if suffix:
+            if suffix and not all(re.search(r"ORDER BY", _, re.I) is not None for _ in (query, suffix)):
                 query += suffix
 
         return query

lib/core/common.py

@@ -215,7 +215,7 @@ class UnicodeRawConfigParser(_configparser.RawConfigParser):
             fp.write("[%s]\n" % _configparser.DEFAULTSECT)
 
             for (key, value) in self._defaults.items():
-                fp.write("%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING).replace('\n', '\n\t')))
+                fp.write("\t%s = %s" % (key, getUnicode(value, UNICODE_ENCODING)))
 
             fp.write("\n")
 
@@ -225,9 +225,9 @@ class UnicodeRawConfigParser(_configparser.RawConfigParser):
             for (key, value) in self._sections[section].items():
                 if key != "__name__":
                     if value is None:
-                        fp.write("%s\n" % (key))
-                    else:
-                        fp.write("%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING).replace('\n', '\n\t')))
+                        fp.write("\t%s\n" % (key))
+                    elif not isListLike(value):
+                        fp.write("\t%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING)))
 
             fp.write("\n")
 
@@ -1749,7 +1749,7 @@ def expandAsteriskForColumns(expression):
     the SQL query string (expression)
     """
 
-    match = re.search(r"(?i)\ASELECT(\s+TOP\s+[\d]+)?\s+\*\s+FROM\s+`?([^`\s()]+)", expression)
+    match = re.search(r"(?i)\ASELECT(\s+TOP\s+[\d]+)?\s+\*\s+FROM\s+((`[^`]+`|[^\s]+)+)", expression)
 
     if match:
         infoMsg = "you did not provide the fields in your query. "
@@ -2071,6 +2071,8 @@ def safeStringFormat(format_, params):
 
     >>> safeStringFormat('SELECT foo FROM %s LIMIT %d', ('bar', '1'))
     'SELECT foo FROM bar LIMIT 1'
+    >>> safeStringFormat("SELECT foo FROM %s WHERE name LIKE '%susan%' LIMIT %d", ('bar', '1'))
+    "SELECT foo FROM bar WHERE name LIKE '%susan%' LIMIT 1"
     """
 
     if format_.count(PAYLOAD_DELIMITER) == 2:
@@ -2114,7 +2116,10 @@ def safeStringFormat(format_, params):
                         warnMsg += "Please report by e-mail content \"%r | %r | %r\" to '%s'" % (format_, params, retVal, DEV_EMAIL_ADDRESS)
                         raise SqlmapValueException(warnMsg)
                     else:
-                        retVal = re.sub(r"(\A|[^A-Za-z0-9])(%s)([^A-Za-z0-9]|\Z)", r"\g<1>%s\g<3>" % params[count], retVal, 1)
+                        try:
+                            retVal = re.sub(r"(\A|[^A-Za-z0-9])(%s)([^A-Za-z0-9]|\Z)", r"\g<1>%s\g<3>" % params[count], retVal, 1)
+                        except re.error:
+                            retVal = retVal.replace(match.group(0), match.group(0) % params[count], 1)
                         count += 1
                 else:
                     break
@@ -3609,7 +3614,7 @@ def isListLike(value):
     False
     """
 
-    return isinstance(value, (list, tuple, set, BigArray))
+    return isinstance(value, (list, tuple, set, OrderedSet, BigArray))
 
 def getSortedInjectionTests():
     """

lib/core/option.py

@@ -8,6 +8,7 @@ See the file 'LICENSE' for copying permission
 from __future__ import division
 
 import codecs
+import collections
 import functools
 import glob
 import inspect
@@ -412,6 +413,37 @@ def _doSearch():
             else:
                 conf.googlePage += 1
 
+def _setStdinPipeTargets():
+    if isinstance(conf.stdinPipe, collections.Iterable):
+        infoMsg = "using 'STDIN' for parsing targets list"
+        logger.info(infoMsg)
+
+        class _(object):
+            def __init__(self):
+                self.__rest = OrderedSet()
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                return self.next()
+
+            def next(self):
+                line = next(conf.stdinPipe)
+                if line:
+                    match = re.search(r"\b(https?://[^\s'\"]+|[\w.]+\.\w{2,3}[/\w+]*\?[^\s'\"]+)", line, re.I)
+                    if match:
+                        return (match.group(0), conf.method, conf.data, conf.cookie, None)
+                elif self.__rest:
+                    return self.__rest.pop()
+
+                raise StopIteration()
+
+            def add(self, elem):
+                self.__rest.add(elem)
+
+        kb.targets = _()
+
 def _setBulkMultipleTargets():
     if not conf.bulkFile:
         return
@@ -431,7 +463,7 @@ def _setBulkMultipleTargets():
         if conf.scope and not re.search(conf.scope, line, re.I):
             continue
 
-        if re.match(r"[^ ]+\?(.+)", line, re.I) or kb.customInjectionMark in line:
+        if re.match(r"[^ ]+\?(.+)", line, re.I) or kb.customInjectionMark in line or conf.data:
             found = True
             kb.targets.add((line.strip(), conf.method, conf.data, conf.cookie, None))
 
@@ -1631,7 +1663,8 @@ def _cleanupOptions():
 
     for key, value in conf.items():
         if value and any(key.endswith(_) for _ in ("Path", "File", "Dir")):
-            conf[key] = safeExpandUser(value)
+            if isinstance(value, str):
+                conf[key] = safeExpandUser(value)
 
     if conf.testParameter:
         conf.testParameter = urldecode(conf.testParameter)
@@ -1658,7 +1691,7 @@ def _cleanupOptions():
 
     if conf.base64Parameter:
         conf.base64Parameter = urldecode(conf.base64Parameter)
-        conf.base64Parameter = conf.base64Parameter.replace(" ", "")
+        conf.base64Parameter = conf.base64Parameter.strip()
         conf.base64Parameter = re.split(PARAMETER_SPLITTING_REGEX, conf.base64Parameter)
     else:
         conf.base64Parameter = []
@@ -2025,12 +2058,12 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.identifiedWafs = set()
     kb.injection = InjectionDict()
     kb.injections = []
+    kb.jsonAggMode = False
     kb.laggingChecked = False
     kb.lastParserStatus = None
-    kb.lastCtrlCTime = None
 
     kb.locks = AttribDict()
-    for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
+    for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "liveCookies", "log", "socket", "redirect", "request", "value"):
         kb.locks[_] = threading.Lock()
 
     kb.matchRatio = None
@@ -2080,7 +2113,6 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.responseTimeMode = None
     kb.responseTimePayload = None
     kb.resumeValues = True
-    kb.rowXmlMode = False
     kb.safeCharEncode = False
     kb.safeReq = AttribDict()
     kb.secondReq = None
@@ -2116,6 +2148,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
         kb.checkSitemap = None
         kb.headerPaths = {}
         kb.keywords = set(getFileItems(paths.SQL_KEYWORDS))
+        kb.lastCtrlCTime = None
         kb.normalizeCrawlingChoice = None
         kb.passwordMgr = None
         kb.postprocessFunctions = []
@@ -2773,7 +2806,7 @@ def init():
 
     parseTargetDirect()
 
-    if any((conf.url, conf.logFile, conf.bulkFile, conf.requestFile, conf.googleDork)):
+    if any((conf.url, conf.logFile, conf.bulkFile, conf.requestFile, conf.googleDork, conf.stdinPipe)):
         _setHostname()
         _setHTTPTimeout()
         _setHTTPExtraHeaders()
@@ -2787,6 +2820,7 @@ def init():
         _setSocketPreConnect()
         _setSafeVisit()
         _doSearch()
+        _setStdinPipeTargets()
         _setBulkMultipleTargets()
         _checkTor()
         _setCrawler()

lib/core/optiondict.py

@@ -27,6 +27,7 @@ optDict = {
         "paramDel": "string",
         "cookie": "string",
         "cookieDel": "string",
+        "liveCookies": "string",
         "loadCookies": "string",
         "dropSetCookie": "boolean",
         "agent": "string",

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.4.10.0"
+VERSION = "1.4.11.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -52,6 +52,9 @@ IPS_WAF_CHECK_RATIO = 0.5
 # Timeout used in heuristic check for WAF/IPS protected targets
 IPS_WAF_CHECK_TIMEOUT = 10
 
+# Timeout used in checking for existence of live-cookies file
+LIVE_COOKIES_TIMEOUT = 120
+
 # Lower and upper values for match ratio in case of stable page
 LOWER_RATIO_BOUND = 0.02
 UPPER_RATIO_BOUND = 0.98

lib/core/testing.py

@@ -47,6 +47,7 @@ def vulnTest():
         ("-u '<url>&id2=1' -p id2 -v 5 --flush-session --level=5 --test-filter='AND boolean-based blind - WHERE or HAVING clause (MySQL comment)'", ("~1AND",)),
         ("--list-tampers", ("between", "MySQL", "xforwardedfor")),
         ("-r <request> --flush-session -v 5 --test-skip='heavy' --save=<tmp>", ("CloudFlare", "possible DBMS: 'SQLite'", "User-agent: foobar", "~Type: time-based blind")),
+        ("<piped> -r <request> -l <log> --flush-session --banner --technique=B", ("banner: '3.", "STDIN")),
         ("-l <log> --flush-session --keep-alive --skip-waf -v 5 --technique=U --union-from=users --banner --parse-errors", ("banner: '3.", "ORDER BY term out of range", "~xp_cmdshell", "Connection: keep-alive")),
         ("-l <log> --offline --banner -v 5", ("banner: '3.", "~[TRAFFIC OUT]")),
         ("-u <base64> -p id --base64=id --data='base64=true' --flush-session --banner --technique=B", ("banner: '3.",)),
@@ -138,6 +139,10 @@ def vulnTest():
             os.close(handle)
             cmd = cmd.replace("<tmp>", tmp)
 
+        if "<piped>" in cmd:
+            cmd = re.sub(r"<piped>\s*", "", cmd)
+            cmd = "echo %s | %s" % (url, cmd)
+
         output = shellExec(cmd)
 
         if not all((check in output if not check.startswith('~') else check[1:] not in output) for check in checks) or "unhandled exception" in output:

lib/parse/cmdline.py

@@ -168,6 +168,9 @@ def cmdLineParser(argv=None):
         request.add_argument("--cookie-del", dest="cookieDel",
             help="Character used for splitting cookie values (e.g. ;)")
 
+        request.add_argument("--live-cookies", dest="liveCookies",
+            help="Live cookies file used for loading up-to-date values")
+
         request.add_argument("--load-cookies", dest="loadCookies",
             help="File containing cookies in Netscape/wget format")
 
@@ -1035,7 +1038,12 @@ def cmdLineParser(argv=None):
         if args.dummy:
             args.url = args.url or DUMMY_URL
 
-        if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.updateAll, args.smokeTest, args.vulnTest, args.bedTest, args.fuzzTest, args.wizard, args.dependencies, args.purge, args.listTampers, args.hashFile)):
+        if hasattr(sys.stdin, "fileno") and not os.isatty(sys.stdin.fileno()) and '-' not in sys.argv:
+            args.stdinPipe = iter(sys.stdin.readline, None)
+        else:
+            args.stdinPipe = None
+
+        if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.updateAll, args.smokeTest, args.vulnTest, args.bedTest, args.fuzzTest, args.wizard, args.dependencies, args.purge, args.listTampers, args.hashFile, args.stdinPipe)):
             errMsg = "missing a mandatory option (-d, -u, -l, -m, -r, -g, -c, --list-tampers, --wizard, --update, --purge or --dependencies). "
             errMsg += "Use -h for basic and -hh for advanced help\n"
             parser.error(errMsg)

lib/request/connect.py

@@ -7,6 +7,7 @@ See the file 'LICENSE' for copying permission
 
 import binascii
 import logging
+import os
 import random
 import re
 import socket
@@ -25,6 +26,7 @@ except ImportError:
 from lib.core.agent import agent
 from lib.core.common import asciifyUrl
 from lib.core.common import calculateDeltaSeconds
+from lib.core.common import checkFile
 from lib.core.common import checkSameHost
 from lib.core.common import chunkSplitPostData
 from lib.core.common import clearConsoleLine
@@ -100,6 +102,7 @@ from lib.core.settings import IPS_WAF_CHECK_PAYLOAD
 from lib.core.settings import IS_WIN
 from lib.core.settings import JAVASCRIPT_HREF_REGEX
 from lib.core.settings import LARGE_READ_TRIM_MARKER
+from lib.core.settings import LIVE_COOKIES_TIMEOUT
 from lib.core.settings import MAX_CONNECTION_READ_SIZE
 from lib.core.settings import MAX_CONNECTIONS_REGEX
 from lib.core.settings import MAX_CONNECTION_TOTAL_SIZE
@@ -292,6 +295,30 @@ class Connect(object):
 
             return page, headers, code
 
+        if conf.liveCookies:
+            with kb.locks.liveCookies:
+                if not checkFile(conf.liveCookies, raiseOnError=False) or os.path.getsize(conf.liveCookies) == 0:
+                    warnMsg = "[%s] [WARNING] live cookies file '%s' is empty or non-existent. Waiting for timeout (%d seconds)" % (time.strftime("%X"), conf.liveCookies, LIVE_COOKIES_TIMEOUT)
+                    dataToStdout(warnMsg)
+
+                    valid = False
+                    for _ in xrange(LIVE_COOKIES_TIMEOUT):
+                        if checkFile(conf.liveCookies, raiseOnError=False) and os.path.getsize(conf.liveCookies) > 0:
+                            valid = True
+                            break
+                        else:
+                            dataToStdout('.')
+                            time.sleep(1)
+
+                    dataToStdout("\n")
+
+                    if not valid:
+                        errMsg = "problem occurred while loading cookies from file '%s'" % conf.liveCookies
+                        raise SqlmapValueException(errMsg)
+
+                cookie = openFile(conf.liveCookies).read().strip()
+                cookie = re.sub(r"(?i)\ACookie:\s*", "", cookie)
+
         if multipart:
             post = multipart
         else:

lib/request/inject.py

@@ -126,7 +126,7 @@ def _goInference(payload, expression, charsetType=None, firstChar=None, lastChar
         kb.inferenceMode = False
 
         if not kb.bruteMode:
-            debugMsg = "performed %d queries in %.2f seconds" % (count, calculateDeltaSeconds(start))
+            debugMsg = "performed %d quer%s in %.2f seconds" % (count, 'y' if count == 1 else "ies", calculateDeltaSeconds(start))
             logger.debug(debugMsg)
 
     return value

lib/takeover/metasploit.py

@@ -506,7 +506,7 @@ class Metasploit(object):
                 if IS_WIN:
                     timeout = 3
 
-                    inp = ""
+                    inp = b""
                     _ = time.time()
 
                     while True:

lib/techniques/blind/inference.py

@@ -719,7 +719,7 @@ def queryOutputLength(expression, payload):
     lengthExprUnescaped = agent.forgeQueryOutputLength(expression)
     count, length = bisection(payload, lengthExprUnescaped, charsetType=CHARSET_TYPE.DIGITS)
 
-    debugMsg = "performed %d queries in %.2f seconds" % (count, calculateDeltaSeconds(start))
+    debugMsg = "performed %d quer%s in %.2f seconds" % (count, 'y' if count == 1 else "ies", calculateDeltaSeconds(start))
     logger.debug(debugMsg)
 
     if length == " ":

lib/techniques/dns/use.py

@@ -108,7 +108,7 @@ def dnsUse(payload, expression):
                     hashDBWrite(expression, output)
 
         if not kb.bruteMode:
-            debugMsg = "performed %d queries in %.2f seconds" % (count, calculateDeltaSeconds(start))
+            debugMsg = "performed %d quer%s in %.2f seconds" % (count, 'y' if count == 1 else "ies", calculateDeltaSeconds(start))
             logger.debug(debugMsg)
 
     elif conf.dnsDomain:

lib/techniques/error/use.py

@@ -462,7 +462,7 @@ def errorUse(expression, dump=False):
     duration = calculateDeltaSeconds(start)
 
     if not kb.bruteMode:
-        debugMsg = "performed %d queries in %.2f seconds" % (kb.counters[getTechnique()], duration)
+        debugMsg = "performed %d quer%s in %.2f seconds" % (kb.counters[getTechnique()], 'y' if kb.counters[getTechnique()] == 1 else "ies", duration)
         logger.debug(debugMsg)
 
     return value

lib/techniques/union/use.py

@@ -5,10 +5,9 @@ Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-import binascii
+import json
 import re
 import time
-import xml.etree.ElementTree
 
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
@@ -32,14 +31,11 @@ from lib.core.common import isNumPosStrValue
 from lib.core.common import listToStrValue
 from lib.core.common import parseUnionPage
 from lib.core.common import removeReflectiveValues
-from lib.core.common import safeStringFormat
 from lib.core.common import singleTimeDebugMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unArrayizeValue
 from lib.core.common import wasLastResponseDBMSError
 from lib.core.compat import xrange
-from lib.core.convert import decodeBase64
-from lib.core.convert import getBytes
 from lib.core.convert import getUnicode
 from lib.core.convert import htmlUnescape
 from lib.core.data import conf
@@ -74,7 +70,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
     if retVal is None:
         vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector
 
-        if not kb.rowXmlMode:
+        if not kb.jsonAggMode:
             injExpression = unescaper.escape(agent.concatQuery(expression, unpack))
             kb.unionDuplicates = vector[7]
             kb.forcePartialUnion = vector[8]
@@ -99,7 +95,35 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
 
         incrementCounter(PAYLOAD.TECHNIQUE.UNION)
 
-        if not kb.rowXmlMode:
+        if kb.jsonAggMode:
+            if Backend.isDbms(DBMS.MSSQL):
+                output = extractRegexResult(r"%s(?P<result>.*)%s" % (kb.chars.start, kb.chars.stop), page or "")
+                if output:
+                    try:
+                        retVal = ""
+                        fields = re.findall(r'"([^"]+)":', extractRegexResult(r"{(?P<result>[^}]+)}", output))
+                        for row in json.loads(output):
+                            retVal += "%s%s%s" % (kb.chars.start, kb.chars.delimiter.join(getUnicode(row[field] or NULL) for field in fields), kb.chars.stop)
+                    except:
+                        pass
+                    else:
+                        retVal = getUnicode(retVal)
+            elif Backend.isDbms(DBMS.PGSQL):
+                output = extractRegexResult(r"(?P<result>%s.*%s)" % (kb.chars.start, kb.chars.stop), page or "")
+                if output:
+                    retVal = output
+            else:
+                output = extractRegexResult(r"%s(?P<result>.*?)%s" % (kb.chars.start, kb.chars.stop), page or "")
+                if output:
+                    try:
+                        retVal = ""
+                        for row in json.loads(output):
+                            retVal += "%s%s%s" % (kb.chars.start, row, kb.chars.stop)
+                    except:
+                        pass
+                    else:
+                        retVal = getUnicode(retVal)
+        else:
             # Parse the returned page to get the exact UNION-based
             # SQL injection output
             def _(regex):
@@ -115,40 +139,6 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
                 page = page.replace(kb.chars.stop[:-1], kb.chars.stop)
 
             retVal = _("(?P<result>%s.*%s)" % (kb.chars.start, kb.chars.stop))
-        else:
-            output = extractRegexResult(r"(?P<result>(<row.+?/>)+)", page)
-            if output:
-                try:
-                    root = xml.etree.ElementTree.fromstring(safeStringFormat("<root>%s</root>", getBytes(output)))
-                    retVal = ""
-                    for column in kb.dumpColumns:
-                        base64 = True
-                        for child in root:
-                            value = child.attrib.get(column, "").strip()
-                            if value and not re.match(r"\A[a-zA-Z0-9+/]+={0,2}\Z", value):
-                                base64 = False
-                                break
-
-                            try:
-                                decodeBase64(value)
-                            except (binascii.Error, TypeError):
-                                base64 = False
-                                break
-
-                        if base64:
-                            for child in root:
-                                child.attrib[column] = decodeBase64(child.attrib.get(column, ""), binary=False) or NULL
-
-                    for child in root:
-                        row = []
-                        for column in kb.dumpColumns:
-                            row.append(child.attrib.get(column, NULL))
-                        retVal += "%s%s%s" % (kb.chars.start, kb.chars.delimiter.join(row), kb.chars.stop)
-
-                except:
-                    pass
-                else:
-                    retVal = getUnicode(retVal)
 
         if retVal is not None:
             retVal = getUnicode(retVal, kb.pageEncoding)
@@ -159,7 +149,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
 
             hashDBWrite("%s%s" % (conf.hexConvert or False, expression), retVal)
 
-        elif not kb.rowXmlMode:
+        elif not kb.jsonAggMode:
             trimmed = _("%s(?P<result>.*?)<" % (kb.chars.start))
 
             if trimmed:
@@ -169,7 +159,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
                 logger.warn(warnMsg)
             elif re.search(r"ORDER BY [^ ]+\Z", expression):
                 debugMsg = "retrying failed SQL query without the ORDER BY clause"
-                logger.debug(debugMsg)
+                singleTimeDebugMessage(debugMsg)
 
                 expression = re.sub(r"\s*ORDER BY [^ ]+\Z", "", expression)
                 retVal = _oneShotUnionUse(expression, unpack, limited)
@@ -236,13 +226,6 @@ def unionUse(expression, unpack=True, dump=False):
     # Set kb.partRun in case the engine is called from the API
     kb.partRun = getPartRun(alias=False) if conf.api else None
 
-    if Backend.isDbms(DBMS.MSSQL) and kb.dumpColumns:
-        kb.rowXmlMode = True
-        _ = "(%s FOR XML RAW, BINARY BASE64)" % expression
-        output = _oneShotUnionUse(_, False)
-        value = parseUnionPage(output)
-        kb.rowXmlMode = False
-
     if expressionFieldsList and len(expressionFieldsList) > 1 and "ORDER BY" in expression.upper():
         # Removed ORDER BY clause because UNION does not play well with it
         expression = re.sub(r"(?i)\s*ORDER BY\s+[\w,]+", "", expression)
@@ -250,6 +233,22 @@ def unionUse(expression, unpack=True, dump=False):
         debugMsg += "it does not play well with UNION query SQL injection"
         singleTimeDebugMessage(debugMsg)
 
+    if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL) and expressionFields:
+        match = re.search(r"SELECT\s*(.+?)\bFROM", expression, re.I)
+        if match and not (Backend.isDbms(DBMS.ORACLE) and FROM_DUMMY_TABLE[DBMS.ORACLE] in expression):
+            kb.jsonAggMode = True
+            if Backend.isDbms(DBMS.MYSQL):
+                query = expression.replace(expressionFields, "CONCAT('%s',JSON_ARRAYAGG(CONCAT_WS('%s',%s)),'%s')" % (kb.chars.start, kb.chars.delimiter, expressionFields, kb.chars.stop), 1)
+            elif Backend.isDbms(DBMS.ORACLE):
+                query = expression.replace(expressionFields, "'%s'||JSON_ARRAYAGG(%s)||'%s'" % (kb.chars.start, ("||'%s'||" % kb.chars.delimiter).join(expressionFieldsList), kb.chars.stop), 1)
+            elif Backend.isDbms(DBMS.PGSQL):    # Note: ARRAY_AGG does CSV alike output, thus enclosing start/end inside each item
+                query = expression.replace(expressionFields, "ARRAY_AGG('%s'||%s||'%s')::text" % (kb.chars.start, ("||'%s'||" % kb.chars.delimiter).join("COALESCE(%s::text,' ')" % field for field in expressionFieldsList), kb.chars.stop), 1)
+            elif Backend.isDbms(DBMS.MSSQL):
+                query = "'%s'+(%s FOR JSON AUTO, INCLUDE_NULL_VALUES)+'%s'" % (kb.chars.start, expression, kb.chars.stop)
+            output = _oneShotUnionUse(query, False)
+            value = parseUnionPage(output)
+            kb.jsonAggMode = False
+
     # We have to check if the SQL query might return multiple entries
     # if the technique is partial UNION query and in such case forge the
     # SQL limiting the query output one entry at a time
@@ -425,7 +424,7 @@ def unionUse(expression, unpack=True, dump=False):
     duration = calculateDeltaSeconds(start)
 
     if not kb.bruteMode:
-        debugMsg = "performed %d queries in %.2f seconds" % (kb.counters[PAYLOAD.TECHNIQUE.UNION], duration)
+        debugMsg = "performed %d quer%s in %.2f seconds" % (kb.counters[PAYLOAD.TECHNIQUE.UNION], 'y' if kb.counters[PAYLOAD.TECHNIQUE.UNION] == 1 else "ies", duration)
         logger.debug(debugMsg)
 
     return value

lib/utils/hash.py

@@ -1147,6 +1147,12 @@ def dictionaryAttack(attack_dict):
                     warnMsg = "user aborted during dictionary-based attack phase (Ctrl+C was pressed)"
                     logger.warn(warnMsg)
 
+                finally:
+                    if _multiprocessing:
+                        gc.enable()
+
+                    # NOTE: https://github.com/sqlmapproject/sqlmap/issues/4367
+                    # NOTE: https://dzone.com/articles/python-101-creating-multiple-processes
                     for process in processes:
                         try:
                             process.terminate()
@@ -1154,10 +1160,6 @@ def dictionaryAttack(attack_dict):
                         except (OSError, AttributeError):
                             pass
 
-                finally:
-                    if _multiprocessing:
-                        gc.enable()
-
                     if retVal:
                         if conf.hashDB:
                             conf.hashDB.beginTransaction()

plugins/dbms/hsqldb/filesystem.py

@@ -5,6 +5,12 @@ Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from lib.core.common import randomStr
+from lib.core.data import kb
+from lib.core.data import logger
+from lib.core.decorators import stackedmethod
+from lib.core.enums import PLACE
+from lib.request import inject
 from lib.core.exception import SqlmapUnsupportedFeatureException
 from plugins.generic.filesystem import Filesystem as GenericFilesystem
 
@@ -13,6 +19,41 @@ class Filesystem(GenericFilesystem):
         errMsg = "on HSQLDB it is not possible to read files"
         raise SqlmapUnsupportedFeatureException(errMsg)
 
-    def writeFile(self, localFile, remoteFile, fileType=None, forceCheck=False):
-        errMsg = "on HSQLDB it is not possible to write files"
-        raise SqlmapUnsupportedFeatureException(errMsg)
+    @stackedmethod
+    def stackedWriteFile(self, localFile, remoteFile, fileType=None, forceCheck=False):
+        funcName = randomStr()
+        max_bytes = 1024 * 1024
+
+        debugMsg = "creating JLP procedure '%s'" % funcName
+        logger.debug(debugMsg)
+
+        addFuncQuery = "CREATE PROCEDURE %s (IN paramString VARCHAR, IN paramArrayOfByte VARBINARY(%s)) " % (funcName, max_bytes)
+        addFuncQuery += "LANGUAGE JAVA DETERMINISTIC NO SQL "
+        addFuncQuery += "EXTERNAL NAME 'CLASSPATH:com.sun.org.apache.xml.internal.security.utils.JavaUtils.writeBytesToFilename'"
+        inject.goStacked(addFuncQuery)
+
+        fcEncodedList = self.fileEncode(localFile, "hex", True)
+        fcEncodedStr = fcEncodedList[0][2:]
+        fcEncodedStrLen = len(fcEncodedStr)
+
+        if kb.injection.place == PLACE.GET and fcEncodedStrLen > 8000:
+            warnMsg = "as the injection is on a GET parameter and the file "
+            warnMsg += "to be written hexadecimal value is %d " % fcEncodedStrLen
+            warnMsg += "bytes, this might cause errors in the file "
+            warnMsg += "writing process"
+            logger.warn(warnMsg)
+
+        debugMsg = "exporting the %s file content to file '%s'" % (fileType, remoteFile)
+        logger.debug(debugMsg)
+
+        # Reference: http://hsqldb.org/doc/guide/sqlroutines-chapt.html#src_jrt_procedures
+        invokeQuery = "CALL %s('%s', CAST('%s' AS VARBINARY(%s)))" % (funcName, remoteFile, fcEncodedStr, max_bytes)
+        inject.goStacked(invokeQuery)
+
+        logger.debug("cleaning up" % funcName)
+        delQuery = "DELETE PROCEDURE %s" % funcName
+        inject.goStacked(delQuery)
+
+        message = "the local file '%s' has been written on the back-end DBMS" % localFile
+        message += "file system ('%s')" % remoteFile
+        logger.info(message)

plugins/dbms/hsqldb/fingerprint.py

@@ -144,3 +144,13 @@ class Fingerprint(GenericFingerprint):
     def getHostname(self):
         warnMsg = "on HSQLDB it is not possible to enumerate the hostname"
         logger.warn(warnMsg)
+
+
+    def checkDbmsOs(self, detailed=False):
+        if Backend.getOs():
+            infoMsg = "the back-end DBMS operating system is %s" % Backend.getOs()
+            logger.info(infoMsg)
+        else:
+            self.userChooseDbmsOs()
+
+

plugins/dbms/mysql/filesystem.py

@@ -96,7 +96,7 @@ class Filesystem(GenericFilesystem):
         fcEncodedStrLen = len(fcEncodedStr)
 
         if kb.injection.place == PLACE.GET and fcEncodedStrLen > 8000:
-            warnMsg = "the injection is on a GET parameter and the file "
+            warnMsg = "as the injection is on a GET parameter and the file "
             warnMsg += "to be written hexadecimal value is %d " % fcEncodedStrLen
             warnMsg += "bytes, this might cause errors in the file "
             warnMsg += "writing process"

plugins/dbms/postgresql/takeover.py

@@ -11,6 +11,7 @@ from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import decloakToTemp
 from lib.core.common import flattenValue
+from lib.core.common import filterNone
 from lib.core.common import isListLike
 from lib.core.common import isNoneValue
 from lib.core.common import isStackingAvailable
@@ -107,6 +108,7 @@ class Takeover(GenericTakeover):
 
             if isListLike(output):
                 output = flattenValue(output)
+                output = filterNone(output)
 
             if not isNoneValue(output):
                 output = os.linesep.join(output)

plugins/generic/databases.py

@@ -10,7 +10,6 @@ import re
 from lib.core.agent import agent
 from lib.core.common import arrayizeValue
 from lib.core.common import Backend
-from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
 from lib.core.common import filterPairValues
 from lib.core.common import flattenValue
@@ -23,7 +22,6 @@ from lib.core.common import isTechniqueAvailable
 from lib.core.common import parseSqliteTableSchema
 from lib.core.common import popValue
 from lib.core.common import pushValue
-from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import safeSQLIdentificatorNaming
 from lib.core.common import safeStringFormat
@@ -54,7 +52,6 @@ from lib.core.settings import REFLECTED_VALUE_MARKER
 from lib.core.settings import UPPER_CASE_DBMSES
 from lib.core.settings import VERTICA_DEFAULT_SCHEMA
 from lib.request import inject
-from lib.techniques.union.use import unionUse
 from lib.utils.brute import columnExists
 from lib.utils.brute import tableExists
 from thirdparty import six
@@ -639,18 +636,6 @@ class Databases(object):
                     logger.info(infoMsg)
 
                     values = None
-                    if Backend.isDbms(DBMS.MSSQL) and isTechniqueAvailable(PAYLOAD.TECHNIQUE.UNION):
-                        expression = query
-                        kb.dumpColumns = []
-                        kb.rowXmlMode = True
-
-                        for column in (extractRegexResult(r"SELECT (?P<result>.+?) FROM", query) or "").split(','):
-                            kb.dumpColumns.append(randomStr().lower())
-                            expression = expression.replace(column, "%s AS %s" % (column, kb.dumpColumns[-1]), 1)
-
-                        values = unionUse(expression)
-                        kb.rowXmlMode = False
-                        kb.dumpColumns = None
 
                     if values is None:
                         values = inject.getValue(query, blind=False, time=False)

plugins/generic/entries.py

@@ -159,7 +159,7 @@ class Entries(object):
                     continue
 
                 kb.dumpColumns = [unsafeSQLIdentificatorNaming(_) for _ in colList]
-                colNames = colString = ", ".join(column for column in colList)
+                colNames = colString = ','.join(column for column in colList)
                 rootQuery = queries[Backend.getIdentifiedDbms()].dump_table
 
                 infoMsg = "fetching entries"

sqlmap.conf

@@ -51,6 +51,9 @@ cookie =
 # Character used for splitting cookie values (e.g. ;).
 cookieDel = 
 
+# Live cookies file used for loading up-to-date values.
+liveCookies = 
+
 # File containing cookies in Netscape/wget format.
 loadCookies = 
 

tamper/0eunion.py

@@ -29,4 +29,4 @@ def tamper(payload, **kwargs):
     '1e0UNION ALL SELECT'
     """
 
-    return re.sub("(\d+)\s+(UNION )", r"\g<1>e0\g<2>", payload, re.I) if payload else payload
+    return re.sub(r"(\d+)\s+(UNION )", r"\g<1>e0\g<2>", payload, re.I) if payload else payload

tamper/dunion.py

@@ -31,4 +31,4 @@ def tamper(payload, **kwargs):
     '1DUNION ALL SELECT'
     """
 
-    return re.sub("(\d+)\s+(UNION )", r"\g<1>D\g<2>", payload, re.I) if payload else payload
+    return re.sub(r"(\d+)\s+(UNION )", r"\g<1>D\g<2>", payload, re.I) if payload else payload

tamper/misunion.py

@@ -33,4 +33,4 @@ def tamper(payload, **kwargs):
     '1"-.1UNION ALL SELECT'
     """
 
-    return re.sub("\s+(UNION )", r"-.1\g<1>", payload, re.I) if payload else payload
+    return re.sub(r"\s+(UNION )", r"-.1\g<1>", payload, re.I) if payload else payload

tamper/schemasplit.py

@@ -28,4 +28,4 @@ def tamper(payload, **kwargs):
     'SELECT id FROM testdb 9.e.users'
     """
 
-    return re.sub("( FROM \w+)\.(\w+)", r"\g<1> 9.e.\g<2>", payload, re.I) if payload else payload
+    return re.sub(r"( FROM \w+)\.(\w+)", r"\g<1> 9.e.\g<2>", payload, re.I) if payload else payload

thirdparty/identywaf/data.json

@@ -335,6 +335,12 @@
                 "c669:RVZXum60OEhCWKpAYKYPkoJyWOpohM4IiUYMrmRWg1qQJLX2uhdOnthsOj+hX7AB16FcPhJPdLsXomtKaK59nui7c4RmkwI2FZjxtDtAeq+c3qA5chW1XaTC"
             ]
         },
+        "gtmc": {
+            "company": "GTMC",
+            "name": "GTMC WAF",
+            "regex": "GTMC WAF1 Protection:|Please consult with administrator or [email protected]",
+            "signatures": []
+        },
         "imunify360": {
             "company": "CloudLinux",
             "name": "Imunify360",

thirdparty/identywaf/identYwaf.py

@@ -60,7 +60,7 @@ else:
     HTTPCookieProcessor = urllib2.HTTPCookieProcessor
 
 NAME = "identYwaf"
-VERSION = "1.0.127"
+VERSION = "1.0.129"
 BANNER = r"""
                                    ` __ __ `
  ____  ___      ___  ____   ______ `|  T  T` __    __   ____  _____