Adding new option --results-file (thank you Hyundai)

README.md

@@ -1,8 +1,8 @@
-# sqlmap
+# sqlmap ![](https://i.imgur.com/fe85aVR.png)
 
 [![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/[email protected])](https://twitter.com/sqlmap)
 
-sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester and a broad range of switches lasting from database fingerprinting, over data fetching from the database, to accessing the underlying file system and executing commands on the operating system via out-of-band connections.
+sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester, and a broad range of switches including database fingerprinting, over data fetching from the database, accessing the underlying file system, and executing commands on the operating system via out-of-band connections.
 
 **The sqlmap project is sponsored by [Netsparker Web Application Security Scanner](https://www.netsparker.com/scan-website-security-issues/?utm_source=sqlmap.org&utm_medium=banner&utm_campaign=github).**
 
@@ -11,7 +11,7 @@ Screenshots
 
 ![Screenshot](https://raw.github.com/wiki/sqlmapproject/sqlmap/images/sqlmap_screenshot.png)
 
-You can visit the [collection of screenshots](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) demonstrating some of features on the wiki.
+You can visit the [collection of screenshots](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) demonstrating some of the features on the wiki.
 
 Installation
 ----
@@ -36,7 +36,7 @@ To get a list of all options and switches use:
     python sqlmap.py -hh
 
 You can find a sample run [here](https://asciinema.org/a/46601).
-To get an overview of sqlmap capabilities, list of supported features and description of all options and switches, along with examples, you are advised to consult the [user's manual](https://github.com/sqlmapproject/sqlmap/wiki/Usage).
+To get an overview of sqlmap capabilities, a list of supported features, and a description of all options and switches, along with examples, you are advised to consult the [user's manual](https://github.com/sqlmapproject/sqlmap/wiki/Usage).
 
 Links
 ----

data/txt/common-files.txt

@@ -203,6 +203,15 @@
 /var/www/nginx-default/index.php
 /srv/www/index.php
 
+# Reference: https://github.com/sqlmapproject/sqlmap/issues/3928
+
+/srv/www/htdocs/index.php
+/usr/local/apache2/htdocs/index.php
+/usr/local/www/data/index.php
+/var/apache2/htdocs/index.php
+/var/www/htdocs/index.php
+/var/www/html/index.php
+
 # Reference: https://www.gracefulsecurity.com/path-traversal-cheat-sheet-linux
 
 /etc/passwd

data/xml/errors.xml

@@ -7,13 +7,14 @@
         <error regexp="Warning.*?\Wmysqli?_"/>
         <error regexp="MySQLSyntaxErrorException"/>
         <error regexp="valid MySQL result"/>
-        <error regexp="check the manual that corresponds to your (MySQL|MariaDB) server version"/>
+        <error regexp="check the manual that (corresponds to|fits) your (MySQL|MariaDB) server version"/>
         <error regexp="Unknown column '[^ ]+' in 'field list'"/>
         <error regexp="MySqlClient\."/>
         <error regexp="com\.mysql\.jdbc"/>
         <error regexp="Zend_Db_(Adapter|Statement)_Mysqli_Exception"/>
         <error regexp="Pdo[./_\\]Mysql"/>
         <error regexp="MySqlException"/>
+        <error regexp="SQLSTATE\[\d+\]: Syntax error or access violation"/>
     </dbms>
 
     <!-- PostgreSQL -->

data/xml/payloads/stacked_queries.xml

@@ -3,7 +3,7 @@
 <root>
     <!-- Stacked queries tests -->
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (comment)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (comment)</title>
         <stype>4</stype>
         <level>2</level>
         <risk>1</risk>
@@ -19,12 +19,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries</title>
         <stype>4</stype>
         <level>3</level>
         <risk>1</risk>
@@ -39,12 +39,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
      <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (query SLEEP - comment)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (query SLEEP - comment)</title>
         <stype>4</stype>
         <level>3</level>
         <risk>1</risk>
@@ -60,12 +60,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (query SLEEP)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (query SLEEP)</title>
         <stype>4</stype>
         <level>4</level>
         <risk>1</risk>
@@ -80,7 +80,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 

data/xml/payloads/time_blind.xml

@@ -169,7 +169,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 AND time-based blind (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 AND time-based blind (heavy query)</title>
         <stype>5</stype>
         <level>2</level>
         <risk>2</risk>
@@ -184,12 +184,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 OR time-based blind (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 OR time-based blind (heavy query)</title>
         <stype>5</stype>
         <level>2</level>
         <risk>3</risk>
@@ -204,12 +204,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 AND time-based blind (heavy query - comment)</title>
+        <title>MySQL &lt; 5.0.12 AND time-based blind (heavy query - comment)</title>
         <stype>5</stype>
         <level>5</level>
         <risk>2</risk>
@@ -225,12 +225,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 OR time-based blind (heavy query - comment)</title>
+        <title>MySQL &lt; 5.0.12 OR time-based blind (heavy query - comment)</title>
         <stype>5</stype>
         <level>5</level>
         <risk>3</risk>
@@ -246,7 +246,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -1506,7 +1506,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 time-based blind - Parameter replace (heavy queries)</title>
+        <title>MySQL &lt; 5.0.12 time-based blind - Parameter replace (heavy queries)</title>
         <stype>5</stype>
         <level>4</level>
         <risk>2</risk>
@@ -1521,7 +1521,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -1861,7 +1861,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 time-based blind - ORDER BY, GROUP BY clause (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 time-based blind - ORDER BY, GROUP BY clause (heavy query)</title>
         <stype>5</stype>
         <level>4</level>
         <risk>2</risk>
@@ -1876,7 +1876,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 

extra/icmpsh/icmpsh_m.py

@@ -76,60 +76,63 @@ def main(src, dst):
     decoder = ImpactDecoder.IPDecoder()
 
     while True:
-        cmd = ''
+        try:
+            cmd = ''
 
-        # Wait for incoming replies
-        if sock in select.select([sock], [], [])[0]:
-            buff = sock.recv(4096)
+            # Wait for incoming replies
+            if sock in select.select([sock], [], [])[0]:
+                buff = sock.recv(4096)
 
-            if 0 == len(buff):
-                # Socket remotely closed
-                sock.close()
-                sys.exit(0)
+                if 0 == len(buff):
+                    # Socket remotely closed
+                    sock.close()
+                    sys.exit(0)
 
-            # Packet received; decode and display it
-            ippacket = decoder.decode(buff)
-            icmppacket = ippacket.child()
+                # Packet received; decode and display it
+                ippacket = decoder.decode(buff)
+                icmppacket = ippacket.child()
 
-            # If the packet matches, report it to the user
-            if ippacket.get_ip_dst() == src and ippacket.get_ip_src() == dst and 8 == icmppacket.get_icmp_type():
-                # Get identifier and sequence number
-                ident = icmppacket.get_icmp_id()
-                seq_id = icmppacket.get_icmp_seq()
-                data = icmppacket.get_data_as_string()
+                # If the packet matches, report it to the user
+                if ippacket.get_ip_dst() == src and ippacket.get_ip_src() == dst and 8 == icmppacket.get_icmp_type():
+                    # Get identifier and sequence number
+                    ident = icmppacket.get_icmp_id()
+                    seq_id = icmppacket.get_icmp_seq()
+                    data = icmppacket.get_data_as_string()
 
-                if len(data) > 0:
-                    sys.stdout.write(data)
+                    if len(data) > 0:
+                        sys.stdout.write(data)
 
-                # Parse command from standard input
-                try:
-                    cmd = sys.stdin.readline()
-                except:
-                    pass
+                    # Parse command from standard input
+                    try:
+                        cmd = sys.stdin.readline()
+                    except:
+                        pass
 
-                if cmd == 'exit\n':
-                    return
+                    if cmd == 'exit\n':
+                        return
 
-                # Set sequence number and identifier
-                icmp.set_icmp_id(ident)
-                icmp.set_icmp_seq(seq_id)
+                    # Set sequence number and identifier
+                    icmp.set_icmp_id(ident)
+                    icmp.set_icmp_seq(seq_id)
 
-                # Include the command as data inside the ICMP packet
-                icmp.contains(ImpactPacket.Data(cmd))
+                    # Include the command as data inside the ICMP packet
+                    icmp.contains(ImpactPacket.Data(cmd))
 
-                # Calculate its checksum
-                icmp.set_icmp_cksum(0)
-                icmp.auto_checksum = 1
+                    # Calculate its checksum
+                    icmp.set_icmp_cksum(0)
+                    icmp.auto_checksum = 1
 
-                # Have the IP packet contain the ICMP packet (along with its payload)
-                ip.contains(icmp)
+                    # Have the IP packet contain the ICMP packet (along with its payload)
+                    ip.contains(icmp)
 
-                try:
-                    # Send it to the target host
-                    sock.sendto(ip.get_packet(), (dst, 0))
-                except socket.error as ex:
-                    sys.stderr.write("'%s'\n" % ex)
-                    sys.stderr.flush()
+                    try:
+                        # Send it to the target host
+                        sock.sendto(ip.get_packet(), (dst, 0))
+                    except socket.error as ex:
+                        sys.stderr.write("'%s'\n" % ex)
+                        sys.stderr.flush()
+        except:
+            break
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:

extra/vulnserver/vulnserver.py

@@ -9,6 +9,7 @@ See the file 'LICENSE' for copying permission
 
 from __future__ import print_function
 
+import json
 import re
 import sqlite3
 import sys
@@ -99,10 +100,13 @@ class ReqHandler(BaseHTTPRequestHandler):
                 return
 
         if hasattr(self, "data"):
-            params.update(parse_qs(self.data))
+            if self.data.startswith('{') and self.data.endswith('}'):
+                params.update(json.loads(self.data))
+            else:
+                params.update(parse_qs(self.data))
 
         for key in params:
-            if params[key]:
+            if params[key] and isinstance(params[key], (tuple, list)):
                 params[key] = params[key][-1]
 
         self.url, self.params = path, params

lib/controller/checks.py

@@ -73,6 +73,7 @@ from lib.core.settings import BOUNDED_INJECTION_MARKER
 from lib.core.settings import CANDIDATE_SENTENCE_MIN_LENGTH
 from lib.core.settings import CHECK_INTERNET_ADDRESS
 from lib.core.settings import CHECK_INTERNET_VALUE
+from lib.core.settings import DEFAULT_COOKIE_DELIMITER
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DUMMY_NON_SQLI_CHECK_APPENDIX
 from lib.core.settings import FI_ERROR_REGEX
@@ -1559,6 +1560,16 @@ def checkConnection(suppressOutput=False):
         kb.originalPage = kb.pageTemplate = threadData.lastPage
         kb.originalCode = threadData.lastCode
 
+    if conf.cj and not conf.cookie and not conf.dropSetCookie:
+        candidate = DEFAULT_COOKIE_DELIMITER.join("%s=%s" % (_.name, _.value) for _ in conf.cj)
+
+        message = "you have not declared cookie(s), while "
+        message += "server wants to set its own ('%s'). " % re.sub(r"(=[^=;]{10}[^=;])[^=;]+([^=;]{10})", r"\g<1>...\g<2>", candidate)
+        message += "Do you want to use those [Y/n] "
+        if readInput(message, default='Y', boolean=True):
+            kb.mergeCookies = True
+            conf.httpHeaders.append((HTTP_HEADER.COOKIE, candidate))
+
     return True
 
 def checkInternet():

lib/controller/controller.py

@@ -31,6 +31,7 @@ from lib.core.common import getSafeExString
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import intersect
+from lib.core.common import isDigit
 from lib.core.common import isListLike
 from lib.core.common import parseTargetUrl
 from lib.core.common import popValue
@@ -129,7 +130,7 @@ def _selectInjection():
         message += "[q] Quit"
         choice = readInput(message, default='0').upper()
 
-        if choice.isdigit() and int(choice) < len(kb.injections) and int(choice) >= 0:
+        if isDigit(choice) and int(choice) < len(kb.injections) and int(choice) >= 0:
             index = int(choice)
         elif choice == 'Q':
             raise SqlmapUserQuitException
@@ -262,7 +263,7 @@ def _saveToResultsFile():
 
         conf.resultsFP.flush()
     except IOError as ex:
-        errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFilename, getSafeExString(ex))
+        errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFile, getSafeExString(ex))
         raise SqlmapSystemException(errMsg)
 
 @stackedmethod
@@ -374,7 +375,7 @@ def start():
                     message += "\nCookie: %s" % conf.cookie
 
                 if conf.data is not None:
-                    message += "\n%s data: %s" % ((conf.method if conf.method != HTTPMETHOD.GET else conf.method) or HTTPMETHOD.POST, urlencode(conf.data) if conf.data else "")
+                    message += "\n%s data: %s" % ((conf.method if conf.method != HTTPMETHOD.GET else conf.method) or HTTPMETHOD.POST, urlencode(conf.data or "") if re.search(r"\A\s*[<{]", conf.data or "") is None else conf.data)
 
                 if conf.forms and conf.method:
                     if conf.method == HTTPMETHOD.GET and targetUrl.find("?") == -1:
@@ -389,7 +390,7 @@ def start():
                         break
                     else:
                         if conf.method != HTTPMETHOD.GET:
-                            message = "Edit %s data [default: %s]%s: " % (conf.method, urlencode(conf.data) if conf.data else "None", " (Warning: blank fields detected)" if conf.data and extractRegexResult(EMPTY_FORM_FIELDS_REGEX, conf.data) else "")
+                            message = "Edit %s data [default: %s]%s: " % (conf.method, urlencode(conf.data or "") if re.search(r"\A\s*[<{]", conf.data or "None") is None else conf.data, " (Warning: blank fields detected)" if conf.data and extractRegexResult(EMPTY_FORM_FIELDS_REGEX, conf.data) else "")
                             conf.data = readInput(message, default=conf.data)
                             conf.data = _randomFillBlankFields(conf.data)
                             conf.data = urldecode(conf.data) if conf.data and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in conf.data else conf.data
@@ -484,9 +485,6 @@ def start():
                     if skip:
                         continue
 
-                    if kb.testOnlyCustom and place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
-                        continue
-
                     if place not in conf.paramDict:
                         continue
 
@@ -740,9 +738,9 @@ def start():
         logger.info("fetched data logged to text files under '%s'" % conf.outputPath)
 
     if conf.multipleTargets:
-        if conf.resultsFilename:
+        if conf.resultsFile:
             infoMsg = "you can find results of scanning in multiple targets "
-            infoMsg += "mode inside the CSV file '%s'" % conf.resultsFilename
+            infoMsg += "mode inside the CSV file '%s'" % conf.resultsFile
             logger.info(infoMsg)
 
     return True

lib/core/agent.py

@@ -176,12 +176,16 @@ class Agent(object):
 
         if place in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
             _ = "%s%s" % (origValue, kb.customInjectionMark)
+
             if kb.postHint == POST_HINT.JSON and not isNumber(newValue) and '"%s"' % _ not in paramString:
-                newValue = '"%s"' % newValue
+                newValue = '"%s"' % self.addPayloadDelimiters(newValue)
             elif kb.postHint == POST_HINT.JSON_LIKE and not isNumber(newValue) and "'%s'" % _ not in paramString:
-                newValue = "'%s'" % newValue
+                newValue = "'%s'" % self.addPayloadDelimiters(newValue)
+            else:
+                newValue = self.addPayloadDelimiters(newValue)
+
             newValue = newValue.replace(kb.customInjectionMark, REPLACEMENT_MARKER)
-            retVal = paramString.replace(_, self.addPayloadDelimiters(newValue))
+            retVal = paramString.replace(_, newValue)
             retVal = retVal.replace(kb.customInjectionMark, "").replace(REPLACEMENT_MARKER, kb.customInjectionMark)
         elif BOUNDED_INJECTION_MARKER in paramDict[parameter]:
             retVal = paramString.replace("%s%s" % (origValue, BOUNDED_INJECTION_MARKER), self.addPayloadDelimiters(newValue))

lib/core/common.py

@@ -1245,6 +1245,22 @@ def isZipFile(filename):
 
     return openFile(filename, "rb", encoding=None).read(len(ZIP_HEADER)) == ZIP_HEADER
 
+def isDigit(value):
+    """
+    Checks if provided (string) value consists of digits (Note: Python's isdigit() is problematic)
+
+    >>> u'\xb2'.isdigit()
+    True
+    >>> isDigit(u'\xb2')
+    False
+    >>> isDigit('123456')
+    True
+    >>> isDigit('3b3')
+    False
+    """
+
+    return re.search(r"\A[0-9]+\Z", value or "") is not None
+
 def checkFile(filename, raiseOnError=True):
     """
     Checks for file existence and readability
@@ -3178,7 +3194,10 @@ def isDBMSVersionAtLeast(minimum):
                 parts[1] = filterStringValue(parts[1], '[0-9]')
                 version = '.'.join(parts)
 
-            version = float(filterStringValue(version, '[0-9.]')) + correction
+            try:
+                version = float(filterStringValue(version, '[0-9.]')) + correction
+            except ValueError:
+                return None
 
             if isinstance(minimum, six.string_types):
                 if '.' in minimum:
@@ -4417,54 +4436,73 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
     except:
         pass
 
-    if forms:
-        for form in forms:
-            try:
-                for control in form.controls:
-                    if hasattr(control, "items") and not any((control.disabled, control.readonly)):
-                        # if control has selectable items select first non-disabled
-                        for item in control.items:
-                            if not item.disabled:
-                                if not item.selected:
-                                    item.selected = True
-                                break
+    for form in forms or []:
+        try:
+            for control in form.controls:
+                if hasattr(control, "items") and not any((control.disabled, control.readonly)):
+                    # if control has selectable items select first non-disabled
+                    for item in control.items:
+                        if not item.disabled:
+                            if not item.selected:
+                                item.selected = True
+                            break
 
-                if conf.crawlExclude and re.search(conf.crawlExclude, form.action or ""):
-                    dbgMsg = "skipping '%s'" % form.action
-                    logger.debug(dbgMsg)
-                    continue
+            if conf.crawlExclude and re.search(conf.crawlExclude, form.action or ""):
+                dbgMsg = "skipping '%s'" % form.action
+                logger.debug(dbgMsg)
+                continue
 
-                request = form.click()
-            except (ValueError, TypeError) as ex:
-                errMsg = "there has been a problem while "
-                errMsg += "processing page forms ('%s')" % getSafeExString(ex)
-                if raise_:
-                    raise SqlmapGenericException(errMsg)
-                else:
-                    logger.debug(errMsg)
+            request = form.click()
+        except (ValueError, TypeError) as ex:
+            errMsg = "there has been a problem while "
+            errMsg += "processing page forms ('%s')" % getSafeExString(ex)
+            if raise_:
+                raise SqlmapGenericException(errMsg)
             else:
-                url = urldecode(request.get_full_url(), kb.pageEncoding)
-                method = request.get_method()
-                data = request.data
-                data = urldecode(data, kb.pageEncoding, spaceplus=False)
+                logger.debug(errMsg)
+        else:
+            url = urldecode(request.get_full_url(), kb.pageEncoding)
+            method = request.get_method()
+            data = request.data
+            data = urldecode(data, kb.pageEncoding, spaceplus=False)
 
-                if not data and method and method.upper() == HTTPMETHOD.POST:
-                    debugMsg = "invalid POST form with blank data detected"
-                    logger.debug(debugMsg)
+            if not data and method and method.upper() == HTTPMETHOD.POST:
+                debugMsg = "invalid POST form with blank data detected"
+                logger.debug(debugMsg)
+                continue
+
+            # flag to know if we are dealing with the same target host
+            _ = checkSameHost(response.geturl(), url)
+
+            if conf.scope:
+                if not re.search(conf.scope, url, re.I):
                     continue
+            elif not _:
+                continue
+            else:
+                target = (url, method, data, conf.cookie, None)
+                retVal.add(target)
 
-                # flag to know if we are dealing with the same target host
-                _ = checkSameHost(response.geturl(), url)
+    for match in re.finditer(r"\.post\(['\"]([^'\"]*)['\"],\s*\{([^}]*)\}", content):
+        url = _urllib.parse.urljoin(url, htmlUnescape(match.group(1)))
+        data = ""
 
-                if conf.scope:
-                    if not re.search(conf.scope, url, re.I):
-                        continue
-                elif not _:
-                    continue
-                else:
-                    target = (url, method, data, conf.cookie, None)
-                    retVal.add(target)
-    else:
+        for name, value in re.findall(r"['\"]?(\w+)['\"]?\s*:\s*(['\"][^'\"]+)?", match.group(2)):
+            data += "%s=%s%s" % (name, value, DEFAULT_GET_POST_DELIMITER)
+
+        data = data.rstrip(DEFAULT_GET_POST_DELIMITER)
+        retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None))
+
+    for match in re.finditer(r"(?s)(\w+)\.open\(['\"]POST['\"],\s*['\"]([^'\"]+)['\"]\).*?\1\.send\(([^)]+)\)", content):
+        url = _urllib.parse.urljoin(url, htmlUnescape(match.group(2)))
+        data = match.group(3)
+
+        data = re.sub(r"\s*\+\s*[^\s'\"]+|[^\s'\"]+\s*\+\s*", "", data)
+
+        data = data.strip("['\"]")
+        retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None))
+
+    if not retVal:
         errMsg = "there were no forms found at the given target URL"
         if raise_:
             raise SqlmapGenericException(errMsg)
@@ -4665,18 +4703,19 @@ def decodeDbmsHexValue(value, raw=False):
             else:
                 retVal = decodeHex(value)
 
-            if not kb.binaryField and not raw:
-                if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
-                    try:
-                        retVal = retVal.decode("utf-16-le")
-                    except UnicodeDecodeError:
-                        pass
+            if not raw:
+                if not kb.binaryField:
+                    if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
+                        try:
+                            retVal = retVal.decode("utf-16-le")
+                        except UnicodeDecodeError:
+                            pass
 
-                elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.H2):
-                    try:
-                        retVal = retVal.decode("utf-16-be")
-                    except UnicodeDecodeError:
-                        pass
+                    elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.H2):
+                        try:
+                            retVal = retVal.decode("utf-16-be")
+                        except UnicodeDecodeError:
+                            pass
 
                 if not isinstance(retVal, six.text_type):
                     retVal = getUnicode(retVal, conf.encoding or UNICODE_ENCODING)
@@ -4976,7 +5015,7 @@ def parseRequestFile(reqFile, checkParams=True):
                     port, request = match.groups()
                     try:
                         request = decodeBase64(request, binary=False)
-                    except binascii.Error:
+                    except (binascii.Error, TypeError):
                         continue
                     _ = re.search(r"%s:.+" % re.escape(HTTP_HEADER.HOST), request)
                     if _:

lib/core/convert.py

@@ -263,7 +263,7 @@ def getOrds(value):
 
 def getUnicode(value, encoding=None, noneToNull=False):
     """
-    Return the unicode representation of the supplied value:
+    Returns the unicode representation of the supplied value
 
     >>> getUnicode('test') == u'test'
     True
@@ -375,3 +375,20 @@ def stdoutEncode(value):
         retVal = value
 
     return retVal
+
+def getConsoleLength(value):
+    """
+    Returns console width of unicode values
+
+    >>> getConsoleLength("abc")
+    3
+    >>> getConsoleLength(u"\\u957f\\u6c5f")
+    4
+    """
+
+    if isinstance(value, six.text_type):
+        retVal = sum((2 if ord(_) >= 0x3000 else 1) for _ in value)
+    else:
+        retVal = len(value)
+
+    return retVal

lib/core/dump.py

@@ -28,6 +28,7 @@ from lib.core.common import safeCSValue
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
+from lib.core.convert import getConsoleLength
 from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.data import conf
@@ -197,7 +198,7 @@ class Dump(object):
             self._areAdmins = userSettings[1]
             userSettings = userSettings[0]
 
-        users = list(userSettings.keys())
+        users = [_ for _ in userSettings.keys() if _ is not None]
         users.sort(key=lambda _: _.lower() if hasattr(_, "lower") else _)
 
         if conf.api:
@@ -245,7 +246,7 @@ class Dump(object):
                     if table and isListLike(table):
                         table = table[0]
 
-                    maxlength = max(maxlength, len(unsafeSQLIdentificatorNaming(normalizeUnicode(table) or getUnicode(table))))
+                    maxlength = max(maxlength, getConsoleLength(unsafeSQLIdentificatorNaming(getUnicode(table))))
 
             lines = "-" * (int(maxlength) + 2)
 
@@ -266,7 +267,7 @@ class Dump(object):
                         table = table[0]
 
                     table = unsafeSQLIdentificatorNaming(table)
-                    blank = " " * (maxlength - len(normalizeUnicode(table) or getUnicode(table)))
+                    blank = " " * (maxlength - getConsoleLength(getUnicode(table)))
                     self._write("| %s%s |" % (table, blank))
 
                 self._write("+%s+\n" % lines)
@@ -361,7 +362,7 @@ class Dump(object):
             for ctables in dbTables.values():
                 for tables in ctables.values():
                     for table in tables:
-                        maxlength1 = max(maxlength1, len(normalizeUnicode(table) or getUnicode(table)))
+                        maxlength1 = max(maxlength1, getConsoleLength(getUnicode(table)))
 
             for db, counts in dbTables.items():
                 self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db else "Current database")
@@ -387,7 +388,7 @@ class Dump(object):
                     tables.sort(key=lambda _: _.lower() if hasattr(_, "lower") else _)
 
                     for table in tables:
-                        blank1 = " " * (maxlength1 - len(normalizeUnicode(table) or getUnicode(table)))
+                        blank1 = " " * (maxlength1 - getConsoleLength(getUnicode(table)))
                         blank2 = " " * (maxlength2 - len(str(count)))
                         self._write("| %s%s | %d%s |" % (table, blank1, count, blank2))
 
@@ -547,7 +548,7 @@ class Dump(object):
 
                 column = unsafeSQLIdentificatorNaming(column)
                 maxlength = int(info["length"])
-                blank = " " * (maxlength - len(column))
+                blank = " " * (maxlength - getConsoleLength(column))
 
                 self._write("| %s%s" % (column, blank), newline=False)
 
@@ -602,7 +603,7 @@ class Dump(object):
 
                     values.append(value)
                     maxlength = int(info["length"])
-                    blank = " " * (maxlength - len(value))
+                    blank = " " * (maxlength - getConsoleLength(value))
                     self._write("| %s%s" % (value, blank), newline=False, console=console)
 
                     if len(value) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value:

lib/core/option.py

@@ -109,6 +109,7 @@ from lib.core.settings import DEFAULT_TOR_HTTP_PORTS
 from lib.core.settings import DEFAULT_TOR_SOCKS_PORTS
 from lib.core.settings import DEFAULT_USER_AGENT
 from lib.core.settings import DUMMY_URL
+from lib.core.settings import IGNORE_CODE_WILDCARD
 from lib.core.settings import IS_WIN
 from lib.core.settings import KB_CHARS_BOUNDARY_CHAR
 from lib.core.settings import KB_CHARS_LOW_FREQUENCY_ALPHABET
@@ -314,7 +315,7 @@ def _setRequestFromFile():
 
             if url is None:
                 errMsg = "specified file '%s' " % requestFile
-                errMsg += "does not contain a valid HTTP request"
+                errMsg += "does not contain a usable HTTP request (with parameters)"
                 raise SqlmapDataException(errMsg)
 
     if conf.secondReq:
@@ -346,6 +347,10 @@ def _setCrawler():
         for i in xrange(len(targets)):
             try:
                 target = targets[i]
+
+                if not re.search(r"(?i)\Ahttp[s]*://", target):
+                    target = "http://%s" % target
+
                 crawl(target)
 
                 if conf.verbose in (1, 2):
@@ -467,7 +472,7 @@ def _findPageForms():
     logger.info(infoMsg)
 
     if not any((conf.bulkFile, conf.googleDork, conf.sitemapUrl)):
-        page, _, _ = Request.queryPage(content=True)
+        page, _, _ = Request.queryPage(content=True, ignoreSecondOrder=True)
         if findPageForms(page, conf.url, True, True):
             found = True
     else:
@@ -483,7 +488,11 @@ def _findPageForms():
 
         for i in xrange(len(targets)):
             try:
-                target = targets[i]
+                target = targets[i].strip()
+
+                if not re.search(r"(?i)\Ahttp[s]*://", target):
+                    target = "http://%s" % target
+
                 page, _, _ = Request.getPage(url=target.strip(), cookie=conf.cookie, crawling=True, raise404=False)
                 if findPageForms(page, target, False, True):
                     found = True
@@ -1158,11 +1167,11 @@ def _setSafeVisit():
             errMsg = "invalid format of a safe request file"
             raise SqlmapSyntaxException(errMsg)
     else:
-        if not re.search(r"\Ahttp[s]*://", conf.safeUrl):
+        if not re.search(r"(?i)\Ahttp[s]*://", conf.safeUrl):
             if ":443/" in conf.safeUrl:
-                conf.safeUrl = "https://" + conf.safeUrl
+                conf.safeUrl = "https://%s" % conf.safeUrl
             else:
-                conf.safeUrl = "http://" + conf.safeUrl
+                conf.safeUrl = "http://%s" % conf.safeUrl
 
     if (conf.safeFreq or 0) <= 0:
         errMsg = "please provide a valid value (>0) for safe frequency (--safe-freq) while using safe visit features"
@@ -1227,7 +1236,7 @@ def _setHTTPAuthentication():
 
     elif not conf.authType and conf.authCred:
         errMsg = "you specified the HTTP authentication credentials, "
-        errMsg += "but did not provide the type"
+        errMsg += "but did not provide the type (e.g. --auth-type=\"basic\")"
         raise SqlmapSyntaxException(errMsg)
 
     elif (conf.authType or "").lower() not in (AUTH_TYPE.BASIC, AUTH_TYPE.DIGEST, AUTH_TYPE.NTLM, AUTH_TYPE.PKI):
@@ -1569,11 +1578,14 @@ def _cleanupOptions():
         conf.testParameter = []
 
     if conf.ignoreCode:
-        try:
-            conf.ignoreCode = [int(_) for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.ignoreCode)]
-        except ValueError:
-            errMsg = "options '--ignore-code' should contain a list of integer values"
-            raise SqlmapSyntaxException(errMsg)
+        if conf.ignoreCode == IGNORE_CODE_WILDCARD:
+            conf.ignoreCode = xrange(0, 1000)
+        else:
+            try:
+                conf.ignoreCode = [int(_) for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.ignoreCode)]
+            except ValueError:
+                errMsg = "options '--ignore-code' should contain a list of integer values or a wildcard value '%s'" % IGNORE_CODE_WILDCARD
+                raise SqlmapSyntaxException(errMsg)
     else:
         conf.ignoreCode = []
 
@@ -1821,7 +1833,6 @@ def _setConfAttributes():
     conf.path = None
     conf.port = None
     conf.proxyList = None
-    conf.resultsFilename = None
     conf.resultsFP = None
     conf.scheme = None
     conf.tests = []
@@ -1941,6 +1952,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.mergeCookies = None
     kb.multipleCtrlC = False
     kb.negativeLogic = False
+    kb.normalizeCrawlingChoice = None
     kb.nullConnection = None
     kb.oldMsf = None
     kb.orderByColumns = None
@@ -2008,6 +2020,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.threadException = False
     kb.tableExistsChoice = None
     kb.uChar = NULL
+    kb.udfFail = False
     kb.unionDuplicates = False
     kb.wizardMode = False
     kb.xpCmdshellAvailable = False
@@ -2206,6 +2219,13 @@ def _mergeOptions(inputOptions, overrideOptions):
         if hasattr(conf, key) and conf[key] is None:
             conf[key] = value
 
+            if conf.unstable:
+                if key in ("timeSec", "retries", "timeout"):
+                    conf[key] *= 2
+
+    if conf.unstable:
+        conf.forcePartial = True
+
     lut = {}
     for group in optDict.keys():
         lut.update((_.upper(), _) for _ in optDict[group])

lib/core/optiondict.py

@@ -238,9 +238,11 @@ optDict = {
         "listTampers": "boolean",
         "offline": "boolean",
         "purge": "boolean",
+        "resultsFile": "string",
         "tmpDir": "string",
-        "wizard": "boolean",
+        "unstable": "boolean",
         "updateAll": "boolean",
+        "wizard": "boolean",
         "verbose": "integer",
     },
 

lib/core/patch.py

@@ -20,6 +20,7 @@ import thirdparty.chardet.universaldetector
 
 from lib.core.common import filterNone
 from lib.core.common import getSafeExString
+from lib.core.common import isDigit
 from lib.core.common import isListLike
 from lib.core.common import readInput
 from lib.core.common import shellExec
@@ -62,6 +63,7 @@ def resolveCrossReferences():
     Place for cross-reference resolution
     """
 
+    lib.core.threads.isDigit = isDigit
     lib.core.threads.readInput = readInput
     lib.core.common.getPageTemplate = getPageTemplate
     lib.core.convert.filterNone = filterNone

lib/core/revision.py

@@ -40,9 +40,14 @@ def getRevisionNumber():
             with openFile(filePath, "r") as f:
                 content = getText(f.read())
                 filePath = None
+
                 if content.startswith("ref: "):
-                    filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
-                else:
+                    try:
+                        filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
+                    except UnicodeError:
+                        pass
+
+                if filePath is None:
                     match = re.match(r"(?i)[0-9a-f]{32}", content)
                     retVal = match.group(0) if match else None
                     break

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.10.0"
+VERSION = "1.3.11.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -413,6 +413,9 @@ CANDIDATE_SENTENCE_MIN_LENGTH = 10
 # Character used for marking injectable position inside provided data
 CUSTOM_INJECTION_MARK_CHAR = '*'
 
+# Wildcard value that can be used in option --ignore-code
+IGNORE_CODE_WILDCARD = '*'
+
 # Other way to declare injection position
 INJECT_HERE_REGEX = r"(?i)%INJECT[_ ]?HERE%"
 
@@ -721,7 +724,7 @@ MAX_HELP_OPTION_LENGTH = 18
 MAX_CONNECT_RETRIES = 100
 
 # Strings for detecting formatting errors
-FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException")
+FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException", "Arguments are of the wrong type")
 
 # Regular expression used for extracting ASP.NET view state values
 VIEWSTATE_REGEX = r'(?i)(?P<name>__VIEWSTATE[^"]*)[^>]+value="(?P<result>[^"]+)'

lib/core/target.py

@@ -561,16 +561,18 @@ def _setResultsFile():
         return
 
     if not conf.resultsFP:
-        conf.resultsFilename = os.path.join(paths.SQLMAP_OUTPUT_PATH, time.strftime(RESULTS_FILE_FORMAT).lower())
+        conf.resultsFile = conf.resultsFile or os.path.join(paths.SQLMAP_OUTPUT_PATH, time.strftime(RESULTS_FILE_FORMAT).lower())
+        found = os.path.exists(conf.resultsFile)
+
         try:
-            conf.resultsFP = openFile(conf.resultsFilename, "a", UNICODE_ENCODING, buffering=0)
+            conf.resultsFP = openFile(conf.resultsFile, "a", UNICODE_ENCODING, buffering=0)
         except (OSError, IOError) as ex:
             try:
-                warnMsg = "unable to create results file '%s' ('%s'). " % (conf.resultsFilename, getUnicode(ex))
-                handle, conf.resultsFilename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.RESULTS, suffix=".csv")
+                warnMsg = "unable to create results file '%s' ('%s'). " % (conf.resultsFile, getUnicode(ex))
+                handle, conf.resultsFile = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.RESULTS, suffix=".csv")
                 os.close(handle)
-                conf.resultsFP = openFile(conf.resultsFilename, "w+", UNICODE_ENCODING, buffering=0)
-                warnMsg += "Using temporary file '%s' instead" % conf.resultsFilename
+                conf.resultsFP = openFile(conf.resultsFile, "w+", UNICODE_ENCODING, buffering=0)
+                warnMsg += "Using temporary file '%s' instead" % conf.resultsFile
                 logger.warn(warnMsg)
             except IOError as _:
                 errMsg = "unable to write to the temporary directory ('%s'). " % _
@@ -579,9 +581,10 @@ def _setResultsFile():
                 errMsg += "create temporary files and/or directories"
                 raise SqlmapSystemException(errMsg)
 
-        conf.resultsFP.writelines("Target URL,Place,Parameter,Technique(s),Note(s)%s" % os.linesep)
+        if not found:
+            conf.resultsFP.writelines("Target URL,Place,Parameter,Technique(s),Note(s)%s" % os.linesep)
 
-        logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFilename)
+        logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFile)
 
 def _createFilesDir():
     """

lib/core/testing.py

@@ -75,6 +75,7 @@ def vulnTest():
 
     for options, checks in (
         ("--flush-session", ("CloudFlare",)),
+        ("--flush-session --data='{\"id\": 1}' --banner", ("Payload: {\"id\"", "banner: '3")),
         ("--flush-session --parse-errors --eval=\"id2=2\" --referer=\"localhost\" --cookie=\"PHPSESSID=d41d8cd98f00b204e9800998ecf8427e\"", (": syntax error", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "back-end DBMS: SQLite", "3 columns")),
         ("--banner --schema --dump -T users --binary-fields=surname --where \"id>3\"", ("banner: '3", "INTEGER", "TEXT", "id", "name", "surname", "2 entries", "6E616D6569736E756C6C")),
         ("--all --tamper=between,randomcase", ("5 entries", "luther", "blisset", "fluffy", "179ad45c6ce2cb97cf1029e212046e81", "NULL", "nameisnull", "testpass")),

lib/core/threads.py

@@ -73,6 +73,10 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
     # It will be overwritten by original from lib.core.common
     pass
 
+def isDigit(value):
+    # It will be overwritten by original from lib.core.common
+    pass
+
 def getCurrentThreadData():
     """
     Returns current thread's local data
@@ -125,10 +129,12 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
             choice = readInput(message, default=str(numThreads))
             if choice:
                 skipThreadCheck = False
+
                 if choice.endswith('!'):
                     choice = choice[:-1]
                     skipThreadCheck = True
-                if choice.isdigit():
+
+                if isDigit(choice):
                     if int(choice) > MAX_NUMBER_OF_THREADS and not skipThreadCheck:
                         errMsg = "maximum number of used threads is %d avoiding potential connection issues" % MAX_NUMBER_OF_THREADS
                         logger.critical(errMsg)

lib/parse/cmdline.py

@@ -728,12 +728,18 @@ def cmdLineParser(argv=None):
         miscellaneous.add_argument("--purge", dest="purge", action="store_true",
             help="Safely remove all content from sqlmap data directory")
 
+        miscellaneous.add_argument("--results-file", dest="resultsFile",
+            help="Location of CSV results file in multiple targets mode")
+
         miscellaneous.add_argument("--sqlmap-shell", dest="sqlmapShell", action="store_true",
             help="Prompt for an interactive sqlmap shell")
 
         miscellaneous.add_argument("--tmp-dir", dest="tmpDir",
             help="Local directory for storing temporary files")
 
+        miscellaneous.add_argument("--unstable", dest="unstable", action="store_true",
+            help="Adjust options for unstable connections")
+
         miscellaneous.add_argument("--update", dest="updateAll", action="store_true",
             help="Update sqlmap")
 
@@ -905,7 +911,7 @@ def cmdLineParser(argv=None):
             longOptions = set(re.findall(r"\-\-([^= ]+?)=", parser.format_help()))
             if argv[i] == "-hh":
                 argv[i] = "-h"
-            elif i == 1 and argv[i].startswith("http"):
+            elif i == 1 and re.search(r"\A(http|www\.|\w[\w.-]+\.\w{2,})", argv[i]) is not None:
                 argv[i] = "--url=%s" % argv[i]
             elif len(argv[i]) > 1 and all(ord(_) in xrange(0x2018, 0x2020) for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0], argv[i][-1])):
                 dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is, well, illegal (%s)\n" % argv[i])

lib/request/basic.py

@@ -34,6 +34,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.decorators import cachedmethod
+from lib.core.decorators import lockedmethod
 from lib.core.dicts import HTML_ENTITIES
 from lib.core.enums import DBMS
 from lib.core.enums import HTTP_HEADER
@@ -57,6 +58,7 @@ from thirdparty.odict import OrderedDict
 from thirdparty.six import unichr as _unichr
 from thirdparty.six.moves import http_client as _http_client
 
+@lockedmethod
 def forgeHeaders(items=None, base=None):
     """
     Prepare HTTP Cookie, HTTP User-Agent and HTTP Referer headers to use when performing
@@ -110,9 +112,9 @@ def forgeHeaders(items=None, base=None):
                     if conf.loadCookies:
                         conf.httpHeaders = filterNone((item if item[0] != HTTP_HEADER.COOKIE else None) for item in conf.httpHeaders)
                     elif kb.mergeCookies is None:
-                        message = "you provided a HTTP %s header value. " % HTTP_HEADER.COOKIE
-                        message += "The target URL provided its own cookies within "
-                        message += "the HTTP %s header which intersect with yours. " % HTTP_HEADER.SET_COOKIE
+                        message = "you provided a HTTP %s header value, while " % HTTP_HEADER.COOKIE
+                        message += "target URL provides its own cookies within "
+                        message += "HTTP %s header which intersect with yours. " % HTTP_HEADER.SET_COOKIE
                         message += "Do you want to merge them in further requests? [Y/n] "
 
                         kb.mergeCookies = readInput(message, default='Y', boolean=True)

lib/request/basicauthhandler.py

@@ -12,6 +12,7 @@ class SmartHTTPBasicAuthHandler(_urllib.request.HTTPBasicAuthHandler):
     Reference: http://selenic.com/hg/rev/6c51a5056020
     Fix for a: http://bugs.python.org/issue8797
     """
+
     def __init__(self, *args, **kwargs):
         _urllib.request.HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
         self.retried_req = set()

lib/request/connect.py

@@ -839,7 +839,7 @@ class Connect(object):
 
     @staticmethod
     @stackedmethod
-    def queryPage(value=None, place=None, content=False, getRatioValue=False, silent=False, method=None, timeBasedCompare=False, noteResponseTime=True, auxHeaders=None, response=False, raise404=None, removeReflection=True, disableTampering=False):
+    def queryPage(value=None, place=None, content=False, getRatioValue=False, silent=False, method=None, timeBasedCompare=False, noteResponseTime=True, auxHeaders=None, response=False, raise404=None, removeReflection=True, disableTampering=False, ignoreSecondOrder=False):
         """
         This method calls a function to get the target URL page content
         and returns its page ratio (0 <= ratio <= 1) or a boolean value
@@ -1339,17 +1339,18 @@ class Connect(object):
                     warnMsg += "behavior in custom WAF/IPS solutions"
                 singleTimeWarnMessage(warnMsg)
 
-        if conf.secondUrl:
-            page, headers, code = Connect.getPage(url=conf.secondUrl, cookie=cookie, ua=ua, silent=silent, auxHeaders=auxHeaders, response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
-        elif kb.secondReq and IPS_WAF_CHECK_PAYLOAD not in _urllib.parse.unquote(value or ""):
-            def _(value):
-                if kb.customInjectionMark in (value or ""):
-                    if payload is None:
-                        value = value.replace(kb.customInjectionMark, "")
-                    else:
-                        value = re.sub(r"\w*%s" % re.escape(kb.customInjectionMark), payload, value)
-                return value
-            page, headers, code = Connect.getPage(url=_(kb.secondReq[0]), post=_(kb.secondReq[2]), method=kb.secondReq[1], cookie=kb.secondReq[3], silent=silent, auxHeaders=dict(auxHeaders, **dict(kb.secondReq[4])), response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
+        if not ignoreSecondOrder:
+            if conf.secondUrl:
+                page, headers, code = Connect.getPage(url=conf.secondUrl, cookie=cookie, ua=ua, silent=silent, auxHeaders=auxHeaders, response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
+            elif kb.secondReq and IPS_WAF_CHECK_PAYLOAD not in _urllib.parse.unquote(value or ""):
+                def _(value):
+                    if kb.customInjectionMark in (value or ""):
+                        if payload is None:
+                            value = value.replace(kb.customInjectionMark, "")
+                        else:
+                            value = re.sub(r"\w*%s" % re.escape(kb.customInjectionMark), payload, value)
+                    return value
+                page, headers, code = Connect.getPage(url=_(kb.secondReq[0]), post=_(kb.secondReq[2]), method=kb.secondReq[1], cookie=kb.secondReq[3], silent=silent, auxHeaders=dict(auxHeaders, **dict(kb.secondReq[4])), response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
 
         threadData.lastQueryDuration = calculateDeltaSeconds(start)
 

lib/request/inject.py

@@ -24,6 +24,7 @@ from lib.core.common import getTechniqueData
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import initTechnique
+from lib.core.common import isDigit
 from lib.core.common import isNoneValue
 from lib.core.common import isNumPosStrValue
 from lib.core.common import isTechniqueAvailable
@@ -235,7 +236,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
                             elif choice == 'Q':
                                 raise SqlmapUserQuitException
 
-                            elif choice.isdigit() and int(choice) > 0 and int(choice) <= count:
+                            elif isDigit(choice) and int(choice) > 0 and int(choice) <= count:
                                 stopLimit = int(choice)
 
                                 infoMsg = "sqlmap is now going to retrieve the "
@@ -246,7 +247,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
                                 message = "how many? "
                                 stopLimit = readInput(message, default="10")
 
-                                if not stopLimit.isdigit():
+                                if not isDigit(stopLimit):
                                     errMsg = "invalid choice"
                                     logger.error(errMsg)
 
@@ -261,7 +262,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
 
                                 return None
 
-                    elif count and not count.isdigit():
+                    elif count and not isDigit(count):
                         warnMsg = "it was not possible to count the number "
                         warnMsg += "of entries for the SQL query provided. "
                         warnMsg += "sqlmap will assume that it returns only "

lib/takeover/abstraction.py

@@ -16,6 +16,7 @@ from lib.core.common import isStackingAvailable
 from lib.core.common import readInput
 from lib.core.convert import getUnicode
 from lib.core.data import conf
+from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.enums import AUTOCOMPLETE_TYPE
 from lib.core.enums import DBMS
@@ -48,7 +49,7 @@ class Abstraction(Web, UDF, XP_cmdshell):
         if Backend.isDbms(DBMS.PGSQL) and self.checkCopyExec():
             self.copyExecCmd(cmd)
 
-        elif self.webBackdoorUrl and not isStackingAvailable():
+        elif self.webBackdoorUrl and (not isStackingAvailable() or kb.udfFail):
             self.webBackdoorRunCmd(cmd)
 
         elif Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL):
@@ -67,7 +68,7 @@ class Abstraction(Web, UDF, XP_cmdshell):
         if Backend.isDbms(DBMS.PGSQL) and self.checkCopyExec():
             retVal = self.copyExecCmd(cmd)
 
-        elif self.webBackdoorUrl and not isStackingAvailable():
+        elif self.webBackdoorUrl and (not isStackingAvailable() or kb.udfFail):
             retVal = self.webBackdoorRunCmd(cmd)
 
         elif Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL):
@@ -104,7 +105,7 @@ class Abstraction(Web, UDF, XP_cmdshell):
             self.execCmd(cmd)
 
     def shell(self):
-        if self.webBackdoorUrl and not isStackingAvailable():
+        if self.webBackdoorUrl and (not isStackingAvailable() or kb.udfFail):
             infoMsg = "calling OS shell. To quit type "
             infoMsg += "'x' or 'q' and press ENTER"
             logger.info(infoMsg)

lib/takeover/metasploit.py

@@ -23,6 +23,7 @@ from lib.core.common import dataToStdout
 from lib.core.common import Backend
 from lib.core.common import getLocalIP
 from lib.core.common import getRemoteIP
+from lib.core.common import isDigit
 from lib.core.common import normalizePath
 from lib.core.common import ntToPosixSlashes
 from lib.core.common import pollProcess
@@ -154,7 +155,7 @@ class Metasploit(object):
 
         choice = readInput(message, default="%d" % default)
 
-        if not choice or not choice.isdigit() or int(choice) > maxValue or int(choice) < 1:
+        if not choice or not isDigit(choice) or int(choice) > maxValue or int(choice) < 1:
             choice = default
 
         choice = int(choice)
@@ -241,7 +242,7 @@ class Metasploit(object):
                         elif Backend.isDbms(DBMS.MSSQL) and Backend.isVersionWithin(("2005", "2008")):
                             break
 
-                    elif not choice.isdigit():
+                    elif not isDigit(choice):
                         logger.warn("invalid value, only digits are allowed")
 
                     elif int(choice) < 1 or int(choice) > 2:
@@ -692,9 +693,9 @@ class Metasploit(object):
         self._runMsfCliSmbrelay()
 
         if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL):
-            self.uncPath = "\\\\\\\\%s\\\\%s" % (self.lhostStr, self._randFile)
+            self.uncPath = r"\\\\%s\\%s" % (self.lhostStr, self._randFile)
         else:
-            self.uncPath = "\\\\%s\\%s" % (self.lhostStr, self._randFile)
+            self.uncPath = r"\\%s\%s" % (self.lhostStr, self._randFile)
 
         debugMsg = "Metasploit Framework console exited with return "
         debugMsg += "code %s" % self._controlMsfCmd(self._msfCliProc, self.uncPathRequest)

lib/takeover/udf.py

@@ -11,6 +11,7 @@ from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import dataToStdout
+from lib.core.common import isDigit
 from lib.core.common import isStackingAvailable
 from lib.core.common import readInput
 from lib.core.common import unArrayizeValue
@@ -339,11 +340,9 @@ class UDF(object):
 
                 if choice == 'Q':
                     break
-                elif hasattr(choice, "isdigit") and choice.isdigit() and int(choice) > 0 and int(choice) <= len(udfList):
+                elif isDigit(choice) and int(choice) > 0 and int(choice) <= len(udfList):
                     choice = int(choice)
                     break
-                elif isinstance(choice, int) and choice > 0 and choice <= len(udfList):
-                    break
                 else:
                     warnMsg = "invalid value, only digits >= 1 and "
                     warnMsg += "<= %d are allowed" % len(udfList)

lib/takeover/web.py

@@ -22,6 +22,7 @@ from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSQLSnippet
 from lib.core.common import getTechnique
 from lib.core.common import getTechniqueData
+from lib.core.common import isDigit
 from lib.core.common import isTechniqueAvailable
 from lib.core.common import isWindowsDriveLetterPath
 from lib.core.common import normalizePath
@@ -200,7 +201,7 @@ class Web(object):
         while True:
             choice = readInput(message, default=str(default))
 
-            if not choice.isdigit():
+            if not isDigit(choice):
                 logger.warn("invalid value, only digits are allowed")
 
             elif int(choice) < 1 or int(choice) > len(choices):

lib/techniques/union/use.py

@@ -123,7 +123,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
 
                             try:
                                 decodeBase64(value)
-                            except binascii.Error:
+                            except (binascii.Error, TypeError):
                                 base64 = False
                                 break
 

lib/utils/brute.py

@@ -163,7 +163,7 @@ def tableExists(tableFile, regex=None):
         if not threadData.shared.files:
             warnMsg = "no table(s) found"
             if conf.db:
-                 warnMsg += "for database '%s'" % conf.db
+                 warnMsg += " for database '%s'" % conf.db
             logger.warn(warnMsg)
         else:
             for item in threadData.shared.files:

lib/utils/crawler.py

@@ -15,6 +15,7 @@ import time
 from lib.core.common import checkSameHost
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
+from lib.core.common import extractRegexResult
 from lib.core.common import findPageForms
 from lib.core.common import getSafeExString
 from lib.core.common import openFile
@@ -92,7 +93,7 @@ def crawl(target):
                         soup = BeautifulSoup(content)
                         tags = soup('a')
 
-                        tags += re.finditer(r'(?i)<a[^>]+href=["\'](?P<href>[^>"\']+)', content)
+                        tags += re.finditer(r'(?i)\s(href|src)=["\'](?P<href>[^>"\']+)', content)
 
                         for tag in tags:
                             href = tag.get("href") if hasattr(tag, "get") else tag.group("href")
@@ -111,7 +112,7 @@ def crawl(target):
                                 elif not _:
                                     continue
 
-                                if url.split('.')[-1].lower() not in CRAWL_EXCLUDE_EXTENSIONS:
+                                if (extractRegexResult(r"\A[^?]+\.(?P<result>\w+)(\?|\Z)", url) or "").lower() not in CRAWL_EXCLUDE_EXTENSIONS:
                                     with kb.locks.value:
                                         threadData.shared.deeper.add(url)
                                         if re.search(r"(.*?)\?(.+)", url):
@@ -194,6 +195,26 @@ def crawl(target):
             for url in threadData.shared.value:
                 kb.targets.add((urldecode(url, kb.pageEncoding), None, None, None, None))
 
+        if kb.normalizeCrawlingChoice is None:
+            message = "do you want to normalize "
+            message += "crawling results [Y/n] "
+
+            kb.normalizeCrawlingChoice = readInput(message, default='Y', boolean=True)
+
+        if kb.normalizeCrawlingChoice:
+            seen = set()
+            results = OrderedSet()
+
+            for target in kb.targets:
+                match = re.search(r"/[^/?]*\?.*\Z", target[0])
+                if match:
+                    key = re.sub(r"=[^=&]*", "=", match.group(0))
+                    if key not in seen:
+                        results.add(target)
+                        seen.add(key)
+
+            kb.targets = results
+
         storeResultsToFile(kb.targets)
 
 def storeResultsToFile(results):

lib/utils/hash.py

@@ -1000,7 +1000,7 @@ def dictionaryAttack(attack_dict):
                                 resumes.append((user, hash_, resumed))
                             keys.add(hash_)
 
-                    except (binascii.Error, IndexError):
+                    except (binascii.Error, TypeError, IndexError):
                         pass
 
         if not attack_info:

plugins/generic/entries.py

@@ -25,6 +25,7 @@ from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unArrayizeValue
 from lib.core.common import unsafeSQLIdentificatorNaming
+from lib.core.convert import getConsoleLength
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -269,7 +270,7 @@ class Entries(object):
                                 else:
                                     colEntry = unArrayizeValue(entry[index]) if index < len(entry) else u''
 
-                                maxLen = max(len(column), len(DUMP_REPLACEMENTS.get(getUnicode(colEntry), getUnicode(colEntry))))
+                                maxLen = max(getConsoleLength(column), getConsoleLength(DUMP_REPLACEMENTS.get(getUnicode(colEntry), getUnicode(colEntry))))
 
                                 if maxLen > kb.data.dumpedTable[column]["length"]:
                                     kb.data.dumpedTable[column]["length"] = maxLen
@@ -506,7 +507,7 @@ class Entries(object):
                         logger.info(infoMsg)
 
     def dumpFoundColumn(self, dbs, foundCols, colConsider):
-        message = "do you want to dump entries? [Y/n] "
+        message = "do you want to dump found column(s) entries? [Y/n] "
 
         if not readInput(message, default='Y', boolean=True):
             return
@@ -573,7 +574,7 @@ class Entries(object):
                     conf.dumper.dbTableValues(data)
 
     def dumpFoundTables(self, tables):
-        message = "do you want to dump tables' entries? [Y/n] "
+        message = "do you want to dump found table(s) entries? [Y/n] "
 
         if not readInput(message, default='Y', boolean=True):
             return

plugins/generic/misc.py

@@ -137,7 +137,7 @@ class Miscellaneous(object):
             self.delRemoteFile(self.webStagerFilePath)
             self.delRemoteFile(self.webBackdoorFilePath)
 
-        if not isStackingAvailable() and not conf.direct:
+        if (not isStackingAvailable() or kb.udfFail) and not conf.direct:
             return
 
         if any((conf.osCmd, conf.osShell)) and Backend.isDbms(DBMS.PGSQL) and kb.copyExecTest:

plugins/generic/takeover.py

@@ -9,11 +9,13 @@ import os
 
 from lib.core.common import Backend
 from lib.core.common import getSafeExString
+from lib.core.common import isDigit
 from lib.core.common import isStackingAvailable
 from lib.core.common import openFile
 from lib.core.common import readInput
 from lib.core.common import runningAsAdmin
 from lib.core.data import conf
+from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.enums import DBMS
 from lib.core.enums import OS
@@ -78,7 +80,20 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
             raise SqlmapNotVulnerableException(errMsg)
 
         self.getRemoteTempPath()
-        self.initEnv(web=web)
+
+        try:
+            self.initEnv(web=web)
+        except SqlmapFilePathException:
+            if not web:
+                infoMsg = "falling back to web backdoor method..."
+                logger.info(infoMsg)
+
+                web = True
+                kb.udfFail = True
+
+                self.initEnv(web=web)
+            else:
+                raise
 
         if not web or (web and self.webBackdoorUrl is not None):
             self.shell()
@@ -101,7 +116,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
             while True:
                 tunnel = readInput(msg, default='1')
 
-                if tunnel.isdigit() and int(tunnel) in (1, 2):
+                if isDigit(tunnel) and int(tunnel) in (1, 2):
                     tunnel = int(tunnel)
                     break
 
@@ -172,7 +187,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
                     while True:
                         choice = readInput(msg, default='1')
 
-                        if choice.isdigit() and int(choice) in (1, 2):
+                        if isDigit(choice) and int(choice) in (1, 2):
                             choice = int(choice)
                             break
 

plugins/generic/users.py

@@ -600,7 +600,8 @@ class Users(object):
 
                     # In Firebird we get one letter for each privilege
                     elif Backend.isDbms(DBMS.FIREBIRD):
-                        privileges.add(FIREBIRD_PRIVS[privilege.strip()])
+                        if privilege.strip() in FIREBIRD_PRIVS:
+                            privileges.add(FIREBIRD_PRIVS[privilege.strip()])
 
                     # In Informix we get one letter for the highest privilege
                     elif Backend.isDbms(DBMS.INFORMIX):

sqlmap.conf

@@ -822,17 +822,24 @@ listTampers = False
 # Valid: True or False
 offline = False
 
+# Location of CSV results file in multiple targets mode.
+resultsFile =
+
 # Local directory for storing temporary files.
 tmpDir =
 
-# Simple wizard interface for beginner users.
+# Adjust options for unstable connections.
 # Valid: True or False
-wizard = False
+unstable = False
 
 # Update sqlmap.
 # Valid: True or False
 updateAll = False
 
+# Simple wizard interface for beginner users.
+# Valid: True or False
+wizard = False
+
 # Verbosity level.
 # Valid: integer between 0 and 6
 # 0: Show only error and critical messages

thirdparty/clientform/clientform.py

@@ -237,7 +237,7 @@ def unescape(data, entities, encoding=DEFAULT_ENCODING):
 
         repl = entities.get(ent)
         if repl is not None:
-            if type(repl) != type(""):
+            if type(repl) != type("") and encoding is not None:
                 try:
                     repl = repl.encode(encoding)
                 except UnicodeError:

thirdparty/identywaf/data.json

@@ -161,10 +161,11 @@
         "barracuda": {
             "company": "Barracuda Networks",
             "name": "Barracuda",
-            "regex": "\\bbarracuda_|barra_counter_session=|when this page occurred and the event ID found at the bottom of the page",
+            "regex": "\\bbarracuda_|barra_counter_session=|when this page occurred and the event ID found at the bottom of the page|<!--(0123456789){15}",
             "signatures": [
                 "2676:RVdXum61OElCWapAYKYPk4JzWOtohM4JiUcMr2RWg1qQJbX3uhdOn9htOj+hXrAB16FcPxJPdLsXo2tKaK99n+i6c4VmkwI3FZjxtDtAeq6c36A4chS1XaTC",
-                "db27:RVdXum61OElCWapAYKYPk4JzWOtohM4JiUcMr2RWg1qQJbX3uhdOn9htOj+hXrAB16FcPxJPdLsXo2tKaK99n+i6c4RmkgI2FZjxtDtAeq6c3qA4chS1XaTC"
+                "db27:RVdXum61OElCWapAYKYPk4JzWOtohM4JiUcMr2RWg1qQJbX3uhdOn9htOj+hXrAB16FcPxJPdLsXo2tKaK99n+i6c4RmkgI2FZjxtDtAeq6c3qA4chS1XaTC",
+                "7e6b:RVdXum61OElCWapBYKYPk4JzWOtohM4JiUcMr2RXg1uQJbX3uhdOn9htOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A4chS1XaTC"
             ]
         },
         "bekchy": {
@@ -356,6 +357,16 @@
             "regex": "The (ISA Server|server) denied the specified Uniform Resource Locator \\(URL\\)",
             "signatures": []
         },
+        "ithemes": {
+            "company": "iThemes",
+            "name": "iThemes Security",
+            "regex": "",
+            "signatures": [
+                "c70f:RVZXum60OEhCWKpAYKYPkoJyWOpohM4IiUYMr2RWg1uQJLX3uhZOnthtOj+hXrAA16FcPhJOdLoXomtKaK59nui7c4RmkgI2FZjxtDtAeq+c36A4chW1XaTC",
+                "71ee:RVZXum60OEhCWKpAYKYPkoJyWOpohM4IiUYMr2RWg1qQJLX2uhZOnthtOj+hXrAA16FcPhJOdLoXomtKaK59nui7c4RmkgI2FZjxtDtAeq+c36A4chW1XaTC"
+            ],
+            "note": "Formerly Better WP Security"
+        },
         "janusec": {
             "company": "Janusec",
             "name": "Janusec Application Gateway",

thirdparty/identywaf/identYwaf.py

@@ -27,7 +27,9 @@ import sys
 import time
 import zlib
 
-if sys.version_info >= (3, 0):
+PY3 = sys.version_info >= (3, 0)
+
+if PY3:
     import http.cookiejar
     import http.client as httplib
     import urllib.request
@@ -58,7 +60,7 @@ else:
     HTTPCookieProcessor = urllib2.HTTPCookieProcessor
 
 NAME = "identYwaf"
-VERSION = "1.0.114"
+VERSION = "1.0.122"
 BANNER = r"""
                                    ` __ __ `
  ____  ___      ___  ____   ______ `|  T  T` __    __   ____  _____ 
@@ -356,7 +358,7 @@ def init():
     os.chdir(os.path.abspath(os.path.dirname(__file__)))
 
     # Reference: http://blog.mathieu-leplatre.info/python-utf-8-print-fails-when-redirecting-stdout.html
-    if IS_TTY:
+    if not PY3 and not IS_TTY:
         sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
 
     print(colorize("[o] initializing handlers..."))