Adding new option --results-file (thank you Hyundai)

README.md

@@ -1,8 +1,8 @@
-# sqlmap
+# sqlmap ![](https://i.imgur.com/fe85aVR.png)
 
 [![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/[email protected])](https://twitter.com/sqlmap)
 
-sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester and a broad range of switches lasting from database fingerprinting, over data fetching from the database, to accessing the underlying file system and executing commands on the operating system via out-of-band connections.
+sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester, and a broad range of switches including database fingerprinting, over data fetching from the database, accessing the underlying file system, and executing commands on the operating system via out-of-band connections.
 
 **The sqlmap project is sponsored by [Netsparker Web Application Security Scanner](https://www.netsparker.com/scan-website-security-issues/?utm_source=sqlmap.org&utm_medium=banner&utm_campaign=github).**
 
@@ -11,7 +11,7 @@ Screenshots
 
 ![Screenshot](https://raw.github.com/wiki/sqlmapproject/sqlmap/images/sqlmap_screenshot.png)
 
-You can visit the [collection of screenshots](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) demonstrating some of features on the wiki.
+You can visit the [collection of screenshots](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) demonstrating some of the features on the wiki.
 
 Installation
 ----
@@ -36,7 +36,7 @@ To get a list of all options and switches use:
     python sqlmap.py -hh
 
 You can find a sample run [here](https://asciinema.org/a/46601).
-To get an overview of sqlmap capabilities, list of supported features and description of all options and switches, along with examples, you are advised to consult the [user's manual](https://github.com/sqlmapproject/sqlmap/wiki/Usage).
+To get an overview of sqlmap capabilities, a list of supported features, and a description of all options and switches, along with examples, you are advised to consult the [user's manual](https://github.com/sqlmapproject/sqlmap/wiki/Usage).
 
 Links
 ----

data/txt/common-files.txt

@@ -192,6 +192,26 @@
 /var/log/mysqld.log
 /var/www/index.php
 
+# Reference: https://github.com/sqlmapproject/sqlmap/blob/master/lib/core/settings.py#L809-L810
+
+/var/www/index.php
+/usr/local/apache/index.php
+/usr/local/apache2/index.php
+/usr/local/www/apache22/index.php
+/usr/local/www/apache24/index.php
+/usr/local/httpd/index.php
+/var/www/nginx-default/index.php
+/srv/www/index.php
+
+# Reference: https://github.com/sqlmapproject/sqlmap/issues/3928
+
+/srv/www/htdocs/index.php
+/usr/local/apache2/htdocs/index.php
+/usr/local/www/data/index.php
+/var/apache2/htdocs/index.php
+/var/www/htdocs/index.php
+/var/www/html/index.php
+
 # Reference: https://www.gracefulsecurity.com/path-traversal-cheat-sheet-linux
 
 /etc/passwd

data/xml/banner/mysql.xml

@@ -1,5 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
+<!--
+     References:
+     * https://en.wikipedia.org/wiki/Debian_version_history
+-->
+
 <root>
     <regexp value="^([\d\.\-]+)[\-\_\ ].*">
         <info dbms_version="1"/>
@@ -36,19 +41,27 @@
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+wheezy">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="7.0" codename="wheezy"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="7" codename="wheezy"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+jessie">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="8.0" codename="jessie"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="8" codename="jessie"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+stretch">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="9.0" codename="stretch"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="9" codename="stretch"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+buster">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="10.0" codename="buster"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="10" codename="buster"/>
+    </regexp>
+
+    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+bullseye">
+        <info dbms_version="1" type="Linux" distrib="Debian" release="11" codename="bullseye"/>
+    </regexp>
+
+    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+bookworm">
+        <info dbms_version="1" type="Linux" distrib="Debian" release="12" codename="bookworm"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+(sid|unstable)">

data/xml/errors.xml

@@ -7,13 +7,14 @@
         <error regexp="Warning.*?\Wmysqli?_"/>
         <error regexp="MySQLSyntaxErrorException"/>
         <error regexp="valid MySQL result"/>
-        <error regexp="check the manual that corresponds to your (MySQL|MariaDB) server version"/>
+        <error regexp="check the manual that (corresponds to|fits) your (MySQL|MariaDB) server version"/>
         <error regexp="Unknown column '[^ ]+' in 'field list'"/>
         <error regexp="MySqlClient\."/>
         <error regexp="com\.mysql\.jdbc"/>
         <error regexp="Zend_Db_(Adapter|Statement)_Mysqli_Exception"/>
         <error regexp="Pdo[./_\\]Mysql"/>
         <error regexp="MySqlException"/>
+        <error regexp="SQLSTATE\[\d+\]: Syntax error or access violation"/>
     </dbms>
 
     <!-- PostgreSQL -->

data/xml/payloads/stacked_queries.xml

@@ -3,7 +3,7 @@
 <root>
     <!-- Stacked queries tests -->
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (comment)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (comment)</title>
         <stype>4</stype>
         <level>2</level>
         <risk>1</risk>
@@ -19,12 +19,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries</title>
         <stype>4</stype>
         <level>3</level>
         <risk>1</risk>
@@ -39,12 +39,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
      <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (query SLEEP - comment)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (query SLEEP - comment)</title>
         <stype>4</stype>
         <level>3</level>
         <risk>1</risk>
@@ -60,12 +60,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (query SLEEP)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (query SLEEP)</title>
         <stype>4</stype>
         <level>4</level>
         <risk>1</risk>
@@ -80,7 +80,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 

data/xml/payloads/time_blind.xml

@@ -169,7 +169,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 AND time-based blind (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 AND time-based blind (heavy query)</title>
         <stype>5</stype>
         <level>2</level>
         <risk>2</risk>
@@ -184,12 +184,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 OR time-based blind (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 OR time-based blind (heavy query)</title>
         <stype>5</stype>
         <level>2</level>
         <risk>3</risk>
@@ -204,12 +204,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 AND time-based blind (heavy query - comment)</title>
+        <title>MySQL &lt; 5.0.12 AND time-based blind (heavy query - comment)</title>
         <stype>5</stype>
         <level>5</level>
         <risk>2</risk>
@@ -225,12 +225,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 OR time-based blind (heavy query - comment)</title>
+        <title>MySQL &lt; 5.0.12 OR time-based blind (heavy query - comment)</title>
         <stype>5</stype>
         <level>5</level>
         <risk>3</risk>
@@ -246,7 +246,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -1506,7 +1506,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 time-based blind - Parameter replace (heavy queries)</title>
+        <title>MySQL &lt; 5.0.12 time-based blind - Parameter replace (heavy queries)</title>
         <stype>5</stype>
         <level>4</level>
         <risk>2</risk>
@@ -1521,7 +1521,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -1861,7 +1861,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 time-based blind - ORDER BY, GROUP BY clause (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 time-based blind - ORDER BY, GROUP BY clause (heavy query)</title>
         <stype>5</stype>
         <level>4</level>
         <risk>2</risk>
@@ -1876,7 +1876,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 

extra/beep/beep.py

@@ -15,11 +15,13 @@ BEEP_WAV_FILENAME = os.path.join(os.path.dirname(__file__), "beep.wav")
 
 def beep():
     try:
-        if sys.platform == "nt":
+        if sys.platform.startswith("win"):
             _win_wav_play(BEEP_WAV_FILENAME)
-        elif sys.platform == "darwin":
+        elif sys.platform.startswith("darwin"):
             _mac_beep()
-        elif sys.platform.startswith("linux"):
+        elif sys.platform.startswith("cygwin"):
+            _cygwin_beep(BEEP_WAV_FILENAME)
+        elif any(sys.platform.startswith(_) for _ in ("linux", "freebsd")):
             _linux_wav_play(BEEP_WAV_FILENAME)
         else:
             _speaker_beep()
@@ -34,6 +36,10 @@ def _speaker_beep():
     except IOError:
         pass
 
+# Reference: https://lists.gnu.org/archive/html/emacs-devel/2014-09/msg00815.html
+def _cygwin_beep(filename):
+    os.system("play-sound-file '%s' 2>/dev/null" % filename)
+
 def _mac_beep():
     import Carbon.Snd
     Carbon.Snd.SysBeep(1)
@@ -57,7 +63,10 @@ def _linux_wav_play(filename):
     class struct_pa_sample_spec(ctypes.Structure):
         _fields_ = [("format", ctypes.c_int), ("rate", ctypes.c_uint32), ("channels", ctypes.c_uint8)]
 
-    pa = ctypes.cdll.LoadLibrary("libpulse-simple.so.0")
+    try:
+        pa = ctypes.cdll.LoadLibrary("libpulse-simple.so.0")
+    except OSError:
+        return
 
     wave_file = wave.open(filename, "rb")
 

extra/icmpsh/icmpsh_m.py

@@ -76,60 +76,63 @@ def main(src, dst):
     decoder = ImpactDecoder.IPDecoder()
 
     while True:
-        cmd = ''
+        try:
+            cmd = ''
 
-        # Wait for incoming replies
-        if sock in select.select([sock], [], [])[0]:
-            buff = sock.recv(4096)
+            # Wait for incoming replies
+            if sock in select.select([sock], [], [])[0]:
+                buff = sock.recv(4096)
 
-            if 0 == len(buff):
-                # Socket remotely closed
-                sock.close()
-                sys.exit(0)
+                if 0 == len(buff):
+                    # Socket remotely closed
+                    sock.close()
+                    sys.exit(0)
 
-            # Packet received; decode and display it
-            ippacket = decoder.decode(buff)
-            icmppacket = ippacket.child()
+                # Packet received; decode and display it
+                ippacket = decoder.decode(buff)
+                icmppacket = ippacket.child()
 
-            # If the packet matches, report it to the user
-            if ippacket.get_ip_dst() == src and ippacket.get_ip_src() == dst and 8 == icmppacket.get_icmp_type():
-                # Get identifier and sequence number
-                ident = icmppacket.get_icmp_id()
-                seq_id = icmppacket.get_icmp_seq()
-                data = icmppacket.get_data_as_string()
+                # If the packet matches, report it to the user
+                if ippacket.get_ip_dst() == src and ippacket.get_ip_src() == dst and 8 == icmppacket.get_icmp_type():
+                    # Get identifier and sequence number
+                    ident = icmppacket.get_icmp_id()
+                    seq_id = icmppacket.get_icmp_seq()
+                    data = icmppacket.get_data_as_string()
 
-                if len(data) > 0:
-                    sys.stdout.write(data)
+                    if len(data) > 0:
+                        sys.stdout.write(data)
 
-                # Parse command from standard input
-                try:
-                    cmd = sys.stdin.readline()
-                except:
-                    pass
+                    # Parse command from standard input
+                    try:
+                        cmd = sys.stdin.readline()
+                    except:
+                        pass
 
-                if cmd == 'exit\n':
-                    return
+                    if cmd == 'exit\n':
+                        return
 
-                # Set sequence number and identifier
-                icmp.set_icmp_id(ident)
-                icmp.set_icmp_seq(seq_id)
+                    # Set sequence number and identifier
+                    icmp.set_icmp_id(ident)
+                    icmp.set_icmp_seq(seq_id)
 
-                # Include the command as data inside the ICMP packet
-                icmp.contains(ImpactPacket.Data(cmd))
+                    # Include the command as data inside the ICMP packet
+                    icmp.contains(ImpactPacket.Data(cmd))
 
-                # Calculate its checksum
-                icmp.set_icmp_cksum(0)
-                icmp.auto_checksum = 1
+                    # Calculate its checksum
+                    icmp.set_icmp_cksum(0)
+                    icmp.auto_checksum = 1
 
-                # Have the IP packet contain the ICMP packet (along with its payload)
-                ip.contains(icmp)
+                    # Have the IP packet contain the ICMP packet (along with its payload)
+                    ip.contains(icmp)
 
-                try:
-                    # Send it to the target host
-                    sock.sendto(ip.get_packet(), (dst, 0))
-                except socket.error as ex:
-                    sys.stderr.write("'%s'\n" % ex)
-                    sys.stderr.flush()
+                    try:
+                        # Send it to the target host
+                        sock.sendto(ip.get_packet(), (dst, 0))
+                    except socket.error as ex:
+                        sys.stderr.write("'%s'\n" % ex)
+                        sys.stderr.flush()
+        except:
+            break
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:

extra/safe2bin/README.txt

@@ -1,17 +0,0 @@
-To use safe2bin.py you need to pass it the original file,
-and optionally the output file name.
-
-Example:
-
-$ python ./safe2bin.py -i output.txt -o output.txt.bin
-
-This will create an binary decoded file output.txt.bin. For example, 
-if the content of output.txt is: "\ttest\t\x32\x33\x34\nnewline" it will 
-be decoded to: "	test	234
-newline"
-
-If you skip the output file name, general rule is that the binary
-file names are suffixed with the string '.bin'. So, that means that 
-the upper example can also be written in the following form:
-
-$ python ./safe2bin.py -i output.txt

extra/safe2bin/__init__.py

@@ -1,8 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
-See the file 'LICENSE' for copying permission
-"""
-
-pass

extra/shutils/autocompletion.sh

@@ -0,0 +1,9 @@
+#/usr/bin/env bash
+
+# source ./extra/shutils/autocompletion.sh
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+WORDLIST=`python "$DIR/../../sqlmap.py" -hh | grep -Eo '\s\--?\w[^ =,]*' | grep -vF '..' | paste -sd "" -`
+
+complete -W "$WORDLIST" sqlmap
+complete -W "$WORDLIST" ./sqlmap.py

extra/shutils/drei.sh

@@ -7,7 +7,7 @@
 
 export SQLMAP_DREI=1
 #for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done
-for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i; done
+for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i | sed 's/Compiling/Checking/g'; done
 unset SQLMAP_DREI
 source `dirname "$0"`"/junk.sh"
 

extra/vulnserver/vulnserver.py

@@ -9,6 +9,7 @@ See the file 'LICENSE' for copying permission
 
 from __future__ import print_function
 
+import json
 import re
 import sqlite3
 import sys
@@ -99,10 +100,13 @@ class ReqHandler(BaseHTTPRequestHandler):
                 return
 
         if hasattr(self, "data"):
-            params.update(parse_qs(self.data))
+            if self.data.startswith('{') and self.data.endswith('}'):
+                params.update(json.loads(self.data))
+            else:
+                params.update(parse_qs(self.data))
 
         for key in params:
-            if params[key]:
+            if params[key] and isinstance(params[key], (tuple, list)):
                 params[key] = params[key][-1]
 
         self.url, self.params = path, params

lib/controller/checks.py

@@ -73,6 +73,7 @@ from lib.core.settings import BOUNDED_INJECTION_MARKER
 from lib.core.settings import CANDIDATE_SENTENCE_MIN_LENGTH
 from lib.core.settings import CHECK_INTERNET_ADDRESS
 from lib.core.settings import CHECK_INTERNET_VALUE
+from lib.core.settings import DEFAULT_COOKIE_DELIMITER
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DUMMY_NON_SQLI_CHECK_APPENDIX
 from lib.core.settings import FI_ERROR_REGEX
@@ -1559,6 +1560,16 @@ def checkConnection(suppressOutput=False):
         kb.originalPage = kb.pageTemplate = threadData.lastPage
         kb.originalCode = threadData.lastCode
 
+    if conf.cj and not conf.cookie and not conf.dropSetCookie:
+        candidate = DEFAULT_COOKIE_DELIMITER.join("%s=%s" % (_.name, _.value) for _ in conf.cj)
+
+        message = "you have not declared cookie(s), while "
+        message += "server wants to set its own ('%s'). " % re.sub(r"(=[^=;]{10}[^=;])[^=;]+([^=;]{10})", r"\g<1>...\g<2>", candidate)
+        message += "Do you want to use those [Y/n] "
+        if readInput(message, default='Y', boolean=True):
+            kb.mergeCookies = True
+            conf.httpHeaders.append((HTTP_HEADER.COOKIE, candidate))
+
     return True
 
 def checkInternet():

lib/controller/controller.py

@@ -31,6 +31,7 @@ from lib.core.common import getSafeExString
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import intersect
+from lib.core.common import isDigit
 from lib.core.common import isListLike
 from lib.core.common import parseTargetUrl
 from lib.core.common import popValue
@@ -129,7 +130,7 @@ def _selectInjection():
         message += "[q] Quit"
         choice = readInput(message, default='0').upper()
 
-        if choice.isdigit() and int(choice) < len(kb.injections) and int(choice) >= 0:
+        if isDigit(choice) and int(choice) < len(kb.injections) and int(choice) >= 0:
             index = int(choice)
         elif choice == 'Q':
             raise SqlmapUserQuitException
@@ -262,7 +263,7 @@ def _saveToResultsFile():
 
         conf.resultsFP.flush()
     except IOError as ex:
-        errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFilename, getSafeExString(ex))
+        errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFile, getSafeExString(ex))
         raise SqlmapSystemException(errMsg)
 
 @stackedmethod
@@ -374,7 +375,7 @@ def start():
                     message += "\nCookie: %s" % conf.cookie
 
                 if conf.data is not None:
-                    message += "\n%s data: %s" % ((conf.method if conf.method != HTTPMETHOD.GET else conf.method) or HTTPMETHOD.POST, urlencode(conf.data) if conf.data else "")
+                    message += "\n%s data: %s" % ((conf.method if conf.method != HTTPMETHOD.GET else conf.method) or HTTPMETHOD.POST, urlencode(conf.data or "") if re.search(r"\A\s*[<{]", conf.data or "") is None else conf.data)
 
                 if conf.forms and conf.method:
                     if conf.method == HTTPMETHOD.GET and targetUrl.find("?") == -1:
@@ -389,7 +390,7 @@ def start():
                         break
                     else:
                         if conf.method != HTTPMETHOD.GET:
-                            message = "Edit %s data [default: %s]%s: " % (conf.method, urlencode(conf.data) if conf.data else "None", " (Warning: blank fields detected)" if conf.data and extractRegexResult(EMPTY_FORM_FIELDS_REGEX, conf.data) else "")
+                            message = "Edit %s data [default: %s]%s: " % (conf.method, urlencode(conf.data or "") if re.search(r"\A\s*[<{]", conf.data or "None") is None else conf.data, " (Warning: blank fields detected)" if conf.data and extractRegexResult(EMPTY_FORM_FIELDS_REGEX, conf.data) else "")
                             conf.data = readInput(message, default=conf.data)
                             conf.data = _randomFillBlankFields(conf.data)
                             conf.data = urldecode(conf.data) if conf.data and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in conf.data else conf.data
@@ -484,9 +485,6 @@ def start():
                     if skip:
                         continue
 
-                    if kb.testOnlyCustom and place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
-                        continue
-
                     if place not in conf.paramDict:
                         continue
 
@@ -740,9 +738,9 @@ def start():
         logger.info("fetched data logged to text files under '%s'" % conf.outputPath)
 
     if conf.multipleTargets:
-        if conf.resultsFilename:
+        if conf.resultsFile:
             infoMsg = "you can find results of scanning in multiple targets "
-            infoMsg += "mode inside the CSV file '%s'" % conf.resultsFilename
+            infoMsg += "mode inside the CSV file '%s'" % conf.resultsFile
             logger.info(infoMsg)
 
     return True

lib/core/agent.py

@@ -5,7 +5,6 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-import base64
 import re
 
 from lib.core.common import Backend
@@ -27,6 +26,7 @@ from lib.core.common import unArrayizeValue
 from lib.core.common import urlencode
 from lib.core.common import zeroDepthSearch
 from lib.core.compat import xrange
+from lib.core.convert import encodeBase64
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -50,6 +50,7 @@ from lib.core.settings import PAYLOAD_DELIMITER
 from lib.core.settings import REPLACEMENT_MARKER
 from lib.core.settings import SINGLE_QUOTE_MARKER
 from lib.core.settings import SLEEP_TIME_MARKER
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.unescaper import unescaper
 from thirdparty import six
 
@@ -170,17 +171,21 @@ class Agent(object):
 
         if re.sub(r" \(.+", "", parameter) in conf.base64Parameter:
             # TODO: support for POST_HINT
-            newValue = base64.b64encode(newValue)
-            origValue = base64.b64encode(origValue)
+            newValue = encodeBase64(newValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
+            origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
 
         if place in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
             _ = "%s%s" % (origValue, kb.customInjectionMark)
+
             if kb.postHint == POST_HINT.JSON and not isNumber(newValue) and '"%s"' % _ not in paramString:
-                newValue = '"%s"' % newValue
+                newValue = '"%s"' % self.addPayloadDelimiters(newValue)
             elif kb.postHint == POST_HINT.JSON_LIKE and not isNumber(newValue) and "'%s'" % _ not in paramString:
-                newValue = "'%s'" % newValue
+                newValue = "'%s'" % self.addPayloadDelimiters(newValue)
+            else:
+                newValue = self.addPayloadDelimiters(newValue)
+
             newValue = newValue.replace(kb.customInjectionMark, REPLACEMENT_MARKER)
-            retVal = paramString.replace(_, self.addPayloadDelimiters(newValue))
+            retVal = paramString.replace(_, newValue)
             retVal = retVal.replace(kb.customInjectionMark, "").replace(REPLACEMENT_MARKER, kb.customInjectionMark)
         elif BOUNDED_INJECTION_MARKER in paramDict[parameter]:
             retVal = paramString.replace("%s%s" % (origValue, BOUNDED_INJECTION_MARKER), self.addPayloadDelimiters(newValue))
@@ -654,7 +659,7 @@ class Agent(object):
                 concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'||" % kb.chars.start, 1)
                 _ = unArrayizeValue(zeroDepthSearch(concatenatedQuery, " FROM "))
                 concatenatedQuery = "%s||'%s'%s" % (concatenatedQuery[:_], kb.chars.stop, concatenatedQuery[_:])
-                concatenatedQuery = re.sub(r"('%s'\|\|)(.+)(%s)" % (kb.chars.start, re.escape(castedFields)), r"\g<2>\g<1>\g<3>", concatenatedQuery)
+                concatenatedQuery = re.sub(r"('%s'\|\|)(.+?)(%s)" % (kb.chars.start, re.escape(castedFields)), r"\g<2>\g<1>\g<3>", concatenatedQuery)
             elif fieldsSelect:
                 concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'||" % kb.chars.start, 1)
                 concatenatedQuery += "||'%s'" % kb.chars.stop

lib/core/common.py

@@ -46,7 +46,6 @@ from xml.sax import SAXParseException
 
 from extra.beep.beep import beep
 from extra.cloak.cloak import decloak
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.bigarray import BigArray
 from lib.core.compat import cmp
 from lib.core.compat import round
@@ -180,6 +179,7 @@ from lib.core.settings import VERSION_STRING
 from lib.core.settings import ZIP_HEADER
 from lib.core.settings import WEBSCARAB_SPLITTER
 from lib.core.threads import getCurrentThreadData
+from lib.utils.safe2bin import safecharencode
 from lib.utils.sqlalchemy import _sqlalchemy
 from thirdparty import six
 from thirdparty.clientform.clientform import ParseResponse
@@ -623,7 +623,7 @@ def paramToDict(place, parameters=None):
                 if parameter in (conf.base64Parameter or []):
                     try:
                         oldValue = value
-                        value = decodeBase64(value, binary=False)
+                        value = decodeBase64(value, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
                         parameters = re.sub(r"\b%s(\b|\Z)" % re.escape(oldValue), value, parameters)
                     except:
                         errMsg = "parameter '%s' does not contain " % parameter
@@ -1245,6 +1245,22 @@ def isZipFile(filename):
 
     return openFile(filename, "rb", encoding=None).read(len(ZIP_HEADER)) == ZIP_HEADER
 
+def isDigit(value):
+    """
+    Checks if provided (string) value consists of digits (Note: Python's isdigit() is problematic)
+
+    >>> u'\xb2'.isdigit()
+    True
+    >>> isDigit(u'\xb2')
+    False
+    >>> isDigit('123456')
+    True
+    >>> isDigit('3b3')
+    False
+    """
+
+    return re.search(r"\A[0-9]+\Z", value or "") is not None
+
 def checkFile(filename, raiseOnError=True):
     """
     Checks for file existence and readability
@@ -3178,7 +3194,10 @@ def isDBMSVersionAtLeast(minimum):
                 parts[1] = filterStringValue(parts[1], '[0-9]')
                 version = '.'.join(parts)
 
-            version = float(filterStringValue(version, '[0-9.]')) + correction
+            try:
+                version = float(filterStringValue(version, '[0-9.]')) + correction
+            except ValueError:
+                return None
 
             if isinstance(minimum, six.string_types):
                 if '.' in minimum:
@@ -4417,54 +4436,73 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
     except:
         pass
 
-    if forms:
-        for form in forms:
-            try:
-                for control in form.controls:
-                    if hasattr(control, "items") and not any((control.disabled, control.readonly)):
-                        # if control has selectable items select first non-disabled
-                        for item in control.items:
-                            if not item.disabled:
-                                if not item.selected:
-                                    item.selected = True
-                                break
+    for form in forms or []:
+        try:
+            for control in form.controls:
+                if hasattr(control, "items") and not any((control.disabled, control.readonly)):
+                    # if control has selectable items select first non-disabled
+                    for item in control.items:
+                        if not item.disabled:
+                            if not item.selected:
+                                item.selected = True
+                            break
 
-                if conf.crawlExclude and re.search(conf.crawlExclude, form.action or ""):
-                    dbgMsg = "skipping '%s'" % form.action
-                    logger.debug(dbgMsg)
-                    continue
+            if conf.crawlExclude and re.search(conf.crawlExclude, form.action or ""):
+                dbgMsg = "skipping '%s'" % form.action
+                logger.debug(dbgMsg)
+                continue
 
-                request = form.click()
-            except (ValueError, TypeError) as ex:
-                errMsg = "there has been a problem while "
-                errMsg += "processing page forms ('%s')" % getSafeExString(ex)
-                if raise_:
-                    raise SqlmapGenericException(errMsg)
-                else:
-                    logger.debug(errMsg)
+            request = form.click()
+        except (ValueError, TypeError) as ex:
+            errMsg = "there has been a problem while "
+            errMsg += "processing page forms ('%s')" % getSafeExString(ex)
+            if raise_:
+                raise SqlmapGenericException(errMsg)
             else:
-                url = urldecode(request.get_full_url(), kb.pageEncoding)
-                method = request.get_method()
-                data = request.data
-                data = urldecode(data, kb.pageEncoding, spaceplus=False)
+                logger.debug(errMsg)
+        else:
+            url = urldecode(request.get_full_url(), kb.pageEncoding)
+            method = request.get_method()
+            data = request.data
+            data = urldecode(data, kb.pageEncoding, spaceplus=False)
 
-                if not data and method and method.upper() == HTTPMETHOD.POST:
-                    debugMsg = "invalid POST form with blank data detected"
-                    logger.debug(debugMsg)
+            if not data and method and method.upper() == HTTPMETHOD.POST:
+                debugMsg = "invalid POST form with blank data detected"
+                logger.debug(debugMsg)
+                continue
+
+            # flag to know if we are dealing with the same target host
+            _ = checkSameHost(response.geturl(), url)
+
+            if conf.scope:
+                if not re.search(conf.scope, url, re.I):
                     continue
+            elif not _:
+                continue
+            else:
+                target = (url, method, data, conf.cookie, None)
+                retVal.add(target)
 
-                # flag to know if we are dealing with the same target host
-                _ = checkSameHost(response.geturl(), url)
+    for match in re.finditer(r"\.post\(['\"]([^'\"]*)['\"],\s*\{([^}]*)\}", content):
+        url = _urllib.parse.urljoin(url, htmlUnescape(match.group(1)))
+        data = ""
 
-                if conf.scope:
-                    if not re.search(conf.scope, url, re.I):
-                        continue
-                elif not _:
-                    continue
-                else:
-                    target = (url, method, data, conf.cookie, None)
-                    retVal.add(target)
-    else:
+        for name, value in re.findall(r"['\"]?(\w+)['\"]?\s*:\s*(['\"][^'\"]+)?", match.group(2)):
+            data += "%s=%s%s" % (name, value, DEFAULT_GET_POST_DELIMITER)
+
+        data = data.rstrip(DEFAULT_GET_POST_DELIMITER)
+        retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None))
+
+    for match in re.finditer(r"(?s)(\w+)\.open\(['\"]POST['\"],\s*['\"]([^'\"]+)['\"]\).*?\1\.send\(([^)]+)\)", content):
+        url = _urllib.parse.urljoin(url, htmlUnescape(match.group(2)))
+        data = match.group(3)
+
+        data = re.sub(r"\s*\+\s*[^\s'\"]+|[^\s'\"]+\s*\+\s*", "", data)
+
+        data = data.strip("['\"]")
+        retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None))
+
+    if not retVal:
         errMsg = "there were no forms found at the given target URL"
         if raise_:
             raise SqlmapGenericException(errMsg)
@@ -4665,18 +4703,19 @@ def decodeDbmsHexValue(value, raw=False):
             else:
                 retVal = decodeHex(value)
 
-            if not kb.binaryField and not raw:
-                if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
-                    try:
-                        retVal = retVal.decode("utf-16-le")
-                    except UnicodeDecodeError:
-                        pass
+            if not raw:
+                if not kb.binaryField:
+                    if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
+                        try:
+                            retVal = retVal.decode("utf-16-le")
+                        except UnicodeDecodeError:
+                            pass
 
-                elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.H2):
-                    try:
-                        retVal = retVal.decode("utf-16-be")
-                    except UnicodeDecodeError:
-                        pass
+                    elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.H2):
+                        try:
+                            retVal = retVal.decode("utf-16-be")
+                        except UnicodeDecodeError:
+                            pass
 
                 if not isinstance(retVal, six.text_type):
                     retVal = getUnicode(retVal, conf.encoding or UNICODE_ENCODING)
@@ -4976,7 +5015,7 @@ def parseRequestFile(reqFile, checkParams=True):
                     port, request = match.groups()
                     try:
                         request = decodeBase64(request, binary=False)
-                    except binascii.Error:
+                    except (binascii.Error, TypeError):
                         continue
                     _ = re.search(r"%s:.+" % re.escape(HTTP_HEADER.HOST), request)
                     if _:

lib/core/convert.py

@@ -184,7 +184,7 @@ def encodeHex(value, binary=True):
 
     return retVal
 
-def decodeBase64(value, binary=True):
+def decodeBase64(value, binary=True, encoding=None):
     """
     Returns a decoded representation of provided Base64 value
 
@@ -197,11 +197,11 @@ def decodeBase64(value, binary=True):
     retVal = base64.b64decode(value)
 
     if not binary:
-        retVal = getText(retVal)
+        retVal = getText(retVal, encoding)
 
     return retVal
 
-def encodeBase64(value, binary=True):
+def encodeBase64(value, binary=True, encoding=None):
     """
     Returns a decoded representation of provided Base64 value
 
@@ -212,12 +212,12 @@ def encodeBase64(value, binary=True):
     """
 
     if isinstance(value, six.text_type):
-        value = value.encode(UNICODE_ENCODING)
+        value = value.encode(encoding or UNICODE_ENCODING)
 
     retVal = base64.b64encode(value)
 
     if not binary:
-        retVal = getText(retVal)
+        retVal = getText(retVal, encoding)
 
     return retVal
 
@@ -263,7 +263,7 @@ def getOrds(value):
 
 def getUnicode(value, encoding=None, noneToNull=False):
     """
-    Return the unicode representation of the supplied value:
+    Returns the unicode representation of the supplied value
 
     >>> getUnicode('test') == u'test'
     True
@@ -305,7 +305,7 @@ def getUnicode(value, encoding=None, noneToNull=False):
         except UnicodeDecodeError:
             return six.text_type(str(value), errors="ignore")  # encoding ignored for non-basestring instances
 
-def getText(value):
+def getText(value, encoding=None):
     """
     Returns textual value of a given value (Note: not necessary Unicode on Python2)
 
@@ -318,7 +318,7 @@ def getText(value):
     retVal = value
 
     if isinstance(value, six.binary_type):
-        retVal = getUnicode(value)
+        retVal = getUnicode(value, encoding)
 
     if six.PY2:
         try:
@@ -375,3 +375,20 @@ def stdoutEncode(value):
         retVal = value
 
     return retVal
+
+def getConsoleLength(value):
+    """
+    Returns console width of unicode values
+
+    >>> getConsoleLength("abc")
+    3
+    >>> getConsoleLength(u"\\u957f\\u6c5f")
+    4
+    """
+
+    if isinstance(value, six.text_type):
+        retVal = sum((2 if ord(_) >= 0x3000 else 1) for _ in value)
+    else:
+        retVal = len(value)
+
+    return retVal

lib/core/dump.py

@@ -13,7 +13,6 @@ import shutil
 import tempfile
 import threading
 
-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import dataToDumpFile
@@ -29,6 +28,7 @@ from lib.core.common import safeCSValue
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
+from lib.core.convert import getConsoleLength
 from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.data import conf
@@ -53,6 +53,7 @@ from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import UNSAFE_DUMP_FILEPATH_REPLACEMENT
 from lib.core.settings import VERSION_STRING
 from lib.core.settings import WINDOWS_RESERVED_NAMES
+from lib.utils.safe2bin import safechardecode
 from thirdparty import six
 from thirdparty.magic import magic
 
@@ -197,7 +198,7 @@ class Dump(object):
             self._areAdmins = userSettings[1]
             userSettings = userSettings[0]
 
-        users = list(userSettings.keys())
+        users = [_ for _ in userSettings.keys() if _ is not None]
         users.sort(key=lambda _: _.lower() if hasattr(_, "lower") else _)
 
         if conf.api:
@@ -245,7 +246,7 @@ class Dump(object):
                     if table and isListLike(table):
                         table = table[0]
 
-                    maxlength = max(maxlength, len(unsafeSQLIdentificatorNaming(normalizeUnicode(table) or getUnicode(table))))
+                    maxlength = max(maxlength, getConsoleLength(unsafeSQLIdentificatorNaming(getUnicode(table))))
 
             lines = "-" * (int(maxlength) + 2)
 
@@ -266,7 +267,7 @@ class Dump(object):
                         table = table[0]
 
                     table = unsafeSQLIdentificatorNaming(table)
-                    blank = " " * (maxlength - len(normalizeUnicode(table) or getUnicode(table)))
+                    blank = " " * (maxlength - getConsoleLength(getUnicode(table)))
                     self._write("| %s%s |" % (table, blank))
 
                 self._write("+%s+\n" % lines)
@@ -361,7 +362,7 @@ class Dump(object):
             for ctables in dbTables.values():
                 for tables in ctables.values():
                     for table in tables:
-                        maxlength1 = max(maxlength1, len(normalizeUnicode(table) or getUnicode(table)))
+                        maxlength1 = max(maxlength1, getConsoleLength(getUnicode(table)))
 
             for db, counts in dbTables.items():
                 self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db else "Current database")
@@ -387,7 +388,7 @@ class Dump(object):
                     tables.sort(key=lambda _: _.lower() if hasattr(_, "lower") else _)
 
                     for table in tables:
-                        blank1 = " " * (maxlength1 - len(normalizeUnicode(table) or getUnicode(table)))
+                        blank1 = " " * (maxlength1 - getConsoleLength(getUnicode(table)))
                         blank2 = " " * (maxlength2 - len(str(count)))
                         self._write("| %s%s | %d%s |" % (table, blank1, count, blank2))
 
@@ -547,7 +548,7 @@ class Dump(object):
 
                 column = unsafeSQLIdentificatorNaming(column)
                 maxlength = int(info["length"])
-                blank = " " * (maxlength - len(column))
+                blank = " " * (maxlength - getConsoleLength(column))
 
                 self._write("| %s%s" % (column, blank), newline=False)
 
@@ -602,7 +603,7 @@ class Dump(object):
 
                     values.append(value)
                     maxlength = int(info["length"])
-                    blank = " " * (maxlength - len(value))
+                    blank = " " * (maxlength - getConsoleLength(value))
                     self._write("| %s%s" % (value, blank), newline=False, console=console)
 
                     if len(value) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value:

lib/core/option.py

@@ -109,6 +109,7 @@ from lib.core.settings import DEFAULT_TOR_HTTP_PORTS
 from lib.core.settings import DEFAULT_TOR_SOCKS_PORTS
 from lib.core.settings import DEFAULT_USER_AGENT
 from lib.core.settings import DUMMY_URL
+from lib.core.settings import IGNORE_CODE_WILDCARD
 from lib.core.settings import IS_WIN
 from lib.core.settings import KB_CHARS_BOUNDARY_CHAR
 from lib.core.settings import KB_CHARS_LOW_FREQUENCY_ALPHABET
@@ -314,7 +315,7 @@ def _setRequestFromFile():
 
             if url is None:
                 errMsg = "specified file '%s' " % requestFile
-                errMsg += "does not contain a valid HTTP request"
+                errMsg += "does not contain a usable HTTP request (with parameters)"
                 raise SqlmapDataException(errMsg)
 
     if conf.secondReq:
@@ -341,10 +342,15 @@ def _setCrawler():
         if conf.bulkFile:
             targets = getFileItems(conf.bulkFile)
         else:
-            targets = parseSitemap(conf.sitemapUrl)
+            targets = list(parseSitemap(conf.sitemapUrl))
+
         for i in xrange(len(targets)):
             try:
                 target = targets[i]
+
+                if not re.search(r"(?i)\Ahttp[s]*://", target):
+                    target = "http://%s" % target
+
                 crawl(target)
 
                 if conf.verbose in (1, 2):
@@ -466,20 +472,27 @@ def _findPageForms():
     logger.info(infoMsg)
 
     if not any((conf.bulkFile, conf.googleDork, conf.sitemapUrl)):
-        page, _, _ = Request.queryPage(content=True)
+        page, _, _ = Request.queryPage(content=True, ignoreSecondOrder=True)
         if findPageForms(page, conf.url, True, True):
             found = True
     else:
         if conf.bulkFile:
             targets = getFileItems(conf.bulkFile)
         elif conf.sitemapUrl:
-            targets = parseSitemap(conf.sitemapUrl)
+            targets = list(parseSitemap(conf.sitemapUrl))
         elif conf.googleDork:
             targets = [_[0] for _ in kb.targets]
             kb.targets.clear()
+        else:
+            targets = []
+
         for i in xrange(len(targets)):
             try:
-                target = targets[i]
+                target = targets[i].strip()
+
+                if not re.search(r"(?i)\Ahttp[s]*://", target):
+                    target = "http://%s" % target
+
                 page, _, _ = Request.getPage(url=target.strip(), cookie=conf.cookie, crawling=True, raise404=False)
                 if findPageForms(page, target, False, True):
                     found = True
@@ -1154,11 +1167,11 @@ def _setSafeVisit():
             errMsg = "invalid format of a safe request file"
             raise SqlmapSyntaxException(errMsg)
     else:
-        if not re.search(r"\Ahttp[s]*://", conf.safeUrl):
+        if not re.search(r"(?i)\Ahttp[s]*://", conf.safeUrl):
             if ":443/" in conf.safeUrl:
-                conf.safeUrl = "https://" + conf.safeUrl
+                conf.safeUrl = "https://%s" % conf.safeUrl
             else:
-                conf.safeUrl = "http://" + conf.safeUrl
+                conf.safeUrl = "http://%s" % conf.safeUrl
 
     if (conf.safeFreq or 0) <= 0:
         errMsg = "please provide a valid value (>0) for safe frequency (--safe-freq) while using safe visit features"
@@ -1223,7 +1236,7 @@ def _setHTTPAuthentication():
 
     elif not conf.authType and conf.authCred:
         errMsg = "you specified the HTTP authentication credentials, "
-        errMsg += "but did not provide the type"
+        errMsg += "but did not provide the type (e.g. --auth-type=\"basic\")"
         raise SqlmapSyntaxException(errMsg)
 
     elif (conf.authType or "").lower() not in (AUTH_TYPE.BASIC, AUTH_TYPE.DIGEST, AUTH_TYPE.NTLM, AUTH_TYPE.PKI):
@@ -1273,8 +1286,8 @@ def _setHTTPAuthentication():
                 from ntlm import HTTPNtlmAuthHandler
             except ImportError:
                 errMsg = "sqlmap requires Python NTLM third-party library "
-                errMsg += "in order to authenticate via NTLM, "
-                errMsg += "https://github.com/mullender/python-ntlm"
+                errMsg += "in order to authenticate via NTLM. Download from "
+                errMsg += "'https://github.com/mullender/python-ntlm'"
                 raise SqlmapMissingDependence(errMsg)
 
             authHandler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(kb.passwordMgr)
@@ -1565,11 +1578,14 @@ def _cleanupOptions():
         conf.testParameter = []
 
     if conf.ignoreCode:
-        try:
-            conf.ignoreCode = [int(_) for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.ignoreCode)]
-        except ValueError:
-            errMsg = "options '--ignore-code' should contain a list of integer values"
-            raise SqlmapSyntaxException(errMsg)
+        if conf.ignoreCode == IGNORE_CODE_WILDCARD:
+            conf.ignoreCode = xrange(0, 1000)
+        else:
+            try:
+                conf.ignoreCode = [int(_) for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.ignoreCode)]
+            except ValueError:
+                errMsg = "options '--ignore-code' should contain a list of integer values or a wildcard value '%s'" % IGNORE_CODE_WILDCARD
+                raise SqlmapSyntaxException(errMsg)
     else:
         conf.ignoreCode = []
 
@@ -1817,7 +1833,6 @@ def _setConfAttributes():
     conf.path = None
     conf.port = None
     conf.proxyList = None
-    conf.resultsFilename = None
     conf.resultsFP = None
     conf.scheme = None
     conf.tests = []
@@ -1937,6 +1952,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.mergeCookies = None
     kb.multipleCtrlC = False
     kb.negativeLogic = False
+    kb.normalizeCrawlingChoice = None
     kb.nullConnection = None
     kb.oldMsf = None
     kb.orderByColumns = None
@@ -2004,6 +2020,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.threadException = False
     kb.tableExistsChoice = None
     kb.uChar = NULL
+    kb.udfFail = False
     kb.unionDuplicates = False
     kb.wizardMode = False
     kb.xpCmdshellAvailable = False
@@ -2202,6 +2219,13 @@ def _mergeOptions(inputOptions, overrideOptions):
         if hasattr(conf, key) and conf[key] is None:
             conf[key] = value
 
+            if conf.unstable:
+                if key in ("timeSec", "retries", "timeout"):
+                    conf[key] *= 2
+
+    if conf.unstable:
+        conf.forcePartial = True
+
     lut = {}
     for group in optDict.keys():
         lut.update((_.upper(), _) for _ in optDict[group])
@@ -2447,6 +2471,13 @@ def _basicOptionValidation():
             errMsg = "invalid regular expression '%s' ('%s')" % (conf.regexp, getSafeExString(ex))
             raise SqlmapSyntaxException(errMsg)
 
+    if conf.paramExclude:
+        try:
+            re.compile(conf.paramExclude)
+        except Exception as ex:
+            errMsg = "invalid regular expression '%s' ('%s')" % (conf.paramExclude, getSafeExString(ex))
+            raise SqlmapSyntaxException(errMsg)
+
     if conf.crawlExclude:
         try:
             re.compile(conf.crawlExclude)
@@ -2490,6 +2521,10 @@ def _basicOptionValidation():
         errMsg = "option '--csrf-url' requires usage of option '--csrf-token'"
         raise SqlmapSyntaxException(errMsg)
 
+    if conf.csrfMethod and not conf.csrfToken:
+        errMsg = "option '--csrf-method' requires usage of option '--csrf-token'"
+        raise SqlmapSyntaxException(errMsg)
+
     if conf.csrfToken and conf.threads > 1:
         errMsg = "option '--csrf-url' is incompatible with option '--threads'"
         raise SqlmapSyntaxException(errMsg)

lib/core/optiondict.py

@@ -61,6 +61,7 @@ optDict = {
         "skipUrlEncode": "boolean",
         "csrfToken": "string",
         "csrfUrl": "string",
+        "csrfMethod": "string",
         "forceSSL": "boolean",
         "chunked": "boolean",
         "hpp": "boolean",
@@ -237,9 +238,11 @@ optDict = {
         "listTampers": "boolean",
         "offline": "boolean",
         "purge": "boolean",
+        "resultsFile": "string",
         "tmpDir": "string",
-        "wizard": "boolean",
+        "unstable": "boolean",
         "updateAll": "boolean",
+        "wizard": "boolean",
         "verbose": "integer",
     },
 

lib/core/patch.py

@@ -20,6 +20,7 @@ import thirdparty.chardet.universaldetector
 
 from lib.core.common import filterNone
 from lib.core.common import getSafeExString
+from lib.core.common import isDigit
 from lib.core.common import isListLike
 from lib.core.common import readInput
 from lib.core.common import shellExec
@@ -62,6 +63,7 @@ def resolveCrossReferences():
     Place for cross-reference resolution
     """
 
+    lib.core.threads.isDigit = isDigit
     lib.core.threads.readInput = readInput
     lib.core.common.getPageTemplate = getPageTemplate
     lib.core.convert.filterNone = filterNone

lib/core/replication.py

@@ -7,13 +7,13 @@ See the file 'LICENSE' for copying permission
 
 import sqlite3
 
-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.common import getSafeExString
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapGenericException
 from lib.core.exception import SqlmapValueException
 from lib.core.settings import UNICODE_ENCODING
+from lib.utils.safe2bin import safechardecode
 
 class Replication(object):
     """

lib/core/revision.py

@@ -40,9 +40,14 @@ def getRevisionNumber():
             with openFile(filePath, "r") as f:
                 content = getText(f.read())
                 filePath = None
+
                 if content.startswith("ref: "):
-                    filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
-                else:
+                    try:
+                        filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
+                    except UnicodeError:
+                        pass
+
+                if filePath is None:
                     match = re.match(r"(?i)[0-9a-f]{32}", content)
                     retVal = match.group(0) if match else None
                     break

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.9.0"
+VERSION = "1.3.11.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -413,6 +413,9 @@ CANDIDATE_SENTENCE_MIN_LENGTH = 10
 # Character used for marking injectable position inside provided data
 CUSTOM_INJECTION_MARK_CHAR = '*'
 
+# Wildcard value that can be used in option --ignore-code
+IGNORE_CODE_WILDCARD = '*'
+
 # Other way to declare injection position
 INJECT_HERE_REGEX = r"(?i)%INJECT[_ ]?HERE%"
 
@@ -721,7 +724,7 @@ MAX_HELP_OPTION_LENGTH = 18
 MAX_CONNECT_RETRIES = 100
 
 # Strings for detecting formatting errors
-FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException")
+FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException", "Arguments are of the wrong type")
 
 # Regular expression used for extracting ASP.NET view state values
 VIEWSTATE_REGEX = r'(?i)(?P<name>__VIEWSTATE[^"]*)[^>]+value="(?P<result>[^"]+)'

lib/core/target.py

@@ -151,12 +151,13 @@ def _setRequestParams():
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*".+?)"(?<!\\")', functools.partial(process, repl=r'\g<1>%s"' % kb.customInjectionMark), conf.data)
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)(-?\d[\d\.]*)\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)((true|false|null))\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
-                    match = re.search(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]', conf.data)
-                    if match and not (conf.testParameter and match.group("name") not in conf.testParameter):
-                        _ = match.group(2)
-                        _ = re.sub(r'("[^"]+)"', r'\g<1>%s"' % kb.customInjectionMark, _)
-                        _ = re.sub(r'(\A|,|\s+)(-?\d[\d\.]*\b)', r'\g<0>%s' % kb.customInjectionMark, _)
-                        conf.data = conf.data.replace(match.group(0), match.group(0).replace(match.group(2), _))
+                    for match in re.finditer(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]', conf.data):
+                        if not (conf.testParameter and match.group("name") not in conf.testParameter):
+                            _ = match.group(2)
+                            if kb.customInjectionMark not in _:  # Note: only for unprocessed (simple) forms - i.e. non-associative arrays (e.g. [1,2,3])
+                                _ = re.sub(r'("[^"]+)"', r'\g<1>%s"' % kb.customInjectionMark, _)
+                                _ = re.sub(r'(\A|,|\s+)(-?\d[\d\.]*\b)', r'\g<0>%s' % kb.customInjectionMark, _)
+                                conf.data = conf.data.replace(match.group(0), match.group(0).replace(match.group(2), _))
 
                 kb.postHint = POST_HINT.JSON
 
@@ -560,16 +561,18 @@ def _setResultsFile():
         return
 
     if not conf.resultsFP:
-        conf.resultsFilename = os.path.join(paths.SQLMAP_OUTPUT_PATH, time.strftime(RESULTS_FILE_FORMAT).lower())
+        conf.resultsFile = conf.resultsFile or os.path.join(paths.SQLMAP_OUTPUT_PATH, time.strftime(RESULTS_FILE_FORMAT).lower())
+        found = os.path.exists(conf.resultsFile)
+
         try:
-            conf.resultsFP = openFile(conf.resultsFilename, "a", UNICODE_ENCODING, buffering=0)
+            conf.resultsFP = openFile(conf.resultsFile, "a", UNICODE_ENCODING, buffering=0)
         except (OSError, IOError) as ex:
             try:
-                warnMsg = "unable to create results file '%s' ('%s'). " % (conf.resultsFilename, getUnicode(ex))
-                handle, conf.resultsFilename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.RESULTS, suffix=".csv")
+                warnMsg = "unable to create results file '%s' ('%s'). " % (conf.resultsFile, getUnicode(ex))
+                handle, conf.resultsFile = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.RESULTS, suffix=".csv")
                 os.close(handle)
-                conf.resultsFP = openFile(conf.resultsFilename, "w+", UNICODE_ENCODING, buffering=0)
-                warnMsg += "Using temporary file '%s' instead" % conf.resultsFilename
+                conf.resultsFP = openFile(conf.resultsFile, "w+", UNICODE_ENCODING, buffering=0)
+                warnMsg += "Using temporary file '%s' instead" % conf.resultsFile
                 logger.warn(warnMsg)
             except IOError as _:
                 errMsg = "unable to write to the temporary directory ('%s'). " % _
@@ -578,9 +581,10 @@ def _setResultsFile():
                 errMsg += "create temporary files and/or directories"
                 raise SqlmapSystemException(errMsg)
 
-        conf.resultsFP.writelines("Target URL,Place,Parameter,Technique(s),Note(s)%s" % os.linesep)
+        if not found:
+            conf.resultsFP.writelines("Target URL,Place,Parameter,Technique(s),Note(s)%s" % os.linesep)
 
-        logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFilename)
+        logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFile)
 
 def _createFilesDir():
     """

lib/core/testing.py

@@ -75,6 +75,7 @@ def vulnTest():
 
     for options, checks in (
         ("--flush-session", ("CloudFlare",)),
+        ("--flush-session --data='{\"id\": 1}' --banner", ("Payload: {\"id\"", "banner: '3")),
         ("--flush-session --parse-errors --eval=\"id2=2\" --referer=\"localhost\" --cookie=\"PHPSESSID=d41d8cd98f00b204e9800998ecf8427e\"", (": syntax error", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "back-end DBMS: SQLite", "3 columns")),
         ("--banner --schema --dump -T users --binary-fields=surname --where \"id>3\"", ("banner: '3", "INTEGER", "TEXT", "id", "name", "surname", "2 entries", "6E616D6569736E756C6C")),
         ("--all --tamper=between,randomcase", ("5 entries", "luther", "blisset", "fluffy", "179ad45c6ce2cb97cf1029e212046e81", "NULL", "nameisnull", "testpass")),

lib/core/threads.py

@@ -73,6 +73,10 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
     # It will be overwritten by original from lib.core.common
     pass
 
+def isDigit(value):
+    # It will be overwritten by original from lib.core.common
+    pass
+
 def getCurrentThreadData():
     """
     Returns current thread's local data
@@ -125,10 +129,12 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
             choice = readInput(message, default=str(numThreads))
             if choice:
                 skipThreadCheck = False
+
                 if choice.endswith('!'):
                     choice = choice[:-1]
                     skipThreadCheck = True
-                if choice.isdigit():
+
+                if isDigit(choice):
                     if int(choice) > MAX_NUMBER_OF_THREADS and not skipThreadCheck:
                         errMsg = "maximum number of used threads is %d avoiding potential connection issues" % MAX_NUMBER_OF_THREADS
                         logger.critical(errMsg)

lib/core/wordlist.py

@@ -42,7 +42,7 @@ class Wordlist(six.Iterator):
     def adjust(self):
         self.closeFP()
         if self.index > len(self.filenames):
-            raise StopIteration
+            return  # Note: https://stackoverflow.com/a/30217723 (PEP 479)
         elif self.index == len(self.filenames):
             self.iter = iter(self.custom)
         else:

lib/parse/cmdline.py

@@ -44,6 +44,26 @@ finally:
     def get_groups(parser):
         return getattr(parser, "option_groups", None) or getattr(parser, "_action_groups")
 
+    def get_all_options(parser):
+        retVal = set()
+
+        for option in get_actions(parser):
+            if hasattr(option, "option_strings"):
+                retVal.update(option.option_strings)
+            else:
+                retVal.update(option._long_opts)
+                retVal.update(option._short_opts)
+
+        for group in get_groups(parser):
+            for option in get_actions(group):
+                if hasattr(option, "option_strings"):
+                    retVal.update(option.option_strings)
+                else:
+                    retVal.update(option._long_opts)
+                    retVal.update(option._short_opts)
+
+        return retVal
+
 from lib.core.common import checkOldOptions
 from lib.core.common import checkSystemEncoding
 from lib.core.common import dataToStdout
@@ -245,6 +265,9 @@ def cmdLineParser(argv=None):
         request.add_argument("--csrf-url", dest="csrfUrl",
             help="URL address to visit for extraction of anti-CSRF token")
 
+        request.add_argument("--csrf-method", dest="csrfMethod",
+            help="HTTP method to use during anti-CSRF token page visit")
+
         request.add_argument("--force-ssl", dest="forceSSL", action="store_true",
             help="Force usage of SSL/HTTPS")
 
@@ -705,12 +728,18 @@ def cmdLineParser(argv=None):
         miscellaneous.add_argument("--purge", dest="purge", action="store_true",
             help="Safely remove all content from sqlmap data directory")
 
+        miscellaneous.add_argument("--results-file", dest="resultsFile",
+            help="Location of CSV results file in multiple targets mode")
+
         miscellaneous.add_argument("--sqlmap-shell", dest="sqlmapShell", action="store_true",
             help="Prompt for an interactive sqlmap shell")
 
         miscellaneous.add_argument("--tmp-dir", dest="tmpDir",
             help="Local directory for storing temporary files")
 
+        miscellaneous.add_argument("--unstable", dest="unstable", action="store_true",
+            help="Adjust options for unstable connections")
+
         miscellaneous.add_argument("--update", dest="updateAll", action="store_true",
             help="Update sqlmap")
 
@@ -841,18 +870,10 @@ def cmdLineParser(argv=None):
             parser.usage = ""
             cmdLineOptions.sqlmapShell = True
 
-            _ = ["x", "q", "exit", "quit", "clear"]
+            commands = set(("x", "q", "exit", "quit", "clear"))
+            commands.update(get_all_options(parser))
 
-            for option in get_actions(parser):
-                _.extend(option._long_opts)
-                _.extend(option._short_opts)
-
-            for group in get_groups(parser):
-                for option in get_actions(group):
-                    _.extend(option._long_opts)
-                    _.extend(option._short_opts)
-
-            autoCompletion(AUTOCOMPLETE_TYPE.SQLMAP, commands=_)
+            autoCompletion(AUTOCOMPLETE_TYPE.SQLMAP, commands=commands)
 
             while True:
                 command = None
@@ -887,8 +908,11 @@ def cmdLineParser(argv=None):
                 raise SqlmapSyntaxException("something went wrong during command line parsing ('%s')" % getSafeExString(ex))
 
         for i in xrange(len(argv)):
+            longOptions = set(re.findall(r"\-\-([^= ]+?)=", parser.format_help()))
             if argv[i] == "-hh":
                 argv[i] = "-h"
+            elif i == 1 and re.search(r"\A(http|www\.|\w[\w.-]+\.\w{2,})", argv[i]) is not None:
+                argv[i] = "--url=%s" % argv[i]
             elif len(argv[i]) > 1 and all(ord(_) in xrange(0x2018, 0x2020) for _ in ((argv[i].split('=', 1)[-1].strip() or ' ')[0], argv[i][-1])):
                 dataToStdout("[!] copy-pasting illegal (non-console) quote characters from Internet is, well, illegal (%s)\n" % argv[i])
                 raise SystemExit
@@ -934,6 +958,9 @@ def cmdLineParser(argv=None):
                             found = True
                     if not found:
                         get_groups(parser).remove(group)
+            elif '=' in argv[i] and not argv[i].startswith('-') and argv[i].split('=')[0] in longOptions and re.search(r"\A-\w\Z", argv[i - 1]) is None:
+                dataToStdout("[!] detected usage of long-option without a starting hyphen ('%s')\n" % argv[i])
+                raise SystemExit
 
         for verbosity in (_ for _ in argv if re.search(r"\A\-v+\Z", _)):
             try:

lib/request/basic.py

@@ -34,6 +34,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.decorators import cachedmethod
+from lib.core.decorators import lockedmethod
 from lib.core.dicts import HTML_ENTITIES
 from lib.core.enums import DBMS
 from lib.core.enums import HTTP_HEADER
@@ -57,6 +58,7 @@ from thirdparty.odict import OrderedDict
 from thirdparty.six import unichr as _unichr
 from thirdparty.six.moves import http_client as _http_client
 
+@lockedmethod
 def forgeHeaders(items=None, base=None):
     """
     Prepare HTTP Cookie, HTTP User-Agent and HTTP Referer headers to use when performing
@@ -110,9 +112,9 @@ def forgeHeaders(items=None, base=None):
                     if conf.loadCookies:
                         conf.httpHeaders = filterNone((item if item[0] != HTTP_HEADER.COOKIE else None) for item in conf.httpHeaders)
                     elif kb.mergeCookies is None:
-                        message = "you provided a HTTP %s header value. " % HTTP_HEADER.COOKIE
-                        message += "The target URL provided its own cookies within "
-                        message += "the HTTP %s header which intersect with yours. " % HTTP_HEADER.SET_COOKIE
+                        message = "you provided a HTTP %s header value, while " % HTTP_HEADER.COOKIE
+                        message += "target URL provides its own cookies within "
+                        message += "HTTP %s header which intersect with yours. " % HTTP_HEADER.SET_COOKIE
                         message += "Do you want to merge them in further requests? [Y/n] "
 
                         kb.mergeCookies = readInput(message, default='Y', boolean=True)

lib/request/basicauthhandler.py

@@ -12,6 +12,7 @@ class SmartHTTPBasicAuthHandler(_urllib.request.HTTPBasicAuthHandler):
     Reference: http://selenic.com/hg/rev/6c51a5056020
     Fix for a: http://bugs.python.org/issue8797
     """
+
     def __init__(self, *args, **kwargs):
         _urllib.request.HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
         self.retried_req = set()

lib/request/connect.py

@@ -22,7 +22,6 @@ except ImportError:
     class WebSocketException(Exception):
         pass
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import asciifyUrl
 from lib.core.common import calculateDeltaSeconds
@@ -125,6 +124,7 @@ from lib.request.basic import processResponse
 from lib.request.comparison import comparison
 from lib.request.direct import direct
 from lib.request.methodrequest import MethodRequest
+from lib.utils.safe2bin import safecharencode
 from thirdparty import six
 from thirdparty.odict import OrderedDict
 from thirdparty.six import unichr as _unichr
@@ -839,7 +839,7 @@ class Connect(object):
 
     @staticmethod
     @stackedmethod
-    def queryPage(value=None, place=None, content=False, getRatioValue=False, silent=False, method=None, timeBasedCompare=False, noteResponseTime=True, auxHeaders=None, response=False, raise404=None, removeReflection=True, disableTampering=False):
+    def queryPage(value=None, place=None, content=False, getRatioValue=False, silent=False, method=None, timeBasedCompare=False, noteResponseTime=True, auxHeaders=None, response=False, raise404=None, removeReflection=True, disableTampering=False, ignoreSecondOrder=False):
         """
         This method calls a function to get the target URL page content
         and returns its page ratio (0 <= ratio <= 1) or a boolean value
@@ -1039,7 +1039,7 @@ class Connect(object):
                 return retVal
 
             token = AttribDict()
-            page, headers, code = Connect.getPage(url=conf.csrfUrl or conf.url, data=conf.data if conf.csrfUrl == conf.url else None, method=conf.method if conf.csrfUrl == conf.url else None, cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST))
+            page, headers, code = Connect.getPage(url=conf.csrfUrl or conf.url, data=conf.data if conf.csrfUrl == conf.url else None, method=conf.csrfMethod or (conf.method if conf.csrfUrl == conf.url else None), cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST))
             page = urldecode(page)  # for anti-CSRF tokens with special characters in their name (e.g. 'foo:bar=...')
 
             match = re.search(r"(?i)<input[^>]+\bname=[\"']?(?P<name>%s)\b[^>]*\bvalue=[\"']?(?P<value>[^>'\"]*)" % conf.csrfToken, page or "", re.I)
@@ -1339,17 +1339,18 @@ class Connect(object):
                     warnMsg += "behavior in custom WAF/IPS solutions"
                 singleTimeWarnMessage(warnMsg)
 
-        if conf.secondUrl:
-            page, headers, code = Connect.getPage(url=conf.secondUrl, cookie=cookie, ua=ua, silent=silent, auxHeaders=auxHeaders, response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
-        elif kb.secondReq and IPS_WAF_CHECK_PAYLOAD not in _urllib.parse.unquote(value or ""):
-            def _(value):
-                if kb.customInjectionMark in (value or ""):
-                    if payload is None:
-                        value = value.replace(kb.customInjectionMark, "")
-                    else:
-                        value = re.sub(r"\w*%s" % re.escape(kb.customInjectionMark), payload, value)
-                return value
-            page, headers, code = Connect.getPage(url=_(kb.secondReq[0]), post=_(kb.secondReq[2]), method=kb.secondReq[1], cookie=kb.secondReq[3], silent=silent, auxHeaders=dict(auxHeaders, **dict(kb.secondReq[4])), response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
+        if not ignoreSecondOrder:
+            if conf.secondUrl:
+                page, headers, code = Connect.getPage(url=conf.secondUrl, cookie=cookie, ua=ua, silent=silent, auxHeaders=auxHeaders, response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
+            elif kb.secondReq and IPS_WAF_CHECK_PAYLOAD not in _urllib.parse.unquote(value or ""):
+                def _(value):
+                    if kb.customInjectionMark in (value or ""):
+                        if payload is None:
+                            value = value.replace(kb.customInjectionMark, "")
+                        else:
+                            value = re.sub(r"\w*%s" % re.escape(kb.customInjectionMark), payload, value)
+                    return value
+                page, headers, code = Connect.getPage(url=_(kb.secondReq[0]), post=_(kb.secondReq[2]), method=kb.secondReq[1], cookie=kb.secondReq[3], silent=silent, auxHeaders=dict(auxHeaders, **dict(kb.secondReq[4])), response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
 
         threadData.lastQueryDuration = calculateDeltaSeconds(start)
 

lib/request/direct.py

@@ -7,7 +7,6 @@ See the file 'LICENSE' for copying permission
 
 import time
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -26,6 +25,7 @@ from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import TIMEOUT_STATE
 from lib.core.settings import UNICODE_ENCODING
+from lib.utils.safe2bin import safecharencode
 from lib.utils.timeout import timeout
 
 def direct(query, content=True):

lib/request/inject.py

@@ -24,6 +24,7 @@ from lib.core.common import getTechniqueData
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import initTechnique
+from lib.core.common import isDigit
 from lib.core.common import isNoneValue
 from lib.core.common import isNumPosStrValue
 from lib.core.common import isTechniqueAvailable
@@ -235,7 +236,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
                             elif choice == 'Q':
                                 raise SqlmapUserQuitException
 
-                            elif choice.isdigit() and int(choice) > 0 and int(choice) <= count:
+                            elif isDigit(choice) and int(choice) > 0 and int(choice) <= count:
                                 stopLimit = int(choice)
 
                                 infoMsg = "sqlmap is now going to retrieve the "
@@ -246,7 +247,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
                                 message = "how many? "
                                 stopLimit = readInput(message, default="10")
 
-                                if not stopLimit.isdigit():
+                                if not isDigit(stopLimit):
                                     errMsg = "invalid choice"
                                     logger.error(errMsg)
 
@@ -261,7 +262,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
 
                                 return None
 
-                    elif count and not count.isdigit():
+                    elif count and not isDigit(count):
                         warnMsg = "it was not possible to count the number "
                         warnMsg += "of entries for the SQL query provided. "
                         warnMsg += "sqlmap will assume that it returns only "

lib/takeover/abstraction.py

@@ -9,7 +9,6 @@ from __future__ import print_function
 
 import sys
 
-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.common import Backend
 from lib.core.common import dataToStdout
 from lib.core.common import getSQLSnippet
@@ -17,6 +16,7 @@ from lib.core.common import isStackingAvailable
 from lib.core.common import readInput
 from lib.core.convert import getUnicode
 from lib.core.data import conf
+from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.enums import AUTOCOMPLETE_TYPE
 from lib.core.enums import DBMS
@@ -28,6 +28,7 @@ from lib.request import inject
 from lib.takeover.udf import UDF
 from lib.takeover.web import Web
 from lib.takeover.xp_cmdshell import XP_cmdshell
+from lib.utils.safe2bin import safechardecode
 from thirdparty.six.moves import input as _input
 
 class Abstraction(Web, UDF, XP_cmdshell):
@@ -48,7 +49,7 @@ class Abstraction(Web, UDF, XP_cmdshell):
         if Backend.isDbms(DBMS.PGSQL) and self.checkCopyExec():
             self.copyExecCmd(cmd)
 
-        elif self.webBackdoorUrl and not isStackingAvailable():
+        elif self.webBackdoorUrl and (not isStackingAvailable() or kb.udfFail):
             self.webBackdoorRunCmd(cmd)
 
         elif Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL):
@@ -67,7 +68,7 @@ class Abstraction(Web, UDF, XP_cmdshell):
         if Backend.isDbms(DBMS.PGSQL) and self.checkCopyExec():
             retVal = self.copyExecCmd(cmd)
 
-        elif self.webBackdoorUrl and not isStackingAvailable():
+        elif self.webBackdoorUrl and (not isStackingAvailable() or kb.udfFail):
             retVal = self.webBackdoorRunCmd(cmd)
 
         elif Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL):
@@ -104,7 +105,7 @@ class Abstraction(Web, UDF, XP_cmdshell):
             self.execCmd(cmd)
 
     def shell(self):
-        if self.webBackdoorUrl and not isStackingAvailable():
+        if self.webBackdoorUrl and (not isStackingAvailable() or kb.udfFail):
             infoMsg = "calling OS shell. To quit type "
             infoMsg += "'x' or 'q' and press ENTER"
             logger.info(infoMsg)

lib/takeover/metasploit.py

@@ -23,6 +23,7 @@ from lib.core.common import dataToStdout
 from lib.core.common import Backend
 from lib.core.common import getLocalIP
 from lib.core.common import getRemoteIP
+from lib.core.common import isDigit
 from lib.core.common import normalizePath
 from lib.core.common import ntToPosixSlashes
 from lib.core.common import pollProcess
@@ -154,7 +155,7 @@ class Metasploit(object):
 
         choice = readInput(message, default="%d" % default)
 
-        if not choice or not choice.isdigit() or int(choice) > maxValue or int(choice) < 1:
+        if not choice or not isDigit(choice) or int(choice) > maxValue or int(choice) < 1:
             choice = default
 
         choice = int(choice)
@@ -241,7 +242,7 @@ class Metasploit(object):
                         elif Backend.isDbms(DBMS.MSSQL) and Backend.isVersionWithin(("2005", "2008")):
                             break
 
-                    elif not choice.isdigit():
+                    elif not isDigit(choice):
                         logger.warn("invalid value, only digits are allowed")
 
                     elif int(choice) < 1 or int(choice) > 2:
@@ -692,9 +693,9 @@ class Metasploit(object):
         self._runMsfCliSmbrelay()
 
         if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL):
-            self.uncPath = "\\\\\\\\%s\\\\%s" % (self.lhostStr, self._randFile)
+            self.uncPath = r"\\\\%s\\%s" % (self.lhostStr, self._randFile)
         else:
-            self.uncPath = "\\\\%s\\%s" % (self.lhostStr, self._randFile)
+            self.uncPath = r"\\%s\%s" % (self.lhostStr, self._randFile)
 
         debugMsg = "Metasploit Framework console exited with return "
         debugMsg += "code %s" % self._controlMsfCmd(self._msfCliProc, self.uncPathRequest)

lib/takeover/udf.py

@@ -11,6 +11,7 @@ from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import dataToStdout
+from lib.core.common import isDigit
 from lib.core.common import isStackingAvailable
 from lib.core.common import readInput
 from lib.core.common import unArrayizeValue
@@ -339,11 +340,9 @@ class UDF(object):
 
                 if choice == 'Q':
                     break
-                elif hasattr(choice, "isdigit") and choice.isdigit() and int(choice) > 0 and int(choice) <= len(udfList):
+                elif isDigit(choice) and int(choice) > 0 and int(choice) <= len(udfList):
                     choice = int(choice)
                     break
-                elif isinstance(choice, int) and choice > 0 and choice <= len(udfList):
-                    break
                 else:
                     warnMsg = "invalid value, only digits >= 1 and "
                     warnMsg += "<= %d are allowed" % len(udfList)

lib/takeover/web.py

@@ -22,6 +22,7 @@ from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSQLSnippet
 from lib.core.common import getTechnique
 from lib.core.common import getTechniqueData
+from lib.core.common import isDigit
 from lib.core.common import isTechniqueAvailable
 from lib.core.common import isWindowsDriveLetterPath
 from lib.core.common import normalizePath
@@ -200,7 +201,7 @@ class Web(object):
         while True:
             choice = readInput(message, default=str(default))
 
-            if not choice.isdigit():
+            if not isDigit(choice):
                 logger.warn("invalid value, only digits are allowed")
 
             elif int(choice) < 1 or int(choice) > len(choices):

lib/techniques/blind/inference.py

@@ -10,7 +10,6 @@ from __future__ import division
 import re
 import time
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -58,6 +57,7 @@ from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.utils.progress import ProgressBar
+from lib.utils.safe2bin import safecharencode
 from lib.utils.xrange import xrange
 
 def bisection(payload, expression, length=None, charsetType=None, firstChar=None, lastChar=None, dump=False):

lib/techniques/dns/use.py

@@ -8,7 +8,6 @@ See the file 'LICENSE' for copying permission
 import re
 import time
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -33,6 +32,7 @@ from lib.core.settings import MAX_DNS_LABEL
 from lib.core.settings import PARTIAL_VALUE_MARKER
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
+from lib.utils.safe2bin import safecharencode
 
 def dnsUse(payload, expression):
     """

lib/techniques/error/use.py

@@ -10,7 +10,6 @@ from __future__ import print_function
 import re
 import time
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
 from lib.core.common import Backend
@@ -60,6 +59,7 @@ from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.utils.progress import ProgressBar
+from lib.utils.safe2bin import safecharencode
 from thirdparty import six
 
 def _oneShotErrorUse(expression, field=None, chunkTest=False):

lib/techniques/union/use.py

@@ -10,7 +10,6 @@ import re
 import time
 import xml.etree.ElementTree
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
 from lib.core.common import arrayizeValue
@@ -62,6 +61,7 @@ from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.utils.progress import ProgressBar
+from lib.utils.safe2bin import safecharencode
 from thirdparty import six
 from thirdparty.odict import OrderedDict
 
@@ -123,7 +123,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
 
                             try:
                                 decodeBase64(value)
-                            except binascii.Error:
+                            except (binascii.Error, TypeError):
                                 base64 = False
                                 break
 

lib/utils/brute.py

@@ -163,7 +163,7 @@ def tableExists(tableFile, regex=None):
         if not threadData.shared.files:
             warnMsg = "no table(s) found"
             if conf.db:
-                 warnMsg += "for database '%s'" % conf.db
+                 warnMsg += " for database '%s'" % conf.db
             logger.warn(warnMsg)
         else:
             for item in threadData.shared.files:

lib/utils/crawler.py

@@ -15,6 +15,7 @@ import time
 from lib.core.common import checkSameHost
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
+from lib.core.common import extractRegexResult
 from lib.core.common import findPageForms
 from lib.core.common import getSafeExString
 from lib.core.common import openFile
@@ -92,7 +93,7 @@ def crawl(target):
                         soup = BeautifulSoup(content)
                         tags = soup('a')
 
-                        tags += re.finditer(r'(?i)<a[^>]+href=["\'](?P<href>[^>"\']+)', content)
+                        tags += re.finditer(r'(?i)\s(href|src)=["\'](?P<href>[^>"\']+)', content)
 
                         for tag in tags:
                             href = tag.get("href") if hasattr(tag, "get") else tag.group("href")
@@ -111,7 +112,7 @@ def crawl(target):
                                 elif not _:
                                     continue
 
-                                if url.split('.')[-1].lower() not in CRAWL_EXCLUDE_EXTENSIONS:
+                                if (extractRegexResult(r"\A[^?]+\.(?P<result>\w+)(\?|\Z)", url) or "").lower() not in CRAWL_EXCLUDE_EXTENSIONS:
                                     with kb.locks.value:
                                         threadData.shared.deeper.add(url)
                                         if re.search(r"(.*?)\?(.+)", url):
@@ -194,6 +195,26 @@ def crawl(target):
             for url in threadData.shared.value:
                 kb.targets.add((urldecode(url, kb.pageEncoding), None, None, None, None))
 
+        if kb.normalizeCrawlingChoice is None:
+            message = "do you want to normalize "
+            message += "crawling results [Y/n] "
+
+            kb.normalizeCrawlingChoice = readInput(message, default='Y', boolean=True)
+
+        if kb.normalizeCrawlingChoice:
+            seen = set()
+            results = OrderedSet()
+
+            for target in kb.targets:
+                match = re.search(r"/[^/?]*\?.*\Z", target[0])
+                if match:
+                    key = re.sub(r"=[^=&]*", "=", match.group(0))
+                    if key not in seen:
+                        results.add(target)
+                        seen.add(key)
+
+            kb.targets = results
+
         storeResultsToFile(kb.targets)
 
 def storeResultsToFile(results):

lib/utils/hash.py

@@ -1000,7 +1000,7 @@ def dictionaryAttack(attack_dict):
                                 resumes.append((user, hash_, resumed))
                             keys.add(hash_)
 
-                    except (binascii.Error, IndexError):
+                    except (binascii.Error, TypeError, IndexError):
                         pass
 
         if not attack_info:

lib/utils/pivotdumptable.py

@@ -7,7 +7,6 @@ See the file 'LICENSE' for copying permission
 
 import re
 
-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
 from lib.core.common import Backend
@@ -33,6 +32,7 @@ from lib.core.settings import MAX_INT
 from lib.core.settings import NULL
 from lib.core.unescaper import unescaper
 from lib.request import inject
+from lib.utils.safe2bin import safechardecode
 from thirdparty.six import unichr as _unichr
 
 def pivotDumpTable(table, colList, count=None, blind=True, alias=None):

lib/utils/purge.py

@@ -15,6 +15,7 @@ import string
 from lib.core.common import getSafeExString
 from lib.core.common import openFile
 from lib.core.compat import xrange
+from lib.core.convert import getUnicode
 from lib.core.data import logger
 from thirdparty.six import unichr as _unichr
 
@@ -84,4 +85,4 @@ def purge(directory):
     try:
         shutil.rmtree(directory)
     except OSError as ex:
-        logger.error("problem occurred while removing directory '%s' ('%s')" % (directory, getSafeExString(ex)))
+        logger.error("problem occurred while removing directory '%s' ('%s')" % (getUnicode(directory), getSafeExString(ex)))

lib/utils/safe2bin.py

@@ -1,23 +1,15 @@
 #!/usr/bin/env python
 
 """
-safe2bin.py - Simple safe(hex) to binary format converter
-
 Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-from __future__ import print_function
-
 import binascii
 import re
 import string
-import os
 import sys
 
-from optparse import OptionError
-from optparse import OptionParser
-
 if sys.version_info >= (3, 0):
     xrange = range
     text_type = str
@@ -49,10 +41,10 @@ def safecharencode(value):
     """
     Returns safe representation of a given basestring value
 
-    >>> safecharencode(u'test123')
-    u'test123'
-    >>> safecharencode(u'test\x01\x02\xff')
-    u'test\\01\\02\\03\\ff'
+    >>> safecharencode(u'test123') == u'test123'
+    True
+    >>> safecharencode(u'test\x01\x02\xaf') == u'test\\\\x01\\\\x02\\xaf'
+    True
     """
 
     retVal = value
@@ -107,37 +99,3 @@ def safechardecode(value, binary=False):
             retVal[i] = safechardecode(value[i])
 
     return retVal
-
-def main():
-    usage = '%s -i <input file> [-o <output file>]' % sys.argv[0]
-    parser = OptionParser(usage=usage, version='0.1')
-
-    try:
-        parser.add_option('-i', dest='inputFile', help='Input file')
-        parser.add_option('-o', dest='outputFile', help='Output file')
-
-        (args, _) = parser.parse_args()
-
-        if not args.inputFile:
-            parser.error('Missing the input file, -h for help')
-
-    except (OptionError, TypeError) as ex:
-        parser.error(ex)
-
-    if not os.path.isfile(args.inputFile):
-        print('ERROR: the provided input file \'%s\' is not a regular file' % args.inputFile)
-        sys.exit(1)
-
-    f = open(args.inputFile, 'r')
-    data = f.read()
-    f.close()
-
-    if not args.outputFile:
-        args.outputFile = args.inputFile + '.bin'
-
-    f = open(args.outputFile, 'wb')
-    f.write(safechardecode(data))
-    f.close()
-
-if __name__ == '__main__':
-    main()

lib/utils/sqlalchemy.py

@@ -76,7 +76,8 @@ class SQLAlchemy(GenericConnector):
                         pass
                 elif "invalid literal for int() with base 10: '0b" in traceback.format_exc():
                     raise SqlmapConnectionException("SQLAlchemy connection issue ('https://bitbucket.org/zzzeek/sqlalchemy/issues/3975')")
-                raise
+                else:
+                    pass
             except SqlmapFilePathException:
                 raise
             except Exception as ex:

plugins/dbms/informix/fingerprint.py

@@ -96,7 +96,7 @@ class Fingerprint(GenericFingerprint):
             infoMsg = "actively fingerprinting %s" % DBMS.INFORMIX
             logger.info(infoMsg)
 
-            for version in ("12.1", "11.7", "11.5"):
+            for version in ("14.1", "12.1", "11.7", "11.5", "10.0"):
                 output = inject.checkBooleanExpression("EXISTS(SELECT 1 FROM SYSMASTER:SYSDUAL WHERE DBINFO('VERSION,'FULL') LIKE '%%%s%%')" % version)
 
                 if output:

plugins/dbms/mssqlserver/fingerprint.py

@@ -94,7 +94,8 @@ class Fingerprint(GenericFingerprint):
                 ("2008", "SYSDATETIME()=SYSDATETIME()"),
                 ("2012", "CONCAT(NULL,NULL)=CONCAT(NULL,NULL)"),
                 ("2014", "CHARINDEX('12.0.2000',@@version)>0"),
-                ("2016", "ISJSON(NULL) IS NULL")
+                ("2016", "ISJSON(NULL) IS NULL"),
+                ("2017", "TRIM(NULL) IS NULL")
             ):
                 result = inject.checkBooleanExpression(check)
 

plugins/dbms/mysql/fingerprint.py

@@ -53,7 +53,7 @@ class Fingerprint(GenericFingerprint):
             (50600, 50646),  # MySQL 5.6
             (50700, 50726),  # MySQL 5.7
             (60000, 60014),  # MySQL 6.0
-            (80000, 80015),  # MySQL 8.0
+            (80000, 80017),  # MySQL 8.0
         )
 
         index = -1

plugins/dbms/oracle/fingerprint.py

@@ -105,7 +105,7 @@ class Fingerprint(GenericFingerprint):
             logger.info(infoMsg)
 
             # Reference: https://en.wikipedia.org/wiki/Oracle_Database
-            for version in ("18c", "12c", "11g", "10g", "9i", "8i"):
+            for version in ("18c", "12c", "11g", "10g", "9i", "8i", "7"):
                 number = int(re.search(r"([\d]+)", version).group(1))
                 output = inject.checkBooleanExpression("%d=(SELECT SUBSTR((VERSION),1,%d) FROM SYS.PRODUCT_COMPONENT_VERSION WHERE ROWNUM=1)" % (number, 1 if number < 10 else 2))
 

plugins/dbms/postgresql/takeover.py

@@ -10,6 +10,7 @@ import os
 from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import decloakToTemp
+from lib.core.common import flattenValue
 from lib.core.common import isListLike
 from lib.core.common import isStackingAvailable
 from lib.core.common import randomStr
@@ -104,7 +105,7 @@ class Takeover(GenericTakeover):
             output = inject.getValue(query, resumeValue=False)
 
             if isListLike(output):
-                output = os.linesep.join(output)
+                output = os.linesep.join(flattenValue(output))
 
             self._cleanupCmd = "DROP TABLE %s" % self.cmdTblName
             inject.goStacked(self._cleanupCmd)

plugins/generic/entries.py

@@ -25,6 +25,7 @@ from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unArrayizeValue
 from lib.core.common import unsafeSQLIdentificatorNaming
+from lib.core.convert import getConsoleLength
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -269,7 +270,7 @@ class Entries(object):
                                 else:
                                     colEntry = unArrayizeValue(entry[index]) if index < len(entry) else u''
 
-                                maxLen = max(len(column), len(DUMP_REPLACEMENTS.get(getUnicode(colEntry), getUnicode(colEntry))))
+                                maxLen = max(getConsoleLength(column), getConsoleLength(DUMP_REPLACEMENTS.get(getUnicode(colEntry), getUnicode(colEntry))))
 
                                 if maxLen > kb.data.dumpedTable[column]["length"]:
                                     kb.data.dumpedTable[column]["length"] = maxLen
@@ -506,7 +507,7 @@ class Entries(object):
                         logger.info(infoMsg)
 
     def dumpFoundColumn(self, dbs, foundCols, colConsider):
-        message = "do you want to dump entries? [Y/n] "
+        message = "do you want to dump found column(s) entries? [Y/n] "
 
         if not readInput(message, default='Y', boolean=True):
             return
@@ -573,7 +574,7 @@ class Entries(object):
                     conf.dumper.dbTableValues(data)
 
     def dumpFoundTables(self, tables):
-        message = "do you want to dump tables' entries? [Y/n] "
+        message = "do you want to dump found table(s) entries? [Y/n] "
 
         if not readInput(message, default='Y', boolean=True):
             return

plugins/generic/misc.py

@@ -137,7 +137,7 @@ class Miscellaneous(object):
             self.delRemoteFile(self.webStagerFilePath)
             self.delRemoteFile(self.webBackdoorFilePath)
 
-        if not isStackingAvailable() and not conf.direct:
+        if (not isStackingAvailable() or kb.udfFail) and not conf.direct:
             return
 
         if any((conf.osCmd, conf.osShell)) and Backend.isDbms(DBMS.PGSQL) and kb.copyExecTest:

plugins/generic/takeover.py

@@ -9,11 +9,13 @@ import os
 
 from lib.core.common import Backend
 from lib.core.common import getSafeExString
+from lib.core.common import isDigit
 from lib.core.common import isStackingAvailable
 from lib.core.common import openFile
 from lib.core.common import readInput
 from lib.core.common import runningAsAdmin
 from lib.core.data import conf
+from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.enums import DBMS
 from lib.core.enums import OS
@@ -78,7 +80,20 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
             raise SqlmapNotVulnerableException(errMsg)
 
         self.getRemoteTempPath()
-        self.initEnv(web=web)
+
+        try:
+            self.initEnv(web=web)
+        except SqlmapFilePathException:
+            if not web:
+                infoMsg = "falling back to web backdoor method..."
+                logger.info(infoMsg)
+
+                web = True
+                kb.udfFail = True
+
+                self.initEnv(web=web)
+            else:
+                raise
 
         if not web or (web and self.webBackdoorUrl is not None):
             self.shell()
@@ -101,7 +116,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
             while True:
                 tunnel = readInput(msg, default='1')
 
-                if tunnel.isdigit() and int(tunnel) in (1, 2):
+                if isDigit(tunnel) and int(tunnel) in (1, 2):
                     tunnel = int(tunnel)
                     break
 
@@ -172,7 +187,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
                     while True:
                         choice = readInput(msg, default='1')
 
-                        if choice.isdigit() and int(choice) in (1, 2):
+                        if isDigit(choice) and int(choice) in (1, 2):
                             choice = int(choice)
                             break
 

plugins/generic/users.py

@@ -600,7 +600,8 @@ class Users(object):
 
                     # In Firebird we get one letter for each privilege
                     elif Backend.isDbms(DBMS.FIREBIRD):
-                        privileges.add(FIREBIRD_PRIVS[privilege.strip()])
+                        if privilege.strip() in FIREBIRD_PRIVS:
+                            privileges.add(FIREBIRD_PRIVS[privilege.strip()])
 
                     # In Informix we get one letter for the highest privilege
                     elif Backend.isDbms(DBMS.INFORMIX):

sqlmap.conf

@@ -180,16 +180,19 @@ safeReqFile =
 # Default: 0
 safeFreq = 0
 
-# Skip URL encoding of payload data
+# Skip URL encoding of payload data.
 # Valid: True or False
 skipUrlEncode = False
 
-# Parameter used to hold anti-CSRF token
+# Parameter used to hold anti-CSRF token.
 csrfToken = 
 
 # URL address to visit to extract anti-CSRF token
 csrfUrl = 
 
+# HTTP method to use during anti-CSRF token page visit.
+csrfMethod =
+
 # Force usage of SSL/HTTPS
 # Valid: True or False
 forceSSL = False
@@ -819,17 +822,24 @@ listTampers = False
 # Valid: True or False
 offline = False
 
+# Location of CSV results file in multiple targets mode.
+resultsFile =
+
 # Local directory for storing temporary files.
 tmpDir =
 
-# Simple wizard interface for beginner users.
+# Adjust options for unstable connections.
 # Valid: True or False
-wizard = False
+unstable = False
 
 # Update sqlmap.
 # Valid: True or False
 updateAll = False
 
+# Simple wizard interface for beginner users.
+# Valid: True or False
+wizard = False
+
 # Verbosity level.
 # Valid: integer between 0 and 6
 # 0: Show only error and critical messages

sqlmap.py

@@ -263,6 +263,11 @@ def main():
             logger.critical(errMsg)
             raise SystemExit
 
+        elif all(_ in excMsg for _ in ("Access is denied", "subprocess", "metasploit")):
+            errMsg = "permission error occurred while running Metasploit"
+            logger.critical(errMsg)
+            raise SystemExit
+
         elif all(_ in excMsg for _ in ("No such file", "_'")):
             errMsg = "corrupted installation detected ('%s'). " % excMsg.strip().split('\n')[-1]
             errMsg += "You should retrieve the latest development version from official GitHub "

tamper/xforwardedfor.py

@@ -16,20 +16,29 @@ def dependencies():
     pass
 
 def randomIP():
-    numbers = []
+    octets = []
 
-    while not numbers or numbers[0] in (10, 172, 192):
-        numbers = random.sample(xrange(1, 255), 4)
+    while not octets or octets[0] in (10, 172, 192):
+        octets = random.sample(xrange(1, 255), 4)
 
-    return '.'.join(str(_) for _ in numbers)
+    return '.'.join(str(_) for _ in octets)
 
 def tamper(payload, **kwargs):
     """
-    Append a fake HTTP header 'X-Forwarded-For'
+    Append a fake HTTP header 'X-Forwarded-For' (and alike)
     """
 
     headers = kwargs.get("headers", {})
     headers["X-Forwarded-For"] = randomIP()
     headers["X-Client-Ip"] = randomIP()
     headers["X-Real-Ip"] = randomIP()
+    headers["CF-Connecting-IP"] = randomIP()
+    headers["True-Client-IP"] = randomIP()
+
+    # Reference: https://developer.chrome.com/multidevice/data-compression-for-isps#proxy-connection
+    headers["Via"] = "1.1 Chrome-Compression-Proxy"
+
+    # Reference: https://wordpress.org/support/topic/blocked-country-gaining-access-via-cloudflare/#post-9812007
+    headers["CF-IPCountry"] = random.sample(('GB', 'US', 'FR', 'AU', 'CA', 'NZ', 'BE', 'DK', 'FI', 'IE', 'AT', 'IT', 'LU', 'NL', 'NO', 'PT', 'SE', 'ES', 'CH'), 1)[0]
+
     return payload

thirdparty/beautifulsoup/beautifulsoup.py

@@ -894,7 +894,7 @@ class Tag(PageElement):
 
     def recursiveChildGenerator(self):
         if not len(self.contents):
-            raise StopIteration
+            return  # Note: https://stackoverflow.com/a/30217723 (PEP 479)
         stopNode = self._lastRecursiveChild().next
         current = self.contents[0]
         while current is not stopNode:

thirdparty/clientform/clientform.py

@@ -227,6 +227,9 @@ def unescape(data, entities, encoding=DEFAULT_ENCODING):
     if data is None or "&" not in data:
         return data
 
+    if isinstance(data, six.string_types):
+        encoding = None
+
     def replace_entities(match, entities=entities, encoding=encoding):
         ent = match.group()
         if ent[1] == "#":
@@ -234,7 +237,7 @@ def unescape(data, entities, encoding=DEFAULT_ENCODING):
 
         repl = entities.get(ent)
         if repl is not None:
-            if type(repl) != type(""):
+            if type(repl) != type("") and encoding is not None:
                 try:
                     repl = repl.encode(encoding)
                 except UnicodeError:
@@ -279,7 +282,6 @@ def get_entitydefs():
             entitydefs["&%s;" % name] = _unichr(codepoint)
     return entitydefs
 
-
 def issequence(x):
     try:
         x[0]

thirdparty/identywaf/data.json

@@ -161,10 +161,11 @@
         "barracuda": {
             "company": "Barracuda Networks",
             "name": "Barracuda",
-            "regex": "\\bbarracuda_|barra_counter_session=|when this page occurred and the event ID found at the bottom of the page",
+            "regex": "\\bbarracuda_|barra_counter_session=|when this page occurred and the event ID found at the bottom of the page|<!--(0123456789){15}",
             "signatures": [
                 "2676:RVdXum61OElCWapAYKYPk4JzWOtohM4JiUcMr2RWg1qQJbX3uhdOn9htOj+hXrAB16FcPxJPdLsXo2tKaK99n+i6c4VmkwI3FZjxtDtAeq6c36A4chS1XaTC",
-                "db27:RVdXum61OElCWapAYKYPk4JzWOtohM4JiUcMr2RWg1qQJbX3uhdOn9htOj+hXrAB16FcPxJPdLsXo2tKaK99n+i6c4RmkgI2FZjxtDtAeq6c3qA4chS1XaTC"
+                "db27:RVdXum61OElCWapAYKYPk4JzWOtohM4JiUcMr2RWg1qQJbX3uhdOn9htOj+hXrAB16FcPxJPdLsXo2tKaK99n+i6c4RmkgI2FZjxtDtAeq6c3qA4chS1XaTC",
+                "7e6b:RVdXum61OElCWapBYKYPk4JzWOtohM4JiUcMr2RXg1uQJbX3uhdOn9htOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A4chS1XaTC"
             ]
         },
         "bekchy": {
@@ -356,6 +357,16 @@
             "regex": "The (ISA Server|server) denied the specified Uniform Resource Locator \\(URL\\)",
             "signatures": []
         },
+        "ithemes": {
+            "company": "iThemes",
+            "name": "iThemes Security",
+            "regex": "",
+            "signatures": [
+                "c70f:RVZXum60OEhCWKpAYKYPkoJyWOpohM4IiUYMr2RWg1uQJLX3uhZOnthtOj+hXrAA16FcPhJOdLoXomtKaK59nui7c4RmkgI2FZjxtDtAeq+c36A4chW1XaTC",
+                "71ee:RVZXum60OEhCWKpAYKYPkoJyWOpohM4IiUYMr2RWg1qQJLX2uhZOnthtOj+hXrAA16FcPhJOdLoXomtKaK59nui7c4RmkgI2FZjxtDtAeq+c36A4chW1XaTC"
+            ],
+            "note": "Formerly Better WP Security"
+        },
         "janusec": {
             "company": "Janusec",
             "name": "Janusec Application Gateway",

thirdparty/identywaf/identYwaf.py

@@ -27,7 +27,9 @@ import sys
 import time
 import zlib
 
-if sys.version_info >= (3, 0):
+PY3 = sys.version_info >= (3, 0)
+
+if PY3:
     import http.cookiejar
     import http.client as httplib
     import urllib.request
@@ -58,7 +60,7 @@ else:
     HTTPCookieProcessor = urllib2.HTTPCookieProcessor
 
 NAME = "identYwaf"
-VERSION = "1.0.114"
+VERSION = "1.0.122"
 BANNER = r"""
                                    ` __ __ `
  ____  ___      ___  ____   ______ `|  T  T` __    __   ____  _____ 
@@ -356,7 +358,7 @@ def init():
     os.chdir(os.path.abspath(os.path.dirname(__file__)))
 
     # Reference: http://blog.mathieu-leplatre.info/python-utf-8-print-fails-when-redirecting-stdout.html
-    if IS_TTY:
+    if not PY3 and not IS_TTY:
         sys.stdout = codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
 
     print(colorize("[o] initializing handlers..."))