Adding a support for option-less URL arg

Adding autocompletion script
Update regarding #3944
2026-10-12 02:22:55 +00:00 · 2019-10-01 15:37:09 +02:00 · 2019-09-30 17:50:25 +02:00 · 2019-09-27 21:59:20 +02:00 · 2019-09-27 21:35:21 +02:00 · 2019-09-27 21:03:21 +02:00
80 changed files with 1570 additions and 1078 deletions
@@ -1,4 +1,5 @@
 language: python
+dist: trusty
 sudo: false
 git:
    depth: 1
@@ -10,4 +11,4 @@ python:
 script:
    - python -c "import sqlmap; import sqlmapapi"
    - python sqlmap.py --smoke
-    - python sqlmap.py --vuln
+    - python sqlmap.py --vuln
@@ -63,6 +63,7 @@ Translations
 * [Indonesian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-id-ID.md)
 * [Italian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-it-IT.md)
 * [Japanese](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ja-JP.md)
+* [Korean](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ko-KR.md)
 * [Polish](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-pl-PL.md)
 * [Portuguese](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-pt-BR.md)
 * [Russian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ru-RUS.md)
@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+
+<!-- http://angrytools.com/bootstrap/editor/ -->
+
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap.min.css" rel="stylesheet">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap-theme.min.css" rel="stylesheet">
+    
+    <!--[if lt IE 9]><script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script><script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script><![endif]-->
+</head>
+<body>
+    <style>
+  #wrapper { width: 100%; }
+
+  #page-wrapper {
+    padding: 0 15px;
+    min-height: 568px;
+    background-color: #fff;
+  }
+
+  @media(min-width:768px) {
+    #page-wrapper {
+      position: inherit;
+      margin: 0 0 0 250px;
+      padding: 0 30px;
+      border-left: 1px solid #e7e7e7;
+    }
+  } 
+
+  .sidebar .sidebar-nav.navbar-collapse { padding-right: 0; padding-left: 0; }
+  .sidebar .sidebar-search { padding: 15px; }
+  .sidebar ul li { border-bottom: 1px solid #e7e7e7; }
+
+  .sidebar ul li a.active { background-color: #eee; }
+
+  .sidebar .arrow { float: right;}
+  .sidebar .fa.arrow:before { content: "f104";}
+  .sidebar .active>a>.fa.arrow:before { content: "f107"; }
+  .sidebar .nav-second-level li,
+  .sidebar .nav-third-level li {
+    border-bottom: 0!important;
+  }
+
+  .sidebar .nav-second-level li a { padding-left: 37px; }
+  .sidebar .nav-third-level li a { padding-left: 52px; }
+
+  @media(min-width:768px) {
+    .sidebar {
+      z-index: 1;
+      position: absolute;
+      width: 250px;
+      margin-top: 51px;
+    }   
+  } 
+</style>
+<div id="wrapper">
+
+  <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <a class="navbar-brand" href="index.html">sqlmap</a>
+    </div>
+
+    <div class="navbar-default sidebar" role="navigation">
+      <div class="sidebar-nav navbar-collapse">
+        <ul class="nav" id="side-menu">                        
+          <li>
+            <a href="#"><i class="glyphicon glyphicon-home"></i> Options<span class="arrow"></span></a>
+            <ul class="nav nav-second-level">
+              <li><a>Target</a></li>
+              <li><a>Request</a></li>
+              <li><a>Optimization</a></li>
+              <li><a>Injection</a></li>
+              <li><a>Detection</a></li>
+              <li><a>Techniques</a></li>
+              <li><a>Fingerprint</a></li>
+              <li><a>Enumeration</a></li>
+              <li><a>Brute force</a></li>
+              <li><a>User-defined function injection</a></li>
+              <li><a>File system access</a></li>
+              <li><a>Operating system access</a></li>
+              <li><a>Windows registry access</a></li>
+              <li><a>General</a></li>
+              <li><a>Miscellaneous</a></li>
+            </ul> 
+          </li> 
+        </ul>
+      </div>
+    </div>             
+  </nav>
+
+  <div id="page-wrapper"> 
+    <div class="row">
+      <h4>DEMO</h4>
+    </div> 
+  </div>
+</div>
+<script>
+  /*
+ * metismenu - v1.0.3
+ * Easy menu jQuery plugin for Twitter Bootstrap 3
+ * https://github.com/onokumus/metisMenu
+ *
+ * Made by Osman Nuri Okumuş
+ * Under MIT License
+*/
+  !function(a,b,c){function d(b,c){this.element=b,this.settings=a.extend({},f,c),this._defaults=f,this._name=e,this.init()}var e="metisMenu",f={toggle:!0};d.prototype={init:function(){var b=a(this.element),c=this.settings.toggle;this.isIE()<=9?(b.find("li.active").has("ul").children("ul").collapse("show"),b.find("li").not(".active").has("ul").children("ul").collapse("hide")):(b.find("li.active").has("ul").children("ul").addClass("collapse in"),b.find("li").not(".active").has("ul").children("ul").addClass("collapse")),b.find("li").has("ul").children("a").on("click",function(b){b.preventDefault(),a(this).parent("li").toggleClass("active").children("ul").collapse("toggle"),c&&a(this).parent("li").siblings().removeClass("active").children("ul.in").collapse("hide")})},isIE:function(){for(var a,b=3,d=c.createElement("div"),e=d.getElementsByTagName("i");d.innerHTML="<!--[if gt IE "+ ++b+"]><i></i><![endif]-->",e[0];)return b>4?b:a}},a.fn[e]=function(b){return this.each(function(){a.data(this,"plugin_"+e)||a.data(this,"plugin_"+e,new d(this,b))})}}(jQuery,window,document);
+
+  $(function() {
+
+    $('#side-menu').metisMenu();
+
+  });
+
+  //Loads the correct sidebar on window load,
+  //collapses the sidebar on window resize.
+  // Sets the min-height of #page-wrapper to window size
+  $(function() {
+    $(window).bind("load resize", function() {
+      topOffset = 50;
+      width = (this.window.innerWidth > 0) ? this.window.innerWidth : this.screen.width;
+      if (width < 768) {
+        $('div.navbar-collapse').addClass('collapse')
+        topOffset = 100; // 2-row-menu
+      } else {
+        $('div.navbar-collapse').removeClass('collapse')
+      }
+
+      height = (this.window.innerHeight > 0) ? this.window.innerHeight : this.screen.height;
+      height = height - topOffset;
+      if (height < 1) height = 1;
+      if (height > topOffset) {
+        $("#page-wrapper").css("min-height", (height) + "px");
+      }
+    })
+  });
+</script>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/js/bootstrap.min.js"></script>
+</body>
+</html>
@@ -192,6 +192,17 @@
 /var/log/mysqld.log
 /var/www/index.php

+# Reference: https://github.com/sqlmapproject/sqlmap/blob/master/lib/core/settings.py#L809-L810
+
+/var/www/index.php
+/usr/local/apache/index.php
+/usr/local/apache2/index.php
+/usr/local/www/apache22/index.php
+/usr/local/www/apache24/index.php
+/usr/local/httpd/index.php
+/var/www/nginx-default/index.php
+/srv/www/index.php
+
 # Reference: https://www.gracefulsecurity.com/path-traversal-cheat-sheet-linux

 /etc/passwd
@@ -1,5 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>

+<!--
+     References:
+     * https://en.wikipedia.org/wiki/Debian_version_history
+-->
+
 <root>
    <regexp value="^([\d\.\-]+)[\-\_\ ].*">
        <info dbms_version="1"/>
@@ -36,19 +41,27 @@
    </regexp>

    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+wheezy">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="7.0" codename="wheezy"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="7" codename="wheezy"/>
    </regexp>

    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+jessie">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="8.0" codename="jessie"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="8" codename="jessie"/>
    </regexp>

    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+stretch">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="9.0" codename="stretch"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="9" codename="stretch"/>
    </regexp>

    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+buster">
-        <info dbms_version="1" type="Linux" distrib="Debian" release="10.0" codename="buster"/>
+        <info dbms_version="1" type="Linux" distrib="Debian" release="10" codename="buster"/>
+    </regexp>
+
+    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+bullseye">
+        <info dbms_version="1" type="Linux" distrib="Debian" release="11" codename="bullseye"/>
+    </regexp>
+
+    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+bookworm">
+        <info dbms_version="1" type="Linux" distrib="Debian" release="12" codename="bookworm"/>
    </regexp>

    <regexp value="^([\d\.]+)[\-\_]Debian[\-\_][\d\.]+(sid|unstable)">
@@ -0,0 +1,50 @@
+# sqlmap
+
+[![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/[email protected])](https://twitter.com/sqlmap)
+
+sqlmap은 SQL 인젝션 결함 탐지 및 활용, 데이터베이스 서버 장악 프로세스를 자동화 하는 오픈소스 침투 테스팅 도구입니다. 최고의 침투 테스터, 데이터베이스 핑거프린팅 부터 데이터베이스 데이터 읽기, 대역 외 연결을 통한 기반 파일 시스템 접근 및 명령어 실행에 걸치는 광범위한 스위치들을 위한 강력한 탐지 엔진과 다수의 편리한 기능이 탑재되어 있습니다.
+
+스크린샷
+----
+
+![Screenshot](https://raw.github.com/wiki/sqlmapproject/sqlmap/images/sqlmap_screenshot.png)
+
+또는, wiki에 나와있는 몇몇 기능을 보여주는 [스크린샷 모음](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) 을 방문하실 수 있습니다.
+
+설치
+----
+
+[여기](https://github.com/sqlmapproject/sqlmap/tarball/master)를 클릭하여 최신 버전의 tarball 파일, 또는 [여기](https://github.com/sqlmapproject/sqlmap/zipball/master)를 클릭하여 최신 zipball 파일을 다운받으실 수 있습니다.
+
+가장 선호되는 방법으로, [Git](https://github.com/sqlmapproject/sqlmap) 저장소를 복제하여 sqlmap을 다운로드 할 수 있습니다:
+
+    git clone --depth 1 https://github.com/sqlmapproject/sqlmap.git sqlmap-dev
+
+sqlmap은 [Python](http://www.python.org/download/) 버전 **2.6**, **2.7** 그리고 **3.x** 을 통해 모든 플랫폼 위에서 사용 가능합니다.
+
+사용법
+----
+
+기본 옵션과 스위치 목록을 보려면 다음 명령어를 사용하세요:
+
+    python sqlmap.py -h
+
+전체 옵션과 스위치 목록을 보려면 다음 명령어를 사용하세요:
+
+    python sqlmap.py -hh
+
+[여기](https://asciinema.org/a/46601)를 통해 사용 샘플들을 확인할 수 있습니다.
+sqlmap의 능력, 지원되는 기능과 모든 옵션과 스위치들의 목록을 예제와 함께 보려면, [사용자 매뉴얼](https://github.com/sqlmapproject/sqlmap/wiki/Usage)을 참고하시길 권장드립니다.
+
+링크
+----
+
+* 홈페이지: http://sqlmap.org
+* 다운로드: [.tar.gz](https://github.com/sqlmapproject/sqlmap/tarball/master) or [.zip](https://github.com/sqlmapproject/sqlmap/zipball/master)
+* RSS 피드 커밋: https://github.com/sqlmapproject/sqlmap/commits/master.atom
+* Issue tracker: https://github.com/sqlmapproject/sqlmap/issues
+* 사용자 매뉴얼: https://github.com/sqlmapproject/sqlmap/wiki
+* 자주 묻는 질문 (FAQ): https://github.com/sqlmapproject/sqlmap/wiki/FAQ
+* 트위터: [@sqlmap](https://twitter.com/sqlmap)
+* 시연 영상: [http://www.youtube.com/user/inquisb/videos](http://www.youtube.com/user/inquisb/videos)
+* 스크린샷: https://github.com/sqlmapproject/sqlmap/wiki/Screenshots
@@ -15,11 +15,13 @@ BEEP_WAV_FILENAME = os.path.join(os.path.dirname(__file__), "beep.wav")

 def beep():
    try:
-        if sys.platform == "nt":
+        if sys.platform.startswith("win"):
            _win_wav_play(BEEP_WAV_FILENAME)
-        elif sys.platform == "darwin":
+        elif sys.platform.startswith("darwin"):
            _mac_beep()
-        elif sys.platform.startswith("linux"):
+        elif sys.platform.startswith("cygwin"):
+            _cygwin_beep(BEEP_WAV_FILENAME)
+        elif any(sys.platform.startswith(_) for _ in ("linux", "freebsd")):
            _linux_wav_play(BEEP_WAV_FILENAME)
        else:
            _speaker_beep()
@@ -34,6 +36,10 @@ def _speaker_beep():
    except IOError:
        pass

+# Reference: https://lists.gnu.org/archive/html/emacs-devel/2014-09/msg00815.html
+def _cygwin_beep(filename):
+    os.system("play-sound-file '%s' 2>/dev/null" % filename)
+
 def _mac_beep():
    import Carbon.Snd
    Carbon.Snd.SysBeep(1)
@@ -57,7 +63,10 @@ def _linux_wav_play(filename):
    class struct_pa_sample_spec(ctypes.Structure):
        _fields_ = [("format", ctypes.c_int), ("rate", ctypes.c_uint32), ("channels", ctypes.c_uint8)]

-    pa = ctypes.cdll.LoadLibrary("libpulse-simple.so.0")
+    try:
+        pa = ctypes.cdll.LoadLibrary("libpulse-simple.so.0")
+    except OSError:
+        return

    wave_file = wave.open(filename, "rb")

@@ -1,17 +0,0 @@
-To use safe2bin.py you need to pass it the original file,
-and optionally the output file name.
-
-Example:
-
-$ python ./safe2bin.py -i output.txt -o output.txt.bin
-
-This will create an binary decoded file output.txt.bin. For example, 
-if the content of output.txt is: "\ttest\t\x32\x33\x34\nnewline" it will 
-be decoded to: "	test	234
-newline"
-
-If you skip the output file name, general rule is that the binary
-file names are suffixed with the string '.bin'. So, that means that 
-the upper example can also be written in the following form:
-
-$ python ./safe2bin.py -i output.txt
@@ -1,8 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
-See the file 'LICENSE' for copying permission
-"""
-
-pass
@@ -0,0 +1,9 @@
+#/usr/bin/env bash
+
+# source ./extra/shutils/autocompletion.sh
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+WORDLIST=`python "$DIR/../../sqlmap.py" -hh | grep -Eo '\s\--?\w[^ =,]*' | grep -vF '..' | paste -sd "" -`
+
+complete -W "$WORDLIST" sqlmap
+complete -W "$WORDLIST" ./sqlmap.py
@@ -7,7 +7,7 @@

 export SQLMAP_DREI=1
 #for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done
-for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i; done
+for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i | sed 's/Compiling/Checking/g'; done
 unset SQLMAP_DREI
 source `dirname "$0"`"/junk.sh"

@@ -1518,7 +1518,7 @@ def checkConnection(suppressOutput=False):
            warnMsg += "which could interfere with the results of the tests"
            logger.warn(warnMsg)
        elif wasLastResponseHTTPError():
-            if getLastRequestHTTPError() != conf.ignoreCode:
+            if getLastRequestHTTPError() not in (conf.ignoreCode or []):
                warnMsg = "the web server responded with an HTTP error code (%d) " % getLastRequestHTTPError()
                warnMsg += "which could interfere with the results of the tests"
                logger.warn(warnMsg)
@@ -422,6 +422,15 @@ def start():
            if not checkConnection(suppressOutput=conf.forms) or not checkString() or not checkRegexp():
                continue

+            if conf.rParam and kb.originalPage:
+                kb.randomPool = dict([_ for _ in kb.randomPool.items() if isinstance(_[1], list)])
+
+                for match in re.finditer(r"(?si)<select[^>]+\bname\s*=\s*[\"']([^\"']+)(.+?)</select>", kb.originalPage):
+                    name, _ = match.groups()
+                    options = tuple(re.findall(r"<option[^>]+\bvalue\s*=\s*[\"']([^\"']+)", _))
+                    if options:
+                        kb.randomPool[name] = options
+
            checkWaf()

            if conf.nullConnection:
@@ -5,7 +5,6 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """

-import base64
 import re

 from lib.core.common import Backend
@@ -13,6 +12,7 @@ from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
 from lib.core.common import getSQLSnippet
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import isDBMSVersionAtLeast
 from lib.core.common import isNumber
 from lib.core.common import isTechniqueAvailable
@@ -26,6 +26,7 @@ from lib.core.common import unArrayizeValue
 from lib.core.common import urlencode
 from lib.core.common import zeroDepthSearch
 from lib.core.compat import xrange
+from lib.core.convert import encodeBase64
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -49,7 +50,9 @@ from lib.core.settings import PAYLOAD_DELIMITER
 from lib.core.settings import REPLACEMENT_MARKER
 from lib.core.settings import SINGLE_QUOTE_MARKER
 from lib.core.settings import SLEEP_TIME_MARKER
+from lib.core.settings import UNICODE_ENCODING
 from lib.core.unescaper import unescaper
+from thirdparty import six

 class Agent(object):
    """
@@ -91,7 +94,7 @@ class Agent(object):
        if kb.forceWhere:
            where = kb.forceWhere
        elif where is None and isTechniqueAvailable(getTechnique()):
-            where = kb.injection.data[getTechnique()].where
+            where = getTechniqueData().where

        if kb.injection.place is not None:
            place = kb.injection.place
@@ -168,8 +171,8 @@ class Agent(object):

        if re.sub(r" \(.+", "", parameter) in conf.base64Parameter:
            # TODO: support for POST_HINT
-            newValue = base64.b64encode(newValue)
-            origValue = base64.b64encode(origValue)
+            newValue = encodeBase64(newValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
+            origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)

        if place in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
            _ = "%s%s" % (origValue, kb.customInjectionMark)
@@ -236,7 +239,7 @@ class Agent(object):
        query = None

        if where is None and getTechnique() is not None and getTechnique() in kb.injection.data:
-            where = kb.injection.data[getTechnique()].where
+            where = getTechniqueData().where

        # If we are replacing (<where>) the parameter original value with
        # our payload do not prepend with the prefix
@@ -284,8 +287,8 @@ class Agent(object):
        suffix = kb.injection.suffix if kb.injection and suffix is None else suffix

        if getTechnique() is not None and getTechnique() in kb.injection.data:
-            where = kb.injection.data[getTechnique()].where if where is None else where
-            comment = kb.injection.data[getTechnique()].comment if comment is None else comment
+            where = getTechniqueData().where if where is None else where
+            comment = getTechniqueData().comment if comment is None else comment

        if Backend.getIdentifiedDbms() == DBMS.ACCESS and any((comment or "").startswith(_) for _ in ("--", "[GENERIC_SQL_COMMENT]")):
            comment = queries[DBMS.ACCESS].comment.query
@@ -307,7 +310,7 @@ class Agent(object):
        return re.sub(r";\W*;", ";", expression) if trimEmpty else expression

    def cleanupPayload(self, payload, origValue=None):
-        if payload is None:
+        if not isinstance(payload, six.string_types):
            return

        replacements = {
@@ -332,6 +335,7 @@ class Agent(object):

        if origValue is not None:
            origValue = getUnicode(origValue)
+
            if "[ORIGVALUE]" in payload:
                payload = getUnicode(payload).replace("[ORIGVALUE]", origValue if origValue.isdigit() else unescaper.escape("'%s'" % origValue))
            if "[ORIGINAL]" in payload:
@@ -350,6 +354,7 @@ class Agent(object):
                    inferenceQuery = inference.query

                payload = payload.replace(INFERENCE_MARKER, inferenceQuery)
+
            elif not kb.testMode:
                errMsg = "invalid usage of inference payload without "
                errMsg += "knowledge of underlying DBMS"
@@ -392,7 +397,7 @@ class Agent(object):
        if "hex" in rootQuery:
            hexField = rootQuery.hex.query % field
        else:
-            warnMsg = "switch '--hex' is currently not supported on DBMS %s" % Backend.getIdentifiedDbms()
+            warnMsg = "switch '--hex' is currently not supported on DBMS '%s'" % Backend.getIdentifiedDbms()
            singleTimeWarnMessage(warnMsg)

        return hexField
@@ -650,7 +655,7 @@ class Agent(object):
                concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'||" % kb.chars.start, 1)
                _ = unArrayizeValue(zeroDepthSearch(concatenatedQuery, " FROM "))
                concatenatedQuery = "%s||'%s'%s" % (concatenatedQuery[:_], kb.chars.stop, concatenatedQuery[_:])
-                concatenatedQuery = re.sub(r"('%s'\|\|)(.+)(%s)" % (kb.chars.start, re.escape(castedFields)), r"\g<2>\g<1>\g<3>", concatenatedQuery)
+                concatenatedQuery = re.sub(r"('%s'\|\|)(.+?)(%s)" % (kb.chars.start, re.escape(castedFields)), r"\g<2>\g<1>\g<3>", concatenatedQuery)
            elif fieldsSelect:
                concatenatedQuery = concatenatedQuery.replace("SELECT ", "'%s'||" % kb.chars.start, 1)
                concatenatedQuery += "||'%s'" % kb.chars.stop
@@ -1006,7 +1011,7 @@ class Agent(object):
                        limitedQuery = "%s WHERE %s " % (limitedQuery, self.nullAndCastField(uniqueField or field))

                    limitedQuery += "NOT IN (%s" % (limitStr % num)
-                    limitedQuery += "%s %s ORDER BY %s) ORDER BY %s" % (self.nullAndCastField(uniqueField or field), fromFrom, uniqueField or "1", uniqueField or "1")
+                    limitedQuery += "%s %s ORDER BY %s) ORDER BY %s" % (self.nullAndCastField(uniqueField or field), fromFrom, uniqueField or '1', uniqueField or '1')
                else:
                    match = re.search(r" ORDER BY (\w+)\Z", query)
                    field = match.group(1) if match else field
@@ -1080,7 +1085,7 @@ class Agent(object):
        Removes payload delimiters from inside the input string
        """

-        return value.replace(PAYLOAD_DELIMITER, '') if value else value
+        return value.replace(PAYLOAD_DELIMITER, "") if value else value

    def extractPayload(self, value):
        """
@@ -51,6 +51,11 @@ class Cache(object):
 class BigArray(list):
    """
    List-like class used for storing large amounts of data (disk cached)
+
+    >>> _ = BigArray(xrange(100000))
+    >>> _[20] = 0
+    >>> _[100]
+    100
    """

    def __init__(self, items=None):
@@ -46,7 +46,6 @@ from xml.sax import SAXParseException

 from extra.beep.beep import beep
 from extra.cloak.cloak import decloak
-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.bigarray import BigArray
 from lib.core.compat import cmp
 from lib.core.compat import round
@@ -180,6 +179,7 @@ from lib.core.settings import VERSION_STRING
 from lib.core.settings import ZIP_HEADER
 from lib.core.settings import WEBSCARAB_SPLITTER
 from lib.core.threads import getCurrentThreadData
+from lib.utils.safe2bin import safecharencode
 from lib.utils.sqlalchemy import _sqlalchemy
 from thirdparty import six
 from thirdparty.clientform.clientform import ParseResponse
@@ -623,8 +623,8 @@ def paramToDict(place, parameters=None):
                if parameter in (conf.base64Parameter or []):
                    try:
                        oldValue = value
-                        value = decodeBase64(value, binary=False)
-                        parameters = re.sub(r"\b%s\b" % re.escape(oldValue), value, parameters)
+                        value = decodeBase64(value, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
+                        parameters = re.sub(r"\b%s(\b|\Z)" % re.escape(oldValue), value, parameters)
                    except:
                        errMsg = "parameter '%s' does not contain " % parameter
                        errMsg += "valid Base64 encoded value ('%s')" % value
@@ -701,7 +701,7 @@ def paramToDict(place, parameters=None):
                            message += "has boundaries. Do you want to inject inside? ('%s') [y/N] " % getUnicode(_)

                            if readInput(message, default='N', boolean=True):
-                                testableParameters[parameter] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), (r"\g<1>%s" % re.sub(regex, r"\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter])).replace("\\", r"\\"), parameters)
+                                testableParameters[parameter] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), (r"\g<1>%s" % re.sub(regex, r"\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter].replace("\\", r"\\"))), parameters)
                            break

    if conf.testParameter:
@@ -1137,7 +1137,7 @@ def getTechnique():
    Thread-safe getting of currently used technique
    """

-    return getCurrentThreadData().technique or kb.technique
+    return getCurrentThreadData().technique or kb.get("technique")

 def randomRange(start=0, stop=1000, seed=None):
    """
@@ -1285,36 +1285,43 @@ def banner():
    """

    if not any(_ in sys.argv for _ in ("--version", "--api")) and not conf.get("disableBanner"):
-        _ = BANNER
+        result = BANNER

        if not IS_TTY or "--disable-coloring" in sys.argv:
-            _ = clearColors(_)
+            result = clearColors(result)
        elif IS_WIN:
            coloramainit()

-        dataToStdout(_, forceOutput=True)
+        dataToStdout(result, forceOutput=True)

 def parsePasswordHash(password):
    """
    In case of Microsoft SQL Server password hash value is expanded to its components
+
+    >>> pushValue(kb.forcedDbms)
+    >>> kb.forcedDbms = DBMS.MSSQL
+    >>> "salt: 4086ceb6" in parsePasswordHash("0x01004086ceb60c90646a8ab9889fe3ed8e5c150b5460ece8425a")
+    True
+    >>> kb.forcedDbms = popValue()
    """

    blank = " " * 8

-    if not password or password == " ":
-        password = NULL
+    if isNoneValue(password) or password == " ":
+        retVal = NULL
+    else:
+        retVal = password

-    if Backend.isDbms(DBMS.MSSQL) and password != NULL and isHexEncodedString(password):
-        hexPassword = password
-        password = "%s\n" % hexPassword
-        password += "%sheader: %s\n" % (blank, hexPassword[:6])
-        password += "%ssalt: %s\n" % (blank, hexPassword[6:14])
-        password += "%smixedcase: %s\n" % (blank, hexPassword[14:54])
+    if Backend.isDbms(DBMS.MSSQL) and retVal != NULL and isHexEncodedString(password):
+        retVal = "%s\n" % password
+        retVal += "%sheader: %s\n" % (blank, password[:6])
+        retVal += "%ssalt: %s\n" % (blank, password[6:14])
+        retVal += "%smixedcase: %s\n" % (blank, password[14:54])

-        if not Backend.isVersionWithin(("2005", "2008")):
-            password += "%suppercase: %s" % (blank, hexPassword[54:])
+        if password[54:]:
+            retVal += "%suppercase: %s" % (blank, password[54:])

-    return password
+    return retVal

 def cleanQuery(query):
    """
@@ -3222,7 +3229,7 @@ def getTechniqueData(technique=None):
    Returns injection data for technique specified
    """

-    return kb.injection.data.get(technique)
+    return kb.injection.data.get(technique if technique is not None else getTechnique())

 def isTechniqueAvailable(technique):
    """
@@ -3549,7 +3556,7 @@ def openFile(filename, mode='r', encoding=UNICODE_ENCODING, errors="reversible",
        except IOError:
            errMsg = "there has been a file opening error for filename '%s'. " % filename
            errMsg += "Please check %s permissions on a file " % ("write" if mode and ('w' in mode or 'a' in mode or '+' in mode) else "read")
-            errMsg += "and that it's not locked by another process."
+            errMsg += "and that it's not locked by another process"
            raise SqlmapSystemException(errMsg)

 def decodeIntToUnicode(value):
@@ -3575,7 +3582,7 @@ def decodeIntToUnicode(value):
                    # Note: https://github.com/sqlmapproject/sqlmap/issues/1531
                    retVal = getUnicode(raw, conf.encoding or UNICODE_ENCODING)
                elif Backend.isDbms(DBMS.MSSQL):
-                    retVal = getUnicode(raw, "UTF-16-BE")
+                    retVal = getUnicode(raw, "UTF-16-BE")   # References: https://docs.microsoft.com/en-us/sql/relational-databases/collations/collation-and-unicode-support?view=sql-server-2017 and https://stackoverflow.com/a/14488478
                elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE):
                    retVal = _unichr(value)
                else:
@@ -3662,7 +3669,7 @@ def getLatestRevision():
    """

    retVal = None
-    req = _urllib.request.Request(url="https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/lib/core/settings.py")
+    req = _urllib.request.Request(url="https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/lib/core/settings.py", headers={HTTP_HEADER.USER_AGENT: fetchRandomAgent()})

    try:
        content = getUnicode(_urllib.request.urlopen(req).read())
@@ -3762,7 +3769,7 @@ def createGithubIssue(errMsg, excMsg):
            logger.info(infoMsg)

            try:
-                with open(paths.GITHUB_HISTORY, "a+b") as f:
+                with openFile(paths.GITHUB_HISTORY, "a+b") as f:
                    f.write("%s\n" % key)
            except:
                pass
@@ -3794,7 +3801,7 @@ def maskSensitiveData(msg):
            retVal = retVal.replace(value, '*' * len(value))

    # Just in case (for problematic parameters regarding user encoding)
-    for match in re.finditer(r"(?i)[ -]-(u|url|data|cookie|auth-\w+|proxy)( |=)(.*?)(?= -?-[a-z]|\Z)", retVal):
+    for match in re.finditer(r"(?i)[ -]-(u|url|data|cookie|auth-\w+|proxy|host|referer|headers?|H)( |=)(.*?)(?= -?-[a-z]|\Z)", retVal):
        retVal = retVal.replace(match.group(3), '*' * len(match.group(3)))

    # Fail-safe substitutions
@@ -4868,6 +4875,8 @@ def zeroDepthSearch(expression, value):

    >>> _ = "SELECT (SELECT id FROM users WHERE 2>1) AS result FROM DUAL"; _[zeroDepthSearch(_, "FROM")[0]:]
    'FROM DUAL'
+    >>> _ = "a(b; c),d;e"; _[zeroDepthSearch(_, "[;, ]")[0]:]
+    ',d;e'
    """

    retVal = []
@@ -4878,8 +4887,12 @@ def zeroDepthSearch(expression, value):
            depth += 1
        elif expression[index] == ')':
            depth -= 1
-        elif depth == 0 and expression[index:index + len(value)] == value:
-            retVal.append(index)
+        elif depth == 0:
+            if value.startswith('[') and value.endswith(']'):
+                if re.search(value, expression[index:index + 1]):
+                    retVal.append(index)
+            elif expression[index:index + len(value)] == value:
+                retVal.append(index)

    return retVal

@@ -4988,7 +5001,7 @@ def parseRequestFile(reqFile, checkParams=True):
            else:
                scheme, port = None, None

-            if not re.search(r"^[\n]*(%s).*?\sHTTP\/" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), request, re.I | re.M):
+            if "HTTP/" not in request:
                continue

            if re.search(r"^[\n]*%s.*?\.(%s)\sHTTP\/" % (HTTPMETHOD.GET, "|".join(CRAWL_EXCLUDE_EXTENSIONS)), request, re.I | re.M):
@@ -5013,7 +5026,7 @@ def parseRequestFile(reqFile, checkParams=True):

                newline = "\r\n" if line.endswith('\r') else '\n'
                line = line.strip('\r')
-                match = re.search(r"\A(%s) (.+) HTTP/[\d.]+\Z" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), line) if not method else None
+                match = re.search(r"\A([A-Z]+) (.+) HTTP/[\d.]+\Z", line) if not method else None

                if len(line.strip()) == 0 and method and method != HTTPMETHOD.GET and data is None:
                    data = ""
@@ -13,6 +13,7 @@ import math
 import os
 import random
 import sys
+import time
 import uuid

 class WichmannHill(random.Random):
@@ -40,7 +41,6 @@ class WichmannHill(random.Random):
            try:
                a = int(binascii.hexlify(os.urandom(16)), 16)
            except NotImplementedError:
-                import time
                a = int(time.time() * 256)  # use fractional seconds

        if not isinstance(a, int):
@@ -132,7 +132,6 @@ class WichmannHill(random.Random):
            raise ValueError('seeds must be in range(0, 256)')
        if 0 == x == y == z:
            # Initialize from current time
-            import time
            t = int(time.time() * 256)
            t = int((t & 0xffffff) ^ (t >> 24))
            t, x = divmod(t, 256)
@@ -204,6 +203,7 @@ def round(x, d=0):
    else:
        return float(math.ceil((x * p) - 0.5)) / p

+# Reference: https://code.activestate.com/recipes/576653-convert-a-cmp-function-to-a-key-function/
 def cmp_to_key(mycmp):
    """Convert a cmp= function into a key= function"""
    class K(object):
@@ -164,8 +164,13 @@ def encodeHex(value, binary=True):
    True
    >>> encodeHex("123", binary=False)
    '313233'
+    >>> encodeHex(b"123"[0]) == b"31"
+    True
    """

+    if isinstance(value, int):
+        value = six.unichr(value)
+
    if isinstance(value, six.text_type):
        value = value.encode(UNICODE_ENCODING)

@@ -179,7 +184,7 @@ def encodeHex(value, binary=True):

    return retVal

-def decodeBase64(value, binary=True):
+def decodeBase64(value, binary=True, encoding=None):
    """
    Returns a decoded representation of provided Base64 value

@@ -192,11 +197,11 @@ def decodeBase64(value, binary=True):
    retVal = base64.b64decode(value)

    if not binary:
-        retVal = getText(retVal)
+        retVal = getText(retVal, encoding)

    return retVal

-def encodeBase64(value, binary=True):
+def encodeBase64(value, binary=True, encoding=None):
    """
    Returns a decoded representation of provided Base64 value

@@ -207,12 +212,12 @@ def encodeBase64(value, binary=True):
    """

    if isinstance(value, six.text_type):
-        value = value.encode(UNICODE_ENCODING)
+        value = value.encode(encoding or UNICODE_ENCODING)

    retVal = base64.b64encode(value)

    if not binary:
-        retVal = getText(retVal)
+        retVal = getText(retVal, encoding)

    return retVal

@@ -300,7 +305,7 @@ def getUnicode(value, encoding=None, noneToNull=False):
        except UnicodeDecodeError:
            return six.text_type(str(value), errors="ignore")  # encoding ignored for non-basestring instances

-def getText(value):
+def getText(value, encoding=None):
    """
    Returns textual value of a given value (Note: not necessary Unicode on Python2)

@@ -313,7 +318,7 @@ def getText(value):
    retVal = value

    if isinstance(value, six.binary_type):
-        retVal = getUnicode(value)
+        retVal = getUnicode(value, encoding)

    if six.PY2:
        try:
@@ -14,9 +14,11 @@ from lib.core.settings import MAX_CACHE_ITEMS
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData

-_lock = threading.Lock()
+_cache = {}
+_cache_lock = threading.Lock()
+_method_locks = {}

-def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
+def cachedmethod(f):
    """
    Method with a cached content

@@ -33,22 +35,24 @@ def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
    Reference: http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/
    """

+    _cache[f] = LRUDict(capacity=MAX_CACHE_ITEMS)
+
    @functools.wraps(f)
-    def _(*args, **kwargs):
+    def _f(*args, **kwargs):
        key = int(hashlib.md5("|".join(str(_) for _ in (f, args, kwargs)).encode(UNICODE_ENCODING)).hexdigest(), 16) & 0x7fffffffffffffff

        try:
-            with _lock:
-                result = cache[key]
+            with _cache_lock:
+                result = _cache[f][key]
        except KeyError:
            result = f(*args, **kwargs)

-            with _lock:
-                cache[key] = result
+            with _cache_lock:
+                _cache[f][key] = result

        return result

-    return _
+    return _f

 def stackedmethod(f):
    """
@@ -76,3 +80,16 @@ def stackedmethod(f):
        return result

    return _
+
+def lockedmethod(f):
+    @functools.wraps(f)
+    def _(*args, **kwargs):
+        if f not in _method_locks:
+            _method_locks[f] = threading.RLock()
+
+        with _method_locks[f]:
+            result = f(*args, **kwargs)
+
+        return result
+
+    return _
@@ -20,6 +20,7 @@ _defaults = {
    "level": 1,
    "risk": 1,
    "dumpFormat": "CSV",
+    "tablePrefix": "sqlmap",
    "technique": "BEUSTQ",
    "torType": "SOCKS5",
 }
@@ -304,7 +304,7 @@ DUMP_DATA_PREPROCESS = {

 DEFAULT_DOC_ROOTS = {
    OS.WINDOWS: ("C:/xampp/htdocs/", "C:/wamp/www/", "C:/Inetpub/wwwroot/"),
-    OS.LINUX: ("/var/www/", "/var/www/html", "/usr/local/apache2/htdocs", "/var/www/nginx-default", "/srv/www")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
+    OS.LINUX: ("/var/www/", "/var/www/html", "/var/www/htdocs", "/usr/local/apache2/htdocs", "/usr/local/www/data", "/var/apache2/htdocs", "/var/www/nginx-default", "/srv/www/htdocs")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
 }

 PART_RUN_CONTENT_TYPES = {
@@ -334,3 +334,260 @@ PART_RUN_CONTENT_TYPES = {
    "osCmd": CONTENT_TYPE.OS_CMD,
    "regRead": CONTENT_TYPE.REG_READ
 }
+
+# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html
+
+HTML_ENTITIES = {
+    "quot": 34,
+    "amp": 38,
+    "lt": 60,
+    "gt": 62,
+    "nbsp": 160,
+    "iexcl": 161,
+    "cent": 162,
+    "pound": 163,
+    "curren": 164,
+    "yen": 165,
+    "brvbar": 166,
+    "sect": 167,
+    "uml": 168,
+    "copy": 169,
+    "ordf": 170,
+    "laquo": 171,
+    "not": 172,
+    "shy": 173,
+    "reg": 174,
+    "macr": 175,
+    "deg": 176,
+    "plusmn": 177,
+    "sup2": 178,
+    "sup3": 179,
+    "acute": 180,
+    "micro": 181,
+    "para": 182,
+    "middot": 183,
+    "cedil": 184,
+    "sup1": 185,
+    "ordm": 186,
+    "raquo": 187,
+    "frac14": 188,
+    "frac12": 189,
+    "frac34": 190,
+    "iquest": 191,
+    "Agrave": 192,
+    "Aacute": 193,
+    "Acirc": 194,
+    "Atilde": 195,
+    "Auml": 196,
+    "Aring": 197,
+    "AElig": 198,
+    "Ccedil": 199,
+    "Egrave": 200,
+    "Eacute": 201,
+    "Ecirc": 202,
+    "Euml": 203,
+    "Igrave": 204,
+    "Iacute": 205,
+    "Icirc": 206,
+    "Iuml": 207,
+    "ETH": 208,
+    "Ntilde": 209,
+    "Ograve": 210,
+    "Oacute": 211,
+    "Ocirc": 212,
+    "Otilde": 213,
+    "Ouml": 214,
+    "times": 215,
+    "Oslash": 216,
+    "Ugrave": 217,
+    "Uacute": 218,
+    "Ucirc": 219,
+    "Uuml": 220,
+    "Yacute": 221,
+    "THORN": 222,
+    "szlig": 223,
+    "agrave": 224,
+    "aacute": 225,
+    "acirc": 226,
+    "atilde": 227,
+    "auml": 228,
+    "aring": 229,
+    "aelig": 230,
+    "ccedil": 231,
+    "egrave": 232,
+    "eacute": 233,
+    "ecirc": 234,
+    "euml": 235,
+    "igrave": 236,
+    "iacute": 237,
+    "icirc": 238,
+    "iuml": 239,
+    "eth": 240,
+    "ntilde": 241,
+    "ograve": 242,
+    "oacute": 243,
+    "ocirc": 244,
+    "otilde": 245,
+    "ouml": 246,
+    "divide": 247,
+    "oslash": 248,
+    "ugrave": 249,
+    "uacute": 250,
+    "ucirc": 251,
+    "uuml": 252,
+    "yacute": 253,
+    "thorn": 254,
+    "yuml": 255,
+    "OElig": 338,
+    "oelig": 339,
+    "Scaron": 352,
+    "fnof": 402,
+    "scaron": 353,
+    "Yuml": 376,
+    "circ": 710,
+    "tilde": 732,
+    "Alpha": 913,
+    "Beta": 914,
+    "Gamma": 915,
+    "Delta": 916,
+    "Epsilon": 917,
+    "Zeta": 918,
+    "Eta": 919,
+    "Theta": 920,
+    "Iota": 921,
+    "Kappa": 922,
+    "Lambda": 923,
+    "Mu": 924,
+    "Nu": 925,
+    "Xi": 926,
+    "Omicron": 927,
+    "Pi": 928,
+    "Rho": 929,
+    "Sigma": 931,
+    "Tau": 932,
+    "Upsilon": 933,
+    "Phi": 934,
+    "Chi": 935,
+    "Psi": 936,
+    "Omega": 937,
+    "alpha": 945,
+    "beta": 946,
+    "gamma": 947,
+    "delta": 948,
+    "epsilon": 949,
+    "zeta": 950,
+    "eta": 951,
+    "theta": 952,
+    "iota": 953,
+    "kappa": 954,
+    "lambda": 955,
+    "mu": 956,
+    "nu": 957,
+    "xi": 958,
+    "omicron": 959,
+    "pi": 960,
+    "rho": 961,
+    "sigmaf": 962,
+    "sigma": 963,
+    "tau": 964,
+    "upsilon": 965,
+    "phi": 966,
+    "chi": 967,
+    "psi": 968,
+    "omega": 969,
+    "thetasym": 977,
+    "upsih": 978,
+    "piv": 982,
+    "bull": 8226,
+    "hellip": 8230,
+    "prime": 8242,
+    "Prime": 8243,
+    "oline": 8254,
+    "frasl": 8260,
+    "ensp": 8194,
+    "emsp": 8195,
+    "thinsp": 8201,
+    "zwnj": 8204,
+    "zwj": 8205,
+    "lrm": 8206,
+    "rlm": 8207,
+    "ndash": 8211,
+    "mdash": 8212,
+    "lsquo": 8216,
+    "rsquo": 8217,
+    "sbquo": 8218,
+    "ldquo": 8220,
+    "rdquo": 8221,
+    "bdquo": 8222,
+    "dagger": 8224,
+    "Dagger": 8225,
+    "permil": 8240,
+    "lsaquo": 8249,
+    "rsaquo": 8250,
+    "euro": 8364,
+    "weierp": 8472,
+    "image": 8465,
+    "real": 8476,
+    "trade": 8482,
+    "alefsym": 8501,
+    "larr": 8592,
+    "uarr": 8593,
+    "rarr": 8594,
+    "darr": 8595,
+    "harr": 8596,
+    "crarr": 8629,
+    "lArr": 8656,
+    "uArr": 8657,
+    "rArr": 8658,
+    "dArr": 8659,
+    "hArr": 8660,
+    "forall": 8704,
+    "part": 8706,
+    "exist": 8707,
+    "empty": 8709,
+    "nabla": 8711,
+    "isin": 8712,
+    "notin": 8713,
+    "ni": 8715,
+    "prod": 8719,
+    "sum": 8721,
+    "minus": 8722,
+    "lowast": 8727,
+    "radic": 8730,
+    "prop": 8733,
+    "infin": 8734,
+    "ang": 8736,
+    "and": 8743,
+    "or": 8744,
+    "cap": 8745,
+    "cup": 8746,
+    "int": 8747,
+    "there4": 8756,
+    "sim": 8764,
+    "cong": 8773,
+    "asymp": 8776,
+    "ne": 8800,
+    "equiv": 8801,
+    "le": 8804,
+    "ge": 8805,
+    "sub": 8834,
+    "sup": 8835,
+    "nsub": 8836,
+    "sube": 8838,
+    "supe": 8839,
+    "oplus": 8853,
+    "otimes": 8855,
+    "perp": 8869,
+    "sdot": 8901,
+    "lceil": 8968,
+    "rceil": 8969,
+    "lfloor": 8970,
+    "rfloor": 8971,
+    "lang": 9001,
+    "rang": 9002,
+    "loz": 9674,
+    "spades": 9824,
+    "clubs": 9827,
+    "hearts": 9829,
+    "diams": 9830
+}
@@ -13,7 +13,6 @@ import shutil
 import tempfile
 import threading

-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import dataToDumpFile
@@ -53,6 +52,7 @@ from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import UNSAFE_DUMP_FILEPATH_REPLACEMENT
 from lib.core.settings import VERSION_STRING
 from lib.core.settings import WINDOWS_RESERVED_NAMES
+from lib.utils.safe2bin import safechardecode
 from thirdparty import six
 from thirdparty.magic import magic

@@ -617,9 +617,10 @@ class Dump(object):
                                warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath)
                                logger.warn(warnMsg)

-                                with open(filepath, "wb") as f:
+                                with openFile(filepath, "w+b", None) as f:
                                    _ = safechardecode(value, True)
                                    f.write(_)
+
                        except magic.MagicException as ex:
                            logger.debug(getSafeExString(ex))

@@ -675,28 +676,28 @@ class Dump(object):
            else:
                colConsiderStr = " '%s' was" % unsafeSQLIdentificatorNaming(column)

-            msg = "column%s found in the " % colConsiderStr
-            msg += "following databases:"
-            self._write(msg)
-
-            _ = {}
-
+            found = {}
            for db, tblData in dbs.items():
                for tbl, colData in tblData.items():
                    for col, dataType in colData.items():
                        if column.lower() in col.lower():
-                            if db in _:
-                                if tbl in _[db]:
-                                    _[db][tbl][col] = dataType
+                            if db in found:
+                                if tbl in found[db]:
+                                    found[db][tbl][col] = dataType
                                else:
-                                    _[db][tbl] = {col: dataType}
+                                    found[db][tbl] = {col: dataType}
                            else:
-                                _[db] = {}
-                                _[db][tbl] = {col: dataType}
+                                found[db] = {}
+                                found[db][tbl] = {col: dataType}

                            continue

-            self.dbTableColumns(_)
+            if found:
+                msg = "column%s found in the " % colConsiderStr
+                msg += "following databases:"
+                self._write(msg)
+
+                self.dbTableColumns(found)

    def sqlQuery(self, query, queryRes):
        self.string(query, queryRes, content_type=CONTENT_TYPE.SQL_QUERY)
@@ -130,12 +130,12 @@ class HASH(object):
    MSSQL_NEW = r'(?i)\A0x0200[0-9a-f]{8}[0-9a-f]{128}\Z'
    ORACLE = r'(?i)\As:[0-9a-f]{60}\Z'
    ORACLE_OLD = r'(?i)\A[0-9a-f]{16}\Z'
-    MD5_GENERIC = r'(?i)\A[0-9a-f]{32}\Z'
-    SHA1_GENERIC = r'(?i)\A[0-9a-f]{40}\Z'
+    MD5_GENERIC = r'(?i)\A(0x)?[0-9a-f]{32}\Z'
+    SHA1_GENERIC = r'(?i)\A(0x)?[0-9a-f]{40}\Z'
    SHA224_GENERIC = r'(?i)\A[0-9a-f]{56}\Z'
-    SHA256_GENERIC = r'(?i)\A[0-9a-f]{64}\Z'
+    SHA256_GENERIC = r'(?i)\A(0x)?[0-9a-f]{64}\Z'
    SHA384_GENERIC = r'(?i)\A[0-9a-f]{96}\Z'
-    SHA512_GENERIC = r'(?i)\A[0-9a-f]{128}\Z'
+    SHA512_GENERIC = r'(?i)\A(0x)?[0-9a-f]{128}\Z'
    CRYPT_GENERIC = r'\A(?!\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\Z)(?![0-9]+\Z)[./0-9A-Za-z]{13}\Z'
    JOOMLA = r'\A[0-9a-f]{32}:\w{32}\Z'
    WORDPRESS = r'\A\$P\$[./0-9a-zA-Z]{31}\Z'
@@ -244,8 +244,8 @@ class HASHDB_KEYS(object):
    OS = "OS"

 class REDIRECTION(object):
-    YES = "Y"
-    NO = "N"
+    YES = 'Y'
+    NO = 'N'

 class PAYLOAD(object):
    SQLINJECTION = {
@@ -293,6 +293,7 @@ def _setRequestFromFile():
    if conf.requestFile:
        for requestFile in re.split(PARAMETER_SPLITTING_REGEX, conf.requestFile):
            requestFile = safeExpandUser(requestFile)
+            url = None
            seen = set()

            if not checkFile(requestFile, False):
@@ -311,6 +312,11 @@ def _setRequestFromFile():
                        conf.multipleTargets = True
                    seen.add(url)

+            if url is None:
+                errMsg = "specified file '%s' " % requestFile
+                errMsg += "does not contain a valid HTTP request"
+                raise SqlmapDataException(errMsg)
+
    if conf.secondReq:
        conf.secondReq = safeExpandUser(conf.secondReq)

@@ -335,7 +341,8 @@ def _setCrawler():
        if conf.bulkFile:
            targets = getFileItems(conf.bulkFile)
        else:
-            targets = parseSitemap(conf.sitemapUrl)
+            targets = list(parseSitemap(conf.sitemapUrl))
+
        for i in xrange(len(targets)):
            try:
                target = targets[i]
@@ -455,25 +462,31 @@ def _findPageForms():
    if conf.url and not checkConnection():
        return

+    found = False
    infoMsg = "searching for forms"
    logger.info(infoMsg)

    if not any((conf.bulkFile, conf.googleDork, conf.sitemapUrl)):
        page, _, _ = Request.queryPage(content=True)
-        findPageForms(page, conf.url, True, True)
+        if findPageForms(page, conf.url, True, True):
+            found = True
    else:
        if conf.bulkFile:
            targets = getFileItems(conf.bulkFile)
        elif conf.sitemapUrl:
-            targets = parseSitemap(conf.sitemapUrl)
+            targets = list(parseSitemap(conf.sitemapUrl))
        elif conf.googleDork:
            targets = [_[0] for _ in kb.targets]
            kb.targets.clear()
+        else:
+            targets = []
+
        for i in xrange(len(targets)):
            try:
                target = targets[i]
                page, _, _ = Request.getPage(url=target.strip(), cookie=conf.cookie, crawling=True, raise404=False)
-                findPageForms(page, target, False, True)
+                if findPageForms(page, target, False, True):
+                    found = True

                if conf.verbose in (1, 2):
                    status = '%d/%d links visited (%d%%)' % (i + 1, len(targets), round(100.0 * (i + 1) / len(targets)))
@@ -484,6 +497,10 @@ def _findPageForms():
                errMsg = "problem occurred while searching for forms at '%s' ('%s')" % (target, getSafeExString(ex))
                logger.error(errMsg)

+    if not found:
+        warnMsg = "no forms found"
+        logger.warn(warnMsg)
+
 def _setDBMSAuthentication():
    """
    Check and set the DBMS authentication credentials to run statements as
@@ -522,26 +539,14 @@ def _setMetasploit():
            errMsg = "sqlmap requires third-party module 'pywin32' "
            errMsg += "in order to use Metasploit functionalities on "
            errMsg += "Windows. You can download it from "
-            errMsg += "'https://sourceforge.net/projects/pywin32/files/pywin32/'"
+            errMsg += "'https://github.com/mhammond/pywin32'"
            raise SqlmapMissingDependence(errMsg)

        if not conf.msfPath:
-            def _(key, value):
-                retVal = None
-
-                try:
-                    from six.moves.winreg import ConnectRegistry, OpenKey, QueryValueEx, HKEY_LOCAL_MACHINE
-                    _ = ConnectRegistry(None, HKEY_LOCAL_MACHINE)
-                    _ = OpenKey(_, key)
-                    retVal = QueryValueEx(_, value)[0]
-                except:
-                    logger.debug("unable to identify Metasploit installation path via registry key")
-
-                return retVal
-
-            conf.msfPath = _(r"SOFTWARE\Rapid7\Metasploit", "Location")
-            if conf.msfPath:
-                conf.msfPath = os.path.join(conf.msfPath, "msf3")
+            for candidate in os.environ.get("PATH", "").split(';'):
+                if all(_ in candidate for _ in ("metasploit", "bin")):
+                    conf.msfPath = os.path.dirname(candidate.rstrip('\\'))
+                    break

    if conf.osSmb:
        isAdmin = runningAsAdmin()
@@ -1272,8 +1277,8 @@ def _setHTTPAuthentication():
                from ntlm import HTTPNtlmAuthHandler
            except ImportError:
                errMsg = "sqlmap requires Python NTLM third-party library "
-                errMsg += "in order to authenticate via NTLM, "
-                errMsg += "https://github.com/mullender/python-ntlm"
+                errMsg += "in order to authenticate via NTLM. Download from "
+                errMsg += "'https://github.com/mullender/python-ntlm'"
                raise SqlmapMissingDependence(errMsg)

            authHandler = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(kb.passwordMgr)
@@ -1301,6 +1306,9 @@ def _setHTTPExtraHeaders():

                if header and value:
                    conf.httpHeaders.append((header, value))
+            elif headerValue.startswith('@'):
+                checkFile(headerValue[1:])
+                kb.headersFile = headerValue[1:]
            else:
                errMsg = "invalid header value: %s. Valid header format is 'name:value'" % repr(headerValue).lstrip('u')
                raise SqlmapSyntaxException(errMsg)
@@ -1453,6 +1461,9 @@ def _createHomeDirectories():
    Creates directories inside sqlmap's home directory
    """

+    if conf.get("purge"):
+        return
+
    for context in "output", "history":
        directory = paths["SQLMAP_%s_PATH" % context.upper()]
        try:
@@ -1463,7 +1474,7 @@ def _createHomeDirectories():
            open(_, "w+b").close()
            os.remove(_)

-            if conf.outputDir and context == "output":
+            if conf.get("outputDir") and context == "output":
                warnMsg = "using '%s' as the %s directory" % (directory, context)
                logger.warn(warnMsg)
        except (OSError, IOError) as ex:
@@ -1557,6 +1568,15 @@ def _cleanupOptions():
    else:
        conf.testParameter = []

+    if conf.ignoreCode:
+        try:
+            conf.ignoreCode = [int(_) for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.ignoreCode)]
+        except ValueError:
+            errMsg = "options '--ignore-code' should contain a list of integer values"
+            raise SqlmapSyntaxException(errMsg)
+    else:
+        conf.ignoreCode = []
+
    if conf.paramFilter:
        conf.paramFilter = [_.strip() for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.paramFilter.upper())]
    else:
@@ -1576,8 +1596,19 @@ def _cleanupOptions():
        conf.user = conf.user.replace(" ", "")

    if conf.rParam:
-        conf.rParam = conf.rParam.replace(" ", "")
-        conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
+        if all(_ in conf.rParam for _ in ('=', ',')):
+            original = conf.rParam
+            conf.rParam = []
+            for part in original.split(';'):
+                if '=' in part:
+                    left, right = part.split('=', 1)
+                    conf.rParam.append(left)
+                    kb.randomPool[left] = filterNone(_.strip() for _ in right.split(','))
+                else:
+                    conf.rParam.append(part)
+        else:
+            conf.rParam = conf.rParam.replace(" ", "")
+            conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
    else:
        conf.rParam = []

@@ -1881,6 +1912,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
    kb.forceWhere = None
    kb.futileUnion = None
    kb.heavilyDynamic = False
+    kb.headersFile = None
    kb.headersFp = {}
    kb.heuristicDbms = None
    kb.heuristicExtendedDbms = None
@@ -1901,12 +1933,13 @@ def _setKnowledgeBaseAttributes(flushAll=True):
    kb.lastParserStatus = None

    kb.locks = AttribDict()
-    for _ in ("cache", "connError", "count", "handlers", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
+    for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
        kb.locks[_] = threading.Lock()

    kb.matchRatio = None
    kb.maxConnectionsFlag = False
    kb.mergeCookies = None
+    kb.multipleCtrlC = False
    kb.negativeLogic = False
    kb.nullConnection = None
    kb.oldMsf = None
@@ -1939,6 +1972,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
    kb.processUserMarks = None
    kb.proxyAuthHeader = None
    kb.queryCounter = 0
+    kb.randomPool = {}
    kb.redirectChoice = None
    kb.reflectiveMechanism = True
    kb.reflectiveCounters = {REFLECTIVE_COUNTER.MISS: 0, REFLECTIVE_COUNTER.HIT: 0}
@@ -2417,6 +2451,13 @@ def _basicOptionValidation():
            errMsg = "invalid regular expression '%s' ('%s')" % (conf.regexp, getSafeExString(ex))
            raise SqlmapSyntaxException(errMsg)

+    if conf.paramExclude:
+        try:
+            re.compile(conf.paramExclude)
+        except Exception as ex:
+            errMsg = "invalid regular expression '%s' ('%s')" % (conf.paramExclude, getSafeExString(ex))
+            raise SqlmapSyntaxException(errMsg)
+
    if conf.crawlExclude:
        try:
            re.compile(conf.crawlExclude)
@@ -2460,6 +2501,10 @@ def _basicOptionValidation():
        errMsg = "option '--csrf-url' requires usage of option '--csrf-token'"
        raise SqlmapSyntaxException(errMsg)

+    if conf.csrfMethod and not conf.csrfToken:
+        errMsg = "option '--csrf-method' requires usage of option '--csrf-token'"
+        raise SqlmapSyntaxException(errMsg)
+
    if conf.csrfToken and conf.threads > 1:
        errMsg = "option '--csrf-url' is incompatible with option '--threads'"
        raise SqlmapSyntaxException(errMsg)
@@ -31,6 +31,7 @@ optDict = {
        "loadCookies": "string",
        "dropSetCookie": "boolean",
        "agent": "string",
+        "mobile": "boolean",
        "randomAgent": "boolean",
        "host": "string",
        "referer": "string",
@@ -38,7 +39,7 @@ optDict = {
        "authType": "string",
        "authCred": "string",
        "authFile": "string",
-        "ignoreCode": "integer",
+        "ignoreCode": "string",
        "ignoreProxy": "boolean",
        "ignoreRedirects": "boolean",
        "ignoreTimeouts": "boolean",
@@ -60,6 +61,7 @@ optDict = {
        "skipUrlEncode": "boolean",
        "csrfToken": "string",
        "csrfUrl": "string",
+        "csrfMethod": "string",
        "forceSSL": "boolean",
        "chunked": "boolean",
        "hpp": "boolean",
@@ -100,6 +102,7 @@ optDict = {
        "notString": "string",
        "regexp": "string",
        "code": "integer",
+        "smart": "boolean",
        "textOnly": "boolean",
        "titles": "boolean",
    },
@@ -197,10 +200,12 @@ optDict = {

    "General": {
        "trafficFile": "string",
+        "answers": "string",
        "batch": "boolean",
        "binaryFields": "string",
        "charset": "string",
        "checkInternet": "boolean",
+        "cleanup": "boolean",
        "crawlDepth": "integer",
        "crawlExclude": "string",
        "csvDel": "string",
@@ -210,6 +215,7 @@ optDict = {
        "flushSession": "boolean",
        "forms": "boolean",
        "freshQueries": "boolean",
+        "googlePage": "integer",
        "harFile": "string",
        "hexConvert": "boolean",
        "outputDir": "string",
@@ -218,28 +224,23 @@ optDict = {
        "repair": "boolean",
        "saveConfig": "string",
        "scope": "string",
+        "skipWaf": "boolean",
        "testFilter": "string",
        "testSkip": "string",
-        "updateAll": "boolean",
+        "webRoot": "string",
    },

    "Miscellaneous": {
        "alert": "string",
-        "answers": "string",
        "beep": "boolean",
-        "cleanup": "boolean",
        "dependencies": "boolean",
        "disableColoring": "boolean",
-        "googlePage": "integer",
        "listTampers": "boolean",
-        "mobile": "boolean",
        "offline": "boolean",
        "purge": "boolean",
-        "skipWaf": "boolean",
-        "smart": "boolean",
        "tmpDir": "string",
-        "webRoot": "string",
        "wizard": "boolean",
+        "updateAll": "boolean",
        "verbose": "integer",
    },

@@ -7,13 +7,13 @@ See the file 'LICENSE' for copying permission

 import sqlite3

-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.common import getSafeExString
 from lib.core.common import unsafeSQLIdentificatorNaming
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapGenericException
 from lib.core.exception import SqlmapValueException
 from lib.core.settings import UNICODE_ENCODING
+from lib.utils.safe2bin import safechardecode

 class Replication(object):
    """
@@ -38,7 +38,7 @@ def getRevisionNumber():
    while True:
        if filePath and os.path.isfile(filePath):
            with openFile(filePath, "r") as f:
-                content = f.read()
+                content = getText(f.read())
                filePath = None
                if content.startswith("ref: "):
                    filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()
@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr

 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.7.0"
+VERSION = "1.3.10.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -240,7 +240,7 @@ IS_WIN = PLATFORM == "nt"
 IS_TTY = os.isatty(sys.stdout.fileno())

 # DBMS system databases
-MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb")
+MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb", "Resource", "ReportServer", "ReportServerTempDB")
 MYSQL_SYSTEM_DBS = ("information_schema", "mysql", "performance_schema", "sys")
 PGSQL_SYSTEM_DBS = ("information_schema", "pg_catalog", "pg_toast", "pgagent")
 ORACLE_SYSTEM_DBS = ('ANONYMOUS', 'APEX_030200', 'APEX_PUBLIC_USER', 'APPQOSSYS', 'BI', 'CTXSYS', 'DBSNMP', 'DIP', 'EXFSYS', 'FLOWS_%', 'FLOWS_FILES', 'HR', 'IX', 'LBACSYS', 'MDDATA', 'MDSYS', 'MGMT_VIEW', 'OC', 'OE', 'OLAPSYS', 'ORACLE_OCM', 'ORDDATA', 'ORDPLUGINS', 'ORDSYS', 'OUTLN', 'OWBSYS', 'PM', 'SCOTT', 'SH', 'SI_INFORMTN_SCHEMA', 'SPATIAL_CSW_ADMIN_USR', 'SPATIAL_WFS_ADMIN_USR', 'SYS', 'SYSMAN', 'SYSTEM', 'WKPROXY', 'WKSYS', 'WK_TEST', 'WMSYS', 'XDB', 'XS$NULL')
@@ -251,7 +251,7 @@ MAXDB_SYSTEM_DBS = ("SYSINFO", "DOMAIN")
 SYBASE_SYSTEM_DBS = ("master", "model", "sybsystemdb", "sybsystemprocs")
 DB2_SYSTEM_DBS = ("NULLID", "SQLJ", "SYSCAT", "SYSFUN", "SYSIBM", "SYSIBMADM", "SYSIBMINTERNAL", "SYSIBMTS", "SYSPROC", "SYSPUBLIC", "SYSSTAT", "SYSTOOLS")
 HSQLDB_SYSTEM_DBS = ("INFORMATION_SCHEMA", "SYSTEM_LOB")
-H2_SYSTEM_DBS = ("INFORMATION_SCHEMA")
+H2_SYSTEM_DBS = ("INFORMATION_SCHEMA",)
 INFORMIX_SYSTEM_DBS = ("sysmaster", "sysutils", "sysuser", "sysadmin")

 MSSQL_ALIASES = ("microsoft sql server", "mssqlserver", "mssql", "ms")
@@ -333,6 +333,9 @@ BLANK = "<blank>"
 # String representation for current database
 CURRENT_DB = "CD"

+# String representation for current user
+CURRENT_USER = "CU"
+
 # Name of SQLite file used for storing session data
 SESSION_SQLITE_FILE = "session.sqlite"

@@ -540,7 +543,7 @@ BRUTE_TABLE_EXISTS_TEMPLATE = "EXISTS(SELECT %d FROM %s)"
 BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT %s FROM %s)"

 # Data inside shellcodeexec to be filled with random string
-SHELLCODEEXEC_RANDOM_STRING_MARKER = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+SHELLCODEEXEC_RANDOM_STRING_MARKER = b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"

 # Period after last-update to start nagging about the old revision
 LAST_UPDATE_NAGGING_DAYS = 60
@@ -807,9 +810,6 @@ BRUTE_DOC_ROOT_PREFIXES = {
    OS.WINDOWS: ("/xampp", "/Program Files/xampp", "/wamp", "/Program Files/wampp", "/apache", "/Program Files/Apache Group/Apache", "/Program Files/Apache Group/Apache2", "/Program Files/Apache Group/Apache2.2", "/Program Files/Apache Group/Apache2.4", "/Inetpub/wwwroot", "/Inetpub/wwwroot/%TARGET%", "/Inetpub/vhosts/%TARGET%")
 }

-# Table prefix to use in "takeover" functionalities (i.e. auxiliary tables used by sqlmap at the vulnerable DBMS)
-TAKEOVER_TABLE_PREFIX = "sqlmap"
-
 # Suffixes used in brute force search for web server document root
 BRUTE_DOC_ROOT_SUFFIXES = ("", "html", "htdocs", "httpdocs", "php", "public", "src", "site", "build", "web", "www", "data", "sites/all", "www/build")

@@ -13,6 +13,7 @@ import subprocess
 import time

 from lib.core.compat import buffer
+from lib.core.convert import getBytes
 from lib.core.settings import IS_WIN

 if IS_WIN:
@@ -192,6 +193,8 @@ def send_all(p, data):
    if not data:
        return

+    data = getBytes(data)
+
    while len(data):
        sent = p.send(data)
        if not isinstance(sent, int):
@@ -111,7 +111,7 @@ def _setRequestParams():
        def process(match, repl):
            retVal = match.group(0)

-            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]):
+            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]) and match.group("name") == match.group("name").strip('\\'):
                retVal = repl
                while True:
                    _ = re.search(r"\\g<([^>]+)>", retVal)
@@ -121,11 +121,12 @@ def _setRequestParams():
                        break
                if kb.customInjectionMark in retVal:
                    hintNames.append((retVal.split(kb.customInjectionMark)[0], match.group("name")))
+
            return retVal

        if kb.processUserMarks is None and kb.customInjectionMark in conf.data:
-            message = "custom injection marker ('%s') found in option " % kb.customInjectionMark
-            message += "'--data'. Do you want to process it? [Y/n/q] "
+            message = "custom injection marker ('%s') found in POST " % kb.customInjectionMark
+            message += "body. Do you want to process it? [Y/n/q] "
            choice = readInput(message, default='Y').upper()

            if choice == 'Q':
@@ -150,12 +151,13 @@ def _setRequestParams():
                    conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*".+?)"(?<!\\")', functools.partial(process, repl=r'\g<1>%s"' % kb.customInjectionMark), conf.data)
                    conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)(-?\d[\d\.]*)\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
                    conf.data = re.sub(r'("(?P<name>[^"]+)"\s*:\s*)((true|false|null))\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data)
-                    match = re.search(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]', conf.data)
-                    if match and not (conf.testParameter and match.group("name") not in conf.testParameter):
-                        _ = match.group(2)
-                        _ = re.sub(r'("[^"]+)"', r'\g<1>%s"' % kb.customInjectionMark, _)
-                        _ = re.sub(r'(\A|,|\s+)(-?\d[\d\.]*\b)', r'\g<0>%s' % kb.customInjectionMark, _)
-                        conf.data = conf.data.replace(match.group(0), match.group(0).replace(match.group(2), _))
+                    for match in re.finditer(r'(?P<name>[^"]+)"\s*:\s*\[([^\]]+)\]', conf.data):
+                        if not (conf.testParameter and match.group("name") not in conf.testParameter):
+                            _ = match.group(2)
+                            if kb.customInjectionMark not in _:  # Note: only for unprocessed (simple) forms - i.e. non-associative arrays (e.g. [1,2,3])
+                                _ = re.sub(r'("[^"]+)"', r'\g<1>%s"' % kb.customInjectionMark, _)
+                                _ = re.sub(r'(\A|,|\s+)(-?\d[\d\.]*\b)', r'\g<0>%s' % kb.customInjectionMark, _)
+                                conf.data = conf.data.replace(match.group(0), match.group(0).replace(match.group(2), _))

                kb.postHint = POST_HINT.JSON

@@ -95,11 +95,13 @@ def exceptionHandledFunction(threadFunction, silent=False):
        kb.threadException = True
        raise
    except Exception as ex:
-        if not silent and kb.get("threadContinue"):
-            errMsg = ex.message if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, ex.message)
+        from lib.core.common import getSafeExString
+
+        if not silent and kb.get("threadContinue") and not isinstance(ex, SqlmapUserQuitException):
+            errMsg = getSafeExString(ex) if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, getSafeExString(ex))
            logger.error("thread %s: '%s'" % (threading.currentThread().getName(), errMsg))

-            if conf.get("verbose") > 1 and not isinstance(ex, (SqlmapUserQuitException,)):
+            if conf.get("verbose") > 1 and not isinstance(ex, SqlmapConnectionException):
                traceback.print_exc()

 def setDaemon(thread):
@@ -112,6 +114,7 @@ def setDaemon(thread):
 def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardException=True, threadChoice=False, startThreadMsg=True):
    threads = []

+    kb.multipleCtrlC = False
    kb.threadContinue = True
    kb.threadException = False
    kb.technique = ThreadData.technique
@@ -183,6 +186,7 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
                pass

        except KeyboardInterrupt:
+            kb.multipleCtrlC = True
            raise SqlmapThreadException("user aborted (Ctrl+C was pressed multiple times)")

        if forwardException:
@@ -193,17 +197,19 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
        kb.threadException = True
        logger.error("thread %s: '%s'" % (threading.currentThread().getName(), ex))

-        if conf.get("verbose") > 1:
+        if conf.get("verbose") > 1 and isinstance(ex, SqlmapValueException):
            traceback.print_exc()

    except:
-        from lib.core.common import unhandledExceptionMessage
-
        print()
-        kb.threadException = True
-        errMsg = unhandledExceptionMessage()
-        logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
-        traceback.print_exc()
+
+        if not kb.multipleCtrlC:
+            from lib.core.common import unhandledExceptionMessage
+
+            kb.threadException = True
+            errMsg = unhandledExceptionMessage()
+            logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
+            traceback.print_exc()

    finally:
        kb.threadContinue = True
@@ -105,6 +105,7 @@ def update():

        dataToStdout("\r[%s] [INFO] update in progress" % time.strftime("%X"))

+        output = ""
        try:
            process = subprocess.Popen("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=paths.SQLMAP_ROOT_PATH)
            pollProcess(process, True)
@@ -18,7 +18,9 @@ class Wordlist(six.Iterator):
    Iterator for looping over a large dictionaries

    >>> from lib.core.option import paths
-    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.string_types)
+    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.binary_type)
+    True
+    >>> isinstance(next(Wordlist(paths.WORDLIST)), six.binary_type)
    True
    """

@@ -40,7 +42,7 @@ class Wordlist(six.Iterator):
    def adjust(self):
        self.closeFP()
        if self.index > len(self.filenames):
-            raise StopIteration
+            return  # Note: https://stackoverflow.com/a/30217723 (PEP 479)
        elif self.index == len(self.filenames):
            self.iter = iter(self.custom)
        else:
@@ -58,7 +60,7 @@ class Wordlist(six.Iterator):
                    raise SqlmapDataException(errMsg)
                self.fp = _.open(_.namelist()[0])
            else:
-                self.fp = open(self.current, 'r')
+                self.fp = open(self.current, "rb")
            self.iter = iter(self.fp)

        self.index += 1
@@ -34,6 +34,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.decorators import cachedmethod
+from lib.core.dicts import HTML_ENTITIES
 from lib.core.enums import DBMS
 from lib.core.enums import HTTP_HEADER
 from lib.core.enums import PLACE
@@ -49,7 +50,6 @@ from lib.core.settings import SELECT_FROM_TABLE_REGEX
 from lib.core.settings import VIEWSTATE_REGEX
 from lib.parse.headers import headersParser
 from lib.parse.html import htmlParser
-from lib.utils.htmlentities import htmlEntities
 from thirdparty import six
 from thirdparty.chardet import detect
 from thirdparty.identywaf import identYwaf
@@ -341,7 +341,7 @@ def decodePage(page, contentEncoding, contentType):
            page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page)

        # e.g. &amp;
-        page = re.sub(b"&([^;]+);", lambda _: six.int2byte(htmlEntities[getText(_.group(1))]) if htmlEntities.get(getText(_.group(1)), 256) < 256 else _.group(0), page)
+        page = re.sub(b"&([^;]+);", lambda _: six.int2byte(HTML_ENTITIES[getText(_.group(1))]) if HTML_ENTITIES.get(getText(_.group(1)), 256) < 256 else _.group(0), page)

        kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))

@@ -364,7 +364,7 @@ def decodePage(page, contentEncoding, contentType):
            page = re.sub(r"&#(\d+);", _, page)

        # e.g. &zeta;
-        page = re.sub(r"&([^;]+);", lambda _: _unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
+        page = re.sub(r"&([^;]+);", lambda _: _unichr(HTML_ENTITIES[_.group(1)]) if HTML_ENTITIES.get(_.group(1), 0) > 255 else _.group(0), page)

    return page

@@ -7,6 +7,7 @@ See the file 'LICENSE' for copying permission

 import binascii
 import logging
+import random
 import re
 import socket
 import string
@@ -21,7 +22,6 @@ except ImportError:
    class WebSocketException(Exception):
        pass

-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import asciifyUrl
 from lib.core.common import calculateDeltaSeconds
@@ -41,6 +41,7 @@ from lib.core.common import getRequestHeader
 from lib.core.common import getSafeExString
 from lib.core.common import isMultiThreadMode
 from lib.core.common import logHTTPTraffic
+from lib.core.common import openFile
 from lib.core.common import popValue
 from lib.core.common import pushValue
 from lib.core.common import randomizeParameterValue
@@ -59,6 +60,7 @@ from lib.core.common import wasLastResponseDelayed
 from lib.core.compat import patchHeaders
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
+from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -122,6 +124,7 @@ from lib.request.basic import processResponse
 from lib.request.comparison import comparison
 from lib.request.direct import direct
 from lib.request.methodrequest import MethodRequest
+from lib.utils.safe2bin import safecharencode
 from thirdparty import six
 from thirdparty.odict import OrderedDict
 from thirdparty.six import unichr as _unichr
@@ -425,6 +428,14 @@ class Connect(object):
            if auxHeaders:
                headers = forgeHeaders(auxHeaders, headers)

+            if kb.headersFile:
+                content = openFile(kb.headersFile, "rb").read()
+                for line in content.split("\n"):
+                    line = getText(line.strip())
+                    if ':' in line:
+                        header, value = line.split(':', 1)
+                        headers[header] = value
+
            for key, value in list(headers.items()):
                del headers[key]
                if isinstance(value, six.string_types):
@@ -646,7 +657,7 @@ class Connect(object):
            if not multipart:
                logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)

-            if ex.code != conf.ignoreCode:
+            if ex.code not in (conf.ignoreCode or []):
                if ex.code == _http_client.UNAUTHORIZED:
                    errMsg = "not authorized, try to provide right HTTP "
                    errMsg += "authentication type and valid credentials (%d)" % code
@@ -748,21 +759,17 @@ class Connect(object):

                if kb.connErrorCounter >= MAX_CONSECUTIVE_CONNECTION_ERRORS and kb.connErrorChoice is None:
                    message = "there seems to be a continuous problem with connection to the target. "
-                    message += "Are you sure that you want to continue "
-                    message += "with further target testing? [y/N] "
+                    message += "Are you sure that you want to continue? [y/N] "

                    kb.connErrorChoice = readInput(message, default='N', boolean=True)

-                if kb.connErrorChoice is not None:
-                    if kb.connErrorChoice:
-                        raise SqlmapConnectionException(warnMsg)
-                    else:
-                        raise SqlmapUserQuitException
+                if kb.connErrorChoice is False:
+                    raise SqlmapUserQuitException

            if "forcibly closed" in tbMsg:
                logger.critical(warnMsg)
                return None, None, None
-            elif ignoreTimeout and any(_ in tbMsg for _ in ("timed out", "IncompleteRead")):
+            elif ignoreTimeout and any(_ in tbMsg for _ in ("timed out", "IncompleteRead", "Interrupted system call")):
                return None if not conf.ignoreTimeouts else "", None, None
            elif threadData.retriesCount < conf.retries and not kb.threadException:
                warnMsg += ". sqlmap is going to retry the request"
@@ -1032,7 +1039,7 @@ class Connect(object):
                return retVal

            token = AttribDict()
-            page, headers, code = Connect.getPage(url=conf.csrfUrl or conf.url, data=conf.data if conf.csrfUrl == conf.url else None, method=conf.method if conf.csrfUrl == conf.url else None, cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST))
+            page, headers, code = Connect.getPage(url=conf.csrfUrl or conf.url, data=conf.data if conf.csrfUrl == conf.url else None, method=conf.csrfMethod or (conf.method if conf.csrfUrl == conf.url else None), cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST))
            page = urldecode(page)  # for anti-CSRF tokens with special characters in their name (e.g. 'foo:bar=...')

            match = re.search(r"(?i)<input[^>]+\bname=[\"']?(?P<name>%s)\b[^>]*\bvalue=[\"']?(?P<value>[^>'\"]*)" % conf.csrfToken, page or "", re.I)
@@ -1102,7 +1109,8 @@ class Connect(object):
                match = re.search(r"(\A|\b)%s=(?P<value>[^&;]*)" % re.escape(randomParameter), paramString)
                if match:
                    origValue = match.group("value")
-                    retVal = re.sub(r"(\A|\b)%s=[^&;]*" % re.escape(randomParameter), "%s=%s" % (randomParameter, randomizeParameterValue(origValue)), paramString)
+                    newValue = randomizeParameterValue(origValue) if randomParameter not in kb.randomPool else random.sample(kb.randomPool[randomParameter], 1)[0]
+                    retVal = re.sub(r"(\A|\b)%s=[^&;]*" % re.escape(randomParameter), "%s=%s" % (randomParameter, newValue), paramString)
                return retVal

            for randomParameter in conf.rParam:
@@ -7,7 +7,6 @@ See the file 'LICENSE' for copying permission

 import time

-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -25,8 +24,8 @@ from lib.core.enums import CUSTOM_LOGGING
 from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import TIMEOUT_STATE
-from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.core.settings import UNICODE_ENCODING
+from lib.utils.safe2bin import safecharencode
 from lib.utils.timeout import timeout

 def direct(query, content=True):
@@ -54,7 +53,7 @@ def direct(query, content=True):

    if not select and "EXEC " not in query.upper():
        timeout(func=conf.dbmsConnector.execute, args=(query,), duration=conf.timeout, default=None)
-    elif not (output and ("%soutput" % TAKEOVER_TABLE_PREFIX) not in query and ("%sfile" % TAKEOVER_TABLE_PREFIX) not in query):
+    elif not (output and ("%soutput" % conf.tablePrefix) not in query and ("%sfile" % conf.tablePrefix) not in query):
        output, state = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
        if state == TIMEOUT_STATE.NORMAL:
            hashDBWrite(query, output, True)
@@ -84,7 +84,7 @@ class DNSServer(object):
        except:
            pass
        finally:
-            if response and "google" in response:
+            if response and b"google" in response:
                raise socket.error("another DNS service already running on *:53")

    def pop(self, prefix=None, suffix=None):
@@ -39,6 +39,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import queries
+from lib.core.decorators import lockedmethod
 from lib.core.decorators import stackedmethod
 from lib.core.dicts import FROM_DUMMY_TABLE
 from lib.core.enums import CHARSET_TYPE
@@ -164,7 +165,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char

    initTechnique(getTechnique())

-    query = agent.prefixQuery(kb.injection.data[getTechnique()].vector)
+    query = agent.prefixQuery(getTechniqueData().vector)
    query = agent.suffixQuery(query)
    payload = agent.payload(newValue=query)
    count = None
@@ -312,7 +313,7 @@ def _goBooleanProxy(expression):
    initTechnique(getTechnique())

    if conf.dnsDomain:
-        query = agent.prefixQuery(kb.injection.data[getTechnique()].vector)
+        query = agent.prefixQuery(getTechniqueData().vector)
        query = agent.suffixQuery(query)
        payload = agent.payload(newValue=query)
        output = _goDns(payload, expression)
@@ -320,7 +321,7 @@ def _goBooleanProxy(expression):
        if output is not None:
            return output

-    vector = kb.injection.data[getTechnique()].vector
+    vector = getTechniqueData().vector
    vector = vector.replace(INFERENCE_MARKER, expression)
    query = agent.prefixQuery(vector)
    query = agent.suffixQuery(query)
@@ -351,6 +352,7 @@ def _goUnion(expression, unpack=True, dump=False):

    return output

+@lockedmethod
@stackedmethod
 def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True):
    """
@@ -24,5 +24,6 @@ class HTTPRangeHandler(_urllib.request.BaseHandler):

    def http_error_416(self, req, fp, code, msg, hdrs):
        # HTTP's Range Not Satisfiable error
-        errMsg = "Invalid range"
+        errMsg = "there was a problem while connecting "
+        errMsg += "target ('406 - Range Not Satisfiable')"
        raise SqlmapConnectionException(errMsg)
@@ -9,7 +9,6 @@ from __future__ import print_function

 import sys

-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.common import Backend
 from lib.core.common import dataToStdout
 from lib.core.common import getSQLSnippet
@@ -28,6 +27,7 @@ from lib.request import inject
 from lib.takeover.udf import UDF
 from lib.takeover.web import Web
 from lib.takeover.xp_cmdshell import XP_cmdshell
+from lib.utils.safe2bin import safechardecode
 from thirdparty.six.moves import input as _input

 class Abstraction(Web, UDF, XP_cmdshell):
@@ -68,29 +68,11 @@ class Metasploit(object):
        self.payloadConnStr = None
        self.localIP = getLocalIP()
        self.remoteIP = getRemoteIP() or conf.hostname
-        self._msfCli = normalizePath(os.path.join(conf.msfPath, "msfcli"))
-        self._msfConsole = normalizePath(os.path.join(conf.msfPath, "msfconsole"))
-        self._msfEncode = normalizePath(os.path.join(conf.msfPath, "msfencode"))
-        self._msfPayload = normalizePath(os.path.join(conf.msfPath, "msfpayload"))
-        self._msfVenom = normalizePath(os.path.join(conf.msfPath, "msfvenom"))
-
-        if IS_WIN:
-            _ = conf.msfPath
-            while _:
-                if os.path.exists(os.path.join(_, "scripts")):
-                    _ = os.path.join(_, "scripts", "setenv.bat")
-                    break
-                else:
-                    old = _
-                    _ = normalizePath(os.path.join(_, ".."))
-                    if _ == old:
-                        break
-
-            self._msfCli = "%s & ruby %s" % (_, self._msfCli)
-            self._msfConsole = "%s & ruby %s" % (_, self._msfConsole)
-            self._msfEncode = "ruby %s" % self._msfEncode
-            self._msfPayload = "%s & ruby %s" % (_, self._msfPayload)
-            self._msfVenom = "%s & ruby %s" % (_, self._msfVenom)
+        self._msfCli = normalizePath(os.path.join(conf.msfPath, "msfcli%s" % (".bat" if IS_WIN else "")))
+        self._msfConsole = normalizePath(os.path.join(conf.msfPath, "msfconsole%s" % (".bat" if IS_WIN else "")))
+        self._msfEncode = normalizePath(os.path.join(conf.msfPath, "msfencode%s" % (".bat" if IS_WIN else "")))
+        self._msfPayload = normalizePath(os.path.join(conf.msfPath, "msfpayload%s" % (".bat" if IS_WIN else "")))
+        self._msfVenom = normalizePath(os.path.join(conf.msfPath, "msfvenom%s" % (".bat" if IS_WIN else "")))

        self._msfPayloadsList = {
            "windows": {
@@ -245,18 +227,15 @@ class Metasploit(object):

                    if not choice or choice == "2":
                        _payloadStr = "windows/meterpreter"
-
                        break

                    elif choice == "3":
                        _payloadStr = "windows/shell"
-
                        break

                    elif choice == "1":
                        if Backend.isDbms(DBMS.PGSQL):
                            logger.warn("beware that the VNC injection might not work")
-
                            break

                        elif Backend.isDbms(DBMS.MSSQL) and Backend.isVersionWithin(("2005", "2008")):
@@ -652,7 +631,7 @@ class Metasploit(object):
            self.shellcodeexecLocal = os.path.join(self.shellcodeexecLocal, "windows", "shellcodeexec.x%s.exe_" % "32")
            content = decloak(self.shellcodeexecLocal)
            if SHELLCODEEXEC_RANDOM_STRING_MARKER in content:
-                content = content.replace(SHELLCODEEXEC_RANDOM_STRING_MARKER, randomStr(len(SHELLCODEEXEC_RANDOM_STRING_MARKER)))
+                content = content.replace(SHELLCODEEXEC_RANDOM_STRING_MARKER, getBytes(randomStr(len(SHELLCODEEXEC_RANDOM_STRING_MARKER))))
                _ = cloak(data=content)
                handle, self.shellcodeexecLocal = tempfile.mkstemp(suffix="%s.exe_" % "32")
                os.close(handle)
@@ -21,6 +21,7 @@ from lib.core.common import getManualDirectories
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSQLSnippet
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import isTechniqueAvailable
 from lib.core.common import isWindowsDriveLetterPath
 from lib.core.common import normalizePath
@@ -149,7 +150,7 @@ class Web(object):
        query = ""

        if isTechniqueAvailable(getTechnique()):
-            where = kb.injection.data[getTechnique()].where
+            where = getTechniqueData().where

            if where == PAYLOAD.WHERE.NEGATIVE:
                randInt = randomInt()
@@ -8,10 +8,8 @@ See the file 'LICENSE' for copying permission
 from __future__ import division

 import re
-import threading
 import time

-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -23,6 +21,7 @@ from lib.core.common import getCharset
 from lib.core.common import getCounter
 from lib.core.common import getPartRun
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import goGoodSamaritan
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
@@ -58,6 +57,7 @@ from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.utils.progress import ProgressBar
+from lib.utils.safe2bin import safecharencode
 from lib.utils.xrange import xrange

 def bisection(payload, expression, length=None, charsetType=None, firstChar=None, lastChar=None, dump=False):
@@ -161,7 +161,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
            length = None

        showEta = conf.eta and isinstance(length, int)
-        numThreads = min(conf.threads or 0, length or 0) or 1
+
+        if kb.bruteMode:
+            numThreads = 1
+        else:
+            numThreads = min(conf.threads or 0, length or 0) or 1

        if showEta:
            progress = ProgressBar(maxValue=length)
@@ -173,22 +177,20 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
            else:
                numThreads = 1

-        if conf.threads == 1 and not timeBasedCompare and not conf.predictOutput:
+        if conf.threads == 1 and not any((timeBasedCompare, conf.predictOutput)):
            warnMsg = "running in a single-thread mode. Please consider "
            warnMsg += "usage of option '--threads' for faster data retrieval"
            singleTimeWarnMessage(warnMsg)

-        if conf.verbose in (1, 2) and not showEta and not conf.api:
-            if isinstance(length, int) and conf.threads > 1:
+        if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
+            if isinstance(length, int) and numThreads > 1:
                dataToStdout("[%s] [INFO] retrieved: %s" % (time.strftime("%X"), "_" * min(length, conf.progressWidth)))
                dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X"))
            else:
                dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X"))

-        hintlock = threading.Lock()
-
        def tryHint(idx):
-            with hintlock:
+            with kb.locks.hint:
                hintValue = kb.hintValue

            if payload is not None and len(hintValue or "") > 0 and len(hintValue) >= idx:
@@ -207,7 +209,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                if result:
                    return hintValue[idx - 1]

-            with hintlock:
+            with kb.locks.hint:
                kb.hintValue = ""

            return None
@@ -229,10 +231,10 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None

            result = not Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)

-            if result and timeBasedCompare and kb.injection.data[getTechnique()].trueCode:
-                result = threadData.lastCode == kb.injection.data[getTechnique()].trueCode
+            if result and timeBasedCompare and getTechniqueData().trueCode:
+                result = threadData.lastCode == getTechniqueData().trueCode
                if not result:
-                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, kb.injection.data[getTechnique()].trueCode)
+                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, getTechniqueData().trueCode)
                    singleTimeWarnMessage(warnMsg)

            incrementCounter(getTechnique())
@@ -341,8 +343,8 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                    result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
                    incrementCounter(getTechnique())

-                    if not timeBasedCompare:
-                        unexpectedCode |= threadData.lastCode not in (kb.injection.data[getTechnique()].falseCode, kb.injection.data[getTechnique()].trueCode)
+                    if not timeBasedCompare and getTechniqueData() is not None:
+                        unexpectedCode |= threadData.lastCode not in (getTechniqueData().falseCode, getTechniqueData().trueCode)
                        if unexpectedCode:
                            warnMsg = "unexpected HTTP code '%s' detected. Will use (extra) validation step in similar cases" % threadData.lastCode
                            singleTimeWarnMessage(warnMsg)
@@ -458,7 +460,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                        return decodeIntToUnicode(candidates[0])

        # Go multi-threading (--threads > 1)
-        if conf.threads > 1 and isinstance(length, int) and length > 1:
+        if numThreads > 1 and isinstance(length, int) and length > 1:
            threadData.shared.value = [None] * length
            threadData.shared.index = [firstChar]    # As list for python nested function scoping
            threadData.shared.start = firstChar
@@ -516,7 +518,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                                if (endCharIndex - startCharIndex == conf.progressWidth) and (endCharIndex < length - 1):
                                    output = output[:-2] + ".."

-                                if conf.verbose in (1, 2) and not showEta and not conf.api:
+                                if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
                                    _ = count - firstChar
                                    output += '_' * (min(length, conf.progressWidth) - len(output))
                                    status = ' %d/%d (%d%%)' % (_, length, int(100.0 * _ / length))
@@ -546,7 +548,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                finalValue = "".join(value)
                infoMsg = "\r[%s] [INFO] retrieved: %s" % (time.strftime("%X"), filterControlChars(finalValue))

-            if conf.verbose in (1, 2) and not showEta and infoMsg and not conf.api:
+            if conf.verbose in (1, 2) and infoMsg and not any((showEta, conf.api, kb.bruteMode)):
                dataToStdout(infoMsg)

        # No multi-threading (--threads = 1)
@@ -570,7 +572,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                        # One-shot query containing equals commonValue
                        testValue = unescaper.escape("'%s'" % commonValue) if "'" not in commonValue else unescaper.escape("%s" % commonValue, quote=False)

-                        query = kb.injection.data[getTechnique()].vector
+                        query = getTechniqueData().vector
                        query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)%s%s" % (expressionUnescaped, INFERENCE_EQUALS_CHAR, testValue)))
                        query = agent.suffixQuery(query)

@@ -594,7 +596,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                        subquery = queries[Backend.getIdentifiedDbms()].substring.query % (expressionUnescaped, 1, len(commonPattern))
                        testValue = unescaper.escape("'%s'" % commonPattern) if "'" not in commonPattern else unescaper.escape("%s" % commonPattern, quote=False)

-                        query = kb.injection.data[getTechnique()].vector
+                        query = getTechniqueData().vector
                        query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)=%s" % (subquery, testValue)))
                        query = agent.suffixQuery(query)

@@ -631,7 +633,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None

                if showEta:
                    progress.progress(index)
-                elif conf.verbose in (1, 2) or conf.api:
+                elif (conf.verbose in (1, 2) and not kb.bruteMode) or conf.api:
                    dataToStdout(filterControlChars(val))

                # some DBMSes (e.g. Firebird, DB2, etc.) have issues with trailing spaces
@@ -660,11 +662,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
        elif partialValue:
            hashDBWrite(expression, "%s%s" % (PARTIAL_VALUE_MARKER if not conf.hexConvert else PARTIAL_HEX_VALUE_MARKER, partialValue))

-    if conf.hexConvert and not abortedFlag and not conf.api:
+    if conf.hexConvert and not any((abortedFlag, conf.api, kb.bruteMode)):
        infoMsg = "\r[%s] [INFO] retrieved: %s  %s\n" % (time.strftime("%X"), filterControlChars(finalValue), " " * retrievedLength)
        dataToStdout(infoMsg)
    else:
-        if conf.verbose in (1, 2) and not showEta and not conf.api:
+        if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
            dataToStdout("\n")

        if (conf.verbose in (1, 2) and showEta) or conf.verbose >= 3:
@@ -8,7 +8,6 @@ See the file 'LICENSE' for copying permission
 import re
 import time

-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
@@ -33,6 +32,7 @@ from lib.core.settings import MAX_DNS_LABEL
 from lib.core.settings import PARTIAL_VALUE_MARKER
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
+from lib.utils.safe2bin import safecharencode

 def dnsUse(payload, expression):
    """
@@ -10,7 +10,6 @@ from __future__ import print_function
 import re
 import time

-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
 from lib.core.common import Backend
@@ -22,6 +21,7 @@ from lib.core.common import firstNotNone
 from lib.core.common import getConsoleWidth
 from lib.core.common import getPartRun
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import incrementCounter
@@ -59,6 +59,7 @@ from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.utils.progress import ProgressBar
+from lib.utils.safe2bin import safecharencode
 from thirdparty import six

 def _oneShotErrorUse(expression, field=None, chunkTest=False):
@@ -75,7 +76,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):

    threadData.resumed = retVal is not None and not partialValue

-    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.ORACLE)) and kb.errorChunkLength is None and not chunkTest and not kb.testMode:
+    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)) and kb.errorChunkLength is None and not chunkTest and not kb.testMode:
        debugMsg = "searching for error chunk length..."
        logger.debug(debugMsg)

@@ -116,7 +117,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                if field:
                    nulledCastedField = agent.nullAndCastField(field)

-                    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.ORACLE)) and not any(_ in field for _ in ("COUNT", "CASE")) and kb.errorChunkLength and not chunkTest:
+                    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)) and not any(_ in field for _ in ("COUNT", "CASE")) and kb.errorChunkLength and not chunkTest:
                        extendedField = re.search(r"[^ ,]*%s[^ ,]*" % re.escape(field), expression).group(0)
                        if extendedField != field:  # e.g. MIN(surname)
                            nulledCastedField = extendedField.replace(field, nulledCastedField)
@@ -124,7 +125,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                        nulledCastedField = queries[Backend.getIdentifiedDbms()].substring.query % (nulledCastedField, offset, kb.errorChunkLength)

                # Forge the error-based SQL injection request
-                vector = kb.injection.data[getTechnique()].vector
+                vector = getTechniqueData().vector
                query = agent.prefixQuery(vector)
                query = agent.suffixQuery(query)
                injExpression = expression.replace(field, nulledCastedField, 1) if field else expression
@@ -176,7 +177,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                            else:
                                output = output.rstrip()

-                if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.ORACLE)):
+                if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)):
                    if offset == 1:
                        retVal = output
                    else:
@@ -187,7 +188,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                    else:
                        break

-                    if output and conf.verbose in (1, 2) and not conf.api:
+                    if output and conf.verbose in (1, 2) and not any((conf.api, kb.bruteMode)):
                        if kb.fileReadMode:
                            dataToStdout(_formatPartialContent(output).replace(r"\n", "\n").replace(r"\t", "\t"))
                        elif offset > 1:
@@ -366,7 +367,7 @@ def errorUse(expression, dump=False):
                    message = "due to huge table size do you want to remove "
                    message += "ORDER BY clause gaining speed over consistency? [y/N] "

-                    if readInput(message, default="N", boolean=True):
+                    if readInput(message, default='N', boolean=True):
                        expression = expression[:expression.index(" ORDER BY ")]

                numThreads = min(conf.threads, (stopLimit - startLimit))
@@ -163,7 +163,7 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=

    if retVal:
        infoMsg = "target URL appears to be UNION injectable with %d columns" % retVal
-        singleTimeLogMessage(infoMsg, logging.INFO, re.sub(r"\d+", "N", infoMsg))
+        singleTimeLogMessage(infoMsg, logging.INFO, re.sub(r"\d+", 'N', infoMsg))

    return retVal

@@ -201,7 +201,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
            if content and phrase in content:
                validPayload = payload
                kb.unionDuplicates = len(re.findall(phrase, content, re.I)) > 1
-                vector = (position, count, comment, prefix, suffix, kb.uChar, where, kb.unionDuplicates, False)
+                vector = (position, count, comment, prefix, suffix, kb.uChar, where, kb.unionDuplicates, conf.forcePartial)

                if where == PAYLOAD.WHERE.ORIGINAL:
                    # Prepare expression with delimiters
@@ -290,7 +290,7 @@ def _unionTestByCharBruteforce(comment, place, parameter, value, prefix, suffix)
            if not conf.uChar and count > 1 and kb.uChar == NULL:
                message = "injection not exploitable with NULL values. Do you want to try with a random integer value for option '--union-char'? [Y/n] "

-                if not readInput(message, default="Y", boolean=True):
+                if not readInput(message, default='Y', boolean=True):
                    warnMsg += "usage of option '--union-char' "
                    warnMsg += "(e.g. '--union-char=1') "
                else:
@@ -10,7 +10,6 @@ import re
 import time
 import xml.etree.ElementTree

-from extra.safe2bin.safe2bin import safecharencode
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
 from lib.core.common import arrayizeValue
@@ -62,6 +61,7 @@ from lib.core.threads import runThreads
 from lib.core.unescaper import unescaper
 from lib.request.connect import Connect as Request
 from lib.utils.progress import ProgressBar
+from lib.utils.safe2bin import safecharencode
 from thirdparty import six
 from thirdparty.odict import OrderedDict

@@ -241,7 +241,7 @@ def unionUse(expression, unpack=True, dump=False):
    # SQL limiting the query output one entry at a time
    # NOTE: we assume that only queries that get data from a table can
    # return multiple entries
-    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or kb.forcePartialUnion or (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or kb.forcePartialUnion or conf.forcePartial or (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
        expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)

        if limitCond:
@@ -60,6 +60,7 @@ def _addPageTextWords():

    return wordsList

+@stackedmethod
 def tableExists(tableFile, regex=None):
    if kb.tableExistsChoice is None and not any(_ for _ in kb.injection.data if _ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) and not conf.direct:
        warnMsg = "it's not recommended to use '%s' and/or '%s' " % (PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.TIME], PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.STACKED])
@@ -74,15 +75,17 @@ def tableExists(tableFile, regex=None):

    result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), randomStr())))

-    if conf.db and Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2):
-        conf.db = conf.db.upper()
-
    if result:
        errMsg = "can't use table existence check because of detected invalid results "
        errMsg += "(most likely caused by inability of the used injection "
        errMsg += "to distinguish erroneous results)"
        raise SqlmapDataException(errMsg)

+    pushValue(conf.db)
+
+    if conf.db and Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2):
+        conf.db = conf.db.upper()
+
    message = "which common tables (wordlist) file do you want to use?\n"
    message += "[1] default '%s' (press Enter)\n" % tableFile
    message += "[2] custom"
@@ -92,80 +95,88 @@ def tableExists(tableFile, regex=None):
        message = "what's the custom common tables file location?\n"
        tableFile = readInput(message) or tableFile

-    infoMsg = "checking table existence using items from '%s'" % tableFile
+    infoMsg = "performing table existence using items from '%s'" % tableFile
    logger.info(infoMsg)

    tables = getFileItems(tableFile, lowercase=Backend.getIdentifiedDbms() in (DBMS.ACCESS,), unique=True)
    tables.extend(_addPageTextWords())
    tables = filterListValue(tables, regex)

-    threadData = getCurrentThreadData()
-    threadData.shared.count = 0
-    threadData.shared.limit = len(tables)
-    threadData.shared.value = []
-    threadData.shared.unique = set()
+    for conf.db in (conf.db.split(',') if conf.db else [conf.db]):
+        if conf.db:
+            infoMsg = "checking database '%s'" % conf.db
+            logger.info(infoMsg)

-    def tableExistsThread():
        threadData = getCurrentThreadData()
+        threadData.shared.count = 0
+        threadData.shared.limit = len(tables)
+        threadData.shared.files = []
+        threadData.shared.unique = set()

-        while kb.threadContinue:
-            kb.locks.count.acquire()
-            if threadData.shared.count < threadData.shared.limit:
-                table = safeSQLIdentificatorNaming(tables[threadData.shared.count], True)
-                threadData.shared.count += 1
-                kb.locks.count.release()
-            else:
-                kb.locks.count.release()
-                break
+        def tableExistsThread():
+            threadData = getCurrentThreadData()

-            if conf.db and METADB_SUFFIX not in conf.db and Backend.getIdentifiedDbms() not in (DBMS.SQLITE, DBMS.ACCESS, DBMS.FIREBIRD):
-                fullTableName = "%s.%s" % (conf.db, table)
-            else:
-                fullTableName = table
+            while kb.threadContinue:
+                kb.locks.count.acquire()
+                if threadData.shared.count < threadData.shared.limit:
+                    table = safeSQLIdentificatorNaming(tables[threadData.shared.count], True)
+                    threadData.shared.count += 1
+                    kb.locks.count.release()
+                else:
+                    kb.locks.count.release()
+                    break

-            result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), fullTableName)))
+                if conf.db and METADB_SUFFIX not in conf.db and Backend.getIdentifiedDbms() not in (DBMS.SQLITE, DBMS.ACCESS, DBMS.FIREBIRD):
+                    fullTableName = "%s.%s" % (conf.db, table)
+                else:
+                    fullTableName = table

-            kb.locks.io.acquire()
+                result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), fullTableName)))

-            if result and table.lower() not in threadData.shared.unique:
-                threadData.shared.value.append(table)
-                threadData.shared.unique.add(table.lower())
+                kb.locks.io.acquire()

-                if conf.verbose in (1, 2) and not conf.api:
-                    clearConsoleLine(True)
-                    infoMsg = "[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), unsafeSQLIdentificatorNaming(table))
-                    dataToStdout(infoMsg, True)
+                if result and table.lower() not in threadData.shared.unique:
+                    threadData.shared.files.append(table)
+                    threadData.shared.unique.add(table.lower())

-            if conf.verbose in (1, 2):
-                status = '%d/%d items (%d%%)' % (threadData.shared.count, threadData.shared.limit, round(100.0 * threadData.shared.count / threadData.shared.limit))
-                dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)
+                    if conf.verbose in (1, 2) and not conf.api:
+                        clearConsoleLine(True)
+                        infoMsg = "[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), unsafeSQLIdentificatorNaming(table))
+                        dataToStdout(infoMsg, True)

-            kb.locks.io.release()
+                if conf.verbose in (1, 2):
+                    status = '%d/%d items (%d%%)' % (threadData.shared.count, threadData.shared.limit, round(100.0 * threadData.shared.count / threadData.shared.limit))
+                    dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)

-    try:
-        runThreads(conf.threads, tableExistsThread, threadChoice=True)
-    except KeyboardInterrupt:
-        warnMsg = "user aborted during table existence "
-        warnMsg += "check. sqlmap will display partial output"
-        logger.warn(warnMsg)
+                kb.locks.io.release()

-    clearConsoleLine(True)
-    dataToStdout("\n")
+        try:
+            runThreads(conf.threads, tableExistsThread, threadChoice=True)
+        except KeyboardInterrupt:
+            warnMsg = "user aborted during table existence "
+            warnMsg += "check. sqlmap will display partial output"
+            logger.warn(warnMsg)

-    if not threadData.shared.value:
-        warnMsg = "no table(s) found"
-        logger.warn(warnMsg)
-    else:
-        for item in threadData.shared.value:
-            if conf.db not in kb.data.cachedTables:
-                kb.data.cachedTables[conf.db] = [item]
-            else:
-                kb.data.cachedTables[conf.db].append(item)
+        clearConsoleLine(True)
+        dataToStdout("\n")

-    for _ in ((conf.db, item) for item in threadData.shared.value):
-        if _ not in kb.brute.tables:
-            kb.brute.tables.append(_)
+        if not threadData.shared.files:
+            warnMsg = "no table(s) found"
+            if conf.db:
+                 warnMsg += "for database '%s'" % conf.db
+            logger.warn(warnMsg)
+        else:
+            for item in threadData.shared.files:
+                if conf.db not in kb.data.cachedTables:
+                    kb.data.cachedTables[conf.db] = [item]
+                else:
+                    kb.data.cachedTables[conf.db].append(item)

+        for _ in ((conf.db, item) for item in threadData.shared.files):
+            if _ not in kb.brute.tables:
+                kb.brute.tables.append(_)
+
+    conf.db = popValue()
    hashDBWrite(HASHDB_KEYS.KB_BRUTE_TABLES, kb.brute.tables, True)

    return kb.data.cachedTables
@@ -224,7 +235,7 @@ def columnExists(columnFile, regex=None):
    threadData = getCurrentThreadData()
    threadData.shared.count = 0
    threadData.shared.limit = len(columns)
-    threadData.shared.value = []
+    threadData.shared.files = []

    def columnExistsThread():
        threadData = getCurrentThreadData()
@@ -244,7 +255,7 @@ def columnExists(columnFile, regex=None):
            kb.locks.io.acquire()

            if result:
-                threadData.shared.value.append(column)
+                threadData.shared.files.append(column)

                if conf.verbose in (1, 2) and not conf.api:
                    clearConsoleLine(True)
@@ -269,13 +280,13 @@ def columnExists(columnFile, regex=None):
    clearConsoleLine(True)
    dataToStdout("\n")

-    if not threadData.shared.value:
+    if not threadData.shared.files:
        warnMsg = "no column(s) found"
        logger.warn(warnMsg)
    else:
        columns = {}

-        for column in threadData.shared.value:
+        for column in threadData.shared.files:
            if Backend.getIdentifiedDbms() in (DBMS.MYSQL,):
                result = not inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s WHERE %s REGEXP '[^0-9]')", (column, table, column)))
            else:
@@ -327,7 +338,7 @@ def fileExists(pathFile):
    threadData = getCurrentThreadData()
    threadData.shared.count = 0
    threadData.shared.limit = len(paths)
-    threadData.shared.value = []
+    threadData.shared.files = []

    def fileExistsThread():
        threadData = getCurrentThreadData()
@@ -350,9 +361,9 @@ def fileExists(pathFile):
            kb.locks.io.acquire()

            if not isNoneValue(result):
-                threadData.shared.value.append(result)
+                threadData.shared.files.append(result)

-                if conf.verbose in (1, 2) and not conf.api:
+                if not conf.api:
                    clearConsoleLine(True)
                    infoMsg = "[%s] [INFO] retrieved: '%s'\n" % (time.strftime("%X"), path)
                    dataToStdout(infoMsg, True)
@@ -379,10 +390,10 @@ def fileExists(pathFile):
    clearConsoleLine(True)
    dataToStdout("\n")

-    if not threadData.shared.value:
+    if not threadData.shared.files:
        warnMsg = "no file(s) found"
        logger.warn(warnMsg)
    else:
-        retVal = threadData.shared.value
+        retVal = threadData.shared.files

    return retVal
@@ -92,8 +92,7 @@ def crawl(target):
                        soup = BeautifulSoup(content)
                        tags = soup('a')

-                        if not tags:
-                            tags = re.finditer(r'(?i)<a[^>]+href=["\'](?P<href>[^>"\']+)', content)
+                        tags += re.finditer(r'(?i)<a[^>]+href=["\'](?P<href>[^>"\']+)', content)

                        for tag in tags:
                            href = tag.get("href") if hasattr(tag, "get") else tag.group("href")
@@ -33,6 +33,7 @@ else:
 import base64
 import binascii
 import gc
+import hashlib
 import os
 import re
 import tempfile
@@ -155,7 +156,24 @@ def postgres_passwd(password, username, uppercase=False):

    return retVal.upper() if uppercase else retVal.lower()

-def mssql_passwd(password, salt, uppercase=False):
+def mssql_new_passwd(password, salt, uppercase=False):  # since version '2012'
+    """
+    Reference(s):
+        http://hashcat.net/forum/thread-1474.html
+        https://sqlity.net/en/2460/sql-password-hash/
+
+    >>> mssql_new_passwd(password='testpass', salt='4086ceb6', uppercase=False)
+    '0x02004086ceb6eb051cdbc5bdae68ffc66c918d4977e592f6bdfc2b444a7214f71fa31c35902c5b7ae773ed5f4c50676d329120ace32ee6bc81c24f70711eb0fc6400e85ebf25'
+    """
+
+    binsalt = decodeHex(salt)
+    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
+
+    retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest())
+
+    return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
+
+def mssql_passwd(password, salt, uppercase=False):  # versions '2005' and '2008'
    """
    Reference(s):
        http://www.leidecker.info/projects/phrasendrescher/mssql.c
@@ -172,7 +190,7 @@ def mssql_passwd(password, salt, uppercase=False):

    return "0x%s" % (retVal.upper() if uppercase else retVal.lower())

-def mssql_old_passwd(password, salt, uppercase=True):  # prior to version '2005'
+def mssql_old_passwd(password, salt, uppercase=True):  # version '2000' and before
    """
    Reference(s):
        www.exploit-db.com/download_pdf/15537/
@@ -190,22 +208,6 @@ def mssql_old_passwd(password, salt, uppercase=True):  # prior to version '2005'

    return "0x%s" % (retVal.upper() if uppercase else retVal.lower())

-def mssql_new_passwd(password, salt, uppercase=False):
-    """
-    Reference(s):
-        http://hashcat.net/forum/thread-1474.html
-
-    >>> mssql_new_passwd(password='testpass', salt='4086ceb6', uppercase=False)
-    '0x02004086ceb6eb051cdbc5bdae68ffc66c918d4977e592f6bdfc2b444a7214f71fa31c35902c5b7ae773ed5f4c50676d329120ace32ee6bc81c24f70711eb0fc6400e85ebf25'
-    """
-
-    binsalt = decodeHex(salt)
-    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
-
-    retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest())
-
-    return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
-
 def oracle_passwd(password, salt, uppercase=True):
    """
    Reference(s):
@@ -246,6 +248,18 @@ def oracle_old_passwd(password, username, uppercase=True):  # prior to version '

    return retVal.upper() if uppercase else retVal.lower()

+def md4_generic_passwd(password, uppercase=False):
+    """
+    >>> md4_generic_passwd(password='testpass', uppercase=False)
+    '5b4d300688f19c8fd65b8d6ccf98e0ae'
+    """
+
+    password = getBytes(password)
+
+    retVal = hashlib.new("md4", password).hexdigest()
+
+    return retVal.upper() if uppercase else retVal.lower()
+
 def md5_generic_passwd(password, uppercase=False):
    """
    >>> md5_generic_passwd(password='testpass', uppercase=False)
@@ -708,6 +722,7 @@ def attackDumpedTable():
                if hash_:
                    key = hash_ if hash_ not in replacements else replacements[hash_]
                    lut[key.lower()] = password
+                    lut["0x%s" % key.lower()] = password

            debugMsg = "post-processing table dump"
            logger.debug(debugMsg)
@@ -942,6 +957,8 @@ def dictionaryAttack(attack_dict):
                        if hash_regex in (HASH.MD5_BASE64, HASH.SHA1_BASE64, HASH.SHA256_BASE64, HASH.SHA512_BASE64):
                            item = [(user, encodeHex(decodeBase64(hash_, binary=True))), {}]
                        elif hash_regex in (HASH.MYSQL, HASH.MYSQL_OLD, HASH.MD5_GENERIC, HASH.SHA1_GENERIC, HASH.SHA224_GENERIC, HASH.SHA256_GENERIC, HASH.SHA384_GENERIC, HASH.SHA512_GENERIC, HASH.APACHE_SHA1):
+                            if hash_.startswith("0x"):  # Reference: https://docs.microsoft.com/en-us/sql/t-sql/functions/hashbytes-transact-sql?view=sql-server-2017
+                                hash_ = hash_[2:]
                            item = [(user, hash_), {}]
                        elif hash_regex in (HASH.SSHA,):
                            item = [(user, hash_), {"salt": decodeBase64(hash_, binary=True)[20:]}]
@@ -141,6 +141,8 @@ class HashDB(object):
                            self.cursor.execute("INSERT INTO storage VALUES (?, ?)", (hash_, value,))
                        except sqlite3.IntegrityError:
                            self.cursor.execute("UPDATE storage SET value=? WHERE id=?", (value, hash_,))
+                    except UnicodeError:  # e.g. surrogates not allowed (Issue #3851)
+                        break
                    except sqlite3.DatabaseError as ex:
                        if not os.path.exists(self.filepath):
                            debugMsg = "session file '%s' does not exist" % self.filepath
@@ -1,263 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
-See the file 'LICENSE' for copying permission
-"""
-
-# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html
-
-htmlEntities = {
-    "quot": 34,
-    "amp": 38,
-    "lt": 60,
-    "gt": 62,
-    "nbsp": 160,
-    "iexcl": 161,
-    "cent": 162,
-    "pound": 163,
-    "curren": 164,
-    "yen": 165,
-    "brvbar": 166,
-    "sect": 167,
-    "uml": 168,
-    "copy": 169,
-    "ordf": 170,
-    "laquo": 171,
-    "not": 172,
-    "shy": 173,
-    "reg": 174,
-    "macr": 175,
-    "deg": 176,
-    "plusmn": 177,
-    "sup2": 178,
-    "sup3": 179,
-    "acute": 180,
-    "micro": 181,
-    "para": 182,
-    "middot": 183,
-    "cedil": 184,
-    "sup1": 185,
-    "ordm": 186,
-    "raquo": 187,
-    "frac14": 188,
-    "frac12": 189,
-    "frac34": 190,
-    "iquest": 191,
-    "Agrave": 192,
-    "Aacute": 193,
-    "Acirc": 194,
-    "Atilde": 195,
-    "Auml": 196,
-    "Aring": 197,
-    "AElig": 198,
-    "Ccedil": 199,
-    "Egrave": 200,
-    "Eacute": 201,
-    "Ecirc": 202,
-    "Euml": 203,
-    "Igrave": 204,
-    "Iacute": 205,
-    "Icirc": 206,
-    "Iuml": 207,
-    "ETH": 208,
-    "Ntilde": 209,
-    "Ograve": 210,
-    "Oacute": 211,
-    "Ocirc": 212,
-    "Otilde": 213,
-    "Ouml": 214,
-    "times": 215,
-    "Oslash": 216,
-    "Ugrave": 217,
-    "Uacute": 218,
-    "Ucirc": 219,
-    "Uuml": 220,
-    "Yacute": 221,
-    "THORN": 222,
-    "szlig": 223,
-    "agrave": 224,
-    "aacute": 225,
-    "acirc": 226,
-    "atilde": 227,
-    "auml": 228,
-    "aring": 229,
-    "aelig": 230,
-    "ccedil": 231,
-    "egrave": 232,
-    "eacute": 233,
-    "ecirc": 234,
-    "euml": 235,
-    "igrave": 236,
-    "iacute": 237,
-    "icirc": 238,
-    "iuml": 239,
-    "eth": 240,
-    "ntilde": 241,
-    "ograve": 242,
-    "oacute": 243,
-    "ocirc": 244,
-    "otilde": 245,
-    "ouml": 246,
-    "divide": 247,
-    "oslash": 248,
-    "ugrave": 249,
-    "uacute": 250,
-    "ucirc": 251,
-    "uuml": 252,
-    "yacute": 253,
-    "thorn": 254,
-    "yuml": 255,
-    "OElig": 338,
-    "oelig": 339,
-    "Scaron": 352,
-    "fnof": 402,
-    "scaron": 353,
-    "Yuml": 376,
-    "circ": 710,
-    "tilde": 732,
-    "Alpha": 913,
-    "Beta": 914,
-    "Gamma": 915,
-    "Delta": 916,
-    "Epsilon": 917,
-    "Zeta": 918,
-    "Eta": 919,
-    "Theta": 920,
-    "Iota": 921,
-    "Kappa": 922,
-    "Lambda": 923,
-    "Mu": 924,
-    "Nu": 925,
-    "Xi": 926,
-    "Omicron": 927,
-    "Pi": 928,
-    "Rho": 929,
-    "Sigma": 931,
-    "Tau": 932,
-    "Upsilon": 933,
-    "Phi": 934,
-    "Chi": 935,
-    "Psi": 936,
-    "Omega": 937,
-    "alpha": 945,
-    "beta": 946,
-    "gamma": 947,
-    "delta": 948,
-    "epsilon": 949,
-    "zeta": 950,
-    "eta": 951,
-    "theta": 952,
-    "iota": 953,
-    "kappa": 954,
-    "lambda": 955,
-    "mu": 956,
-    "nu": 957,
-    "xi": 958,
-    "omicron": 959,
-    "pi": 960,
-    "rho": 961,
-    "sigmaf": 962,
-    "sigma": 963,
-    "tau": 964,
-    "upsilon": 965,
-    "phi": 966,
-    "chi": 967,
-    "psi": 968,
-    "omega": 969,
-    "thetasym": 977,
-    "upsih": 978,
-    "piv": 982,
-    "bull": 8226,
-    "hellip": 8230,
-    "prime": 8242,
-    "Prime": 8243,
-    "oline": 8254,
-    "frasl": 8260,
-    "ensp": 8194,
-    "emsp": 8195,
-    "thinsp": 8201,
-    "zwnj": 8204,
-    "zwj": 8205,
-    "lrm": 8206,
-    "rlm": 8207,
-    "ndash": 8211,
-    "mdash": 8212,
-    "lsquo": 8216,
-    "rsquo": 8217,
-    "sbquo": 8218,
-    "ldquo": 8220,
-    "rdquo": 8221,
-    "bdquo": 8222,
-    "dagger": 8224,
-    "Dagger": 8225,
-    "permil": 8240,
-    "lsaquo": 8249,
-    "rsaquo": 8250,
-    "euro": 8364,
-    "weierp": 8472,
-    "image": 8465,
-    "real": 8476,
-    "trade": 8482,
-    "alefsym": 8501,
-    "larr": 8592,
-    "uarr": 8593,
-    "rarr": 8594,
-    "darr": 8595,
-    "harr": 8596,
-    "crarr": 8629,
-    "lArr": 8656,
-    "uArr": 8657,
-    "rArr": 8658,
-    "dArr": 8659,
-    "hArr": 8660,
-    "forall": 8704,
-    "part": 8706,
-    "exist": 8707,
-    "empty": 8709,
-    "nabla": 8711,
-    "isin": 8712,
-    "notin": 8713,
-    "ni": 8715,
-    "prod": 8719,
-    "sum": 8721,
-    "minus": 8722,
-    "lowast": 8727,
-    "radic": 8730,
-    "prop": 8733,
-    "infin": 8734,
-    "ang": 8736,
-    "and": 8743,
-    "or": 8744,
-    "cap": 8745,
-    "cup": 8746,
-    "int": 8747,
-    "there4": 8756,
-    "sim": 8764,
-    "cong": 8773,
-    "asymp": 8776,
-    "ne": 8800,
-    "equiv": 8801,
-    "le": 8804,
-    "ge": 8805,
-    "sub": 8834,
-    "sup": 8835,
-    "nsub": 8836,
-    "sube": 8838,
-    "supe": 8839,
-    "oplus": 8853,
-    "otimes": 8855,
-    "perp": 8869,
-    "sdot": 8901,
-    "lceil": 8968,
-    "rceil": 8969,
-    "lfloor": 8970,
-    "rfloor": 8971,
-    "lang": 9001,
-    "rang": 9002,
-    "loz": 9674,
-    "spades": 9824,
-    "clubs": 9827,
-    "hearts": 9829,
-    "diams": 9830,
-}
@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+See the file 'LICENSE' for copying permission
+"""
+
+from __future__ import print_function
+
+import mimetypes
+import gzip
+import os
+import re
+import sys
+import threading
+import time
+import traceback
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from lib.core.enums import HTTP_HEADER
+from lib.core.settings import UNICODE_ENCODING
+from lib.core.settings import VERSION_STRING
+from thirdparty import six
+from thirdparty.six.moves import BaseHTTPServer as _BaseHTTPServer
+from thirdparty.six.moves import http_client as _http_client
+from thirdparty.six.moves import socketserver as _socketserver
+from thirdparty.six.moves import urllib as _urllib
+
+HTTP_ADDRESS = "0.0.0.0"
+HTTP_PORT = 8951
+DEBUG = True
+HTML_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "data", "html"))
+DISABLED_CONTENT_EXTENSIONS = (".py", ".pyc", ".md", ".txt", ".bak", ".conf", ".zip", "~")
+
+class ThreadingServer(_socketserver.ThreadingMixIn, _BaseHTTPServer.HTTPServer):
+    def finish_request(self, *args, **kwargs):
+        try:
+            _BaseHTTPServer.HTTPServer.finish_request(self, *args, **kwargs)
+        except Exception:
+            if DEBUG:
+                traceback.print_exc()
+
+class ReqHandler(_BaseHTTPServer.BaseHTTPRequestHandler):
+    def do_GET(self):
+        path, query = self.path.split('?', 1) if '?' in self.path else (self.path, "")
+        params = {}
+        content = None
+
+        if query:
+            params.update(_urllib.parse.parse_qs(query))
+
+        for key in params:
+            if params[key]:
+                params[key] = params[key][-1]
+
+        self.url, self.params = path, params
+
+        if path == '/':
+            path = "index.html"
+
+        path = path.strip('/')
+
+        path = path.replace('/', os.path.sep)
+        path = os.path.abspath(os.path.join(HTML_DIR, path)).strip()
+
+        if not os.path.isfile(path) and os.path.isfile("%s.html" % path):
+            path = "%s.html" % path
+
+        if ".." not in os.path.relpath(path, HTML_DIR) and os.path.isfile(path) and not path.endswith(DISABLED_CONTENT_EXTENSIONS):
+            content = open(path, "rb").read()
+            self.send_response(_http_client.OK)
+            self.send_header(HTTP_HEADER.CONNECTION, "close")
+            self.send_header(HTTP_HEADER.CONTENT_TYPE, mimetypes.guess_type(path)[0] or "application/octet-stream")
+        else:
+            content = ("<!DOCTYPE html><html lang=\"en\"><head><title>404 Not Found</title></head><body><h1>Not Found</h1><p>The requested URL %s was not found on this server.</p></body></html>" % self.path.split('?')[0]).encode(UNICODE_ENCODING)
+            self.send_response(_http_client.NOT_FOUND)
+            self.send_header(HTTP_HEADER.CONNECTION, "close")
+
+        if content is not None:
+            for match in re.finditer(b"<!(\\w+)!>", content):
+                name = match.group(1)
+                _ = getattr(self, "_%s" % name.lower(), None)
+                if _:
+                    content = self._format(content, **{name: _()})
+
+            if "gzip" in self.headers.get(HTTP_HEADER.ACCEPT_ENCODING):
+                self.send_header(HTTP_HEADER.CONTENT_ENCODING, "gzip")
+                _ = six.BytesIO()
+                compress = gzip.GzipFile("", "w+b", 9, _)
+                compress._stream = _
+                compress.write(content)
+                compress.flush()
+                compress.close()
+                content = compress._stream.getvalue()
+
+            self.send_header(HTTP_HEADER.CONTENT_LENGTH, str(len(content)))
+
+        self.end_headers()
+
+        if content:
+            self.wfile.write(content)
+
+        self.wfile.flush()
+
+    def _format(self, content, **params):
+        if content:
+            for key, value in params.items():
+                content = content.replace("<!%s!>" % key, value)
+
+        return content
+
+    def version_string(self):
+        return VERSION_STRING
+
+    def log_message(self, format, *args):
+        return
+
+    def finish(self):
+        try:
+            _BaseHTTPServer.BaseHTTPRequestHandler.finish(self)
+        except Exception:
+            if DEBUG:
+                traceback.print_exc()
+
+def start_httpd():
+    server = ThreadingServer((HTTP_ADDRESS, HTTP_PORT), ReqHandler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.daemon = True
+    thread.start()
+
+    print("[i] running HTTP server at '%s:%d'" % (HTTP_ADDRESS, HTTP_PORT))
+
+if __name__ == "__main__":
+    try:
+        start_httpd()
+
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        pass
@@ -7,7 +7,6 @@ See the file 'LICENSE' for copying permission

 import re

-from extra.safe2bin.safe2bin import safechardecode
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
 from lib.core.common import Backend
@@ -33,6 +32,7 @@ from lib.core.settings import MAX_INT
 from lib.core.settings import NULL
 from lib.core.unescaper import unescaper
 from lib.request import inject
+from lib.utils.safe2bin import safechardecode
 from thirdparty.six import unichr as _unichr

 def pivotDumpTable(table, colList, count=None, blind=True, alias=None):
@@ -15,6 +15,7 @@ import string
 from lib.core.common import getSafeExString
 from lib.core.common import openFile
 from lib.core.compat import xrange
+from lib.core.convert import getUnicode
 from lib.core.data import logger
 from thirdparty.six import unichr as _unichr

@@ -84,4 +85,4 @@ def purge(directory):
    try:
        shutil.rmtree(directory)
    except OSError as ex:
-        logger.error("problem occurred while removing directory '%s' ('%s')" % (directory, getSafeExString(ex)))
+        logger.error("problem occurred while removing directory '%s' ('%s')" % (getUnicode(directory), getSafeExString(ex)))
@@ -1,23 +1,15 @@
 #!/usr/bin/env python

 """
-safe2bin.py - Simple safe(hex) to binary format converter
-
 Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """

-from __future__ import print_function
-
 import binascii
 import re
 import string
-import os
 import sys

-from optparse import OptionError
-from optparse import OptionParser
-
 if sys.version_info >= (3, 0):
    xrange = range
    text_type = str
@@ -49,10 +41,10 @@ def safecharencode(value):
    """
    Returns safe representation of a given basestring value

-    >>> safecharencode(u'test123')
-    u'test123'
-    >>> safecharencode(u'test\x01\x02\xff')
-    u'test\\01\\02\\03\\ff'
+    >>> safecharencode(u'test123') == u'test123'
+    True
+    >>> safecharencode(u'test\x01\x02\xaf') == u'test\\\\x01\\\\x02\\xaf'
+    True
    """

    retVal = value
@@ -107,37 +99,3 @@ def safechardecode(value, binary=False):
            retVal[i] = safechardecode(value[i])

    return retVal
-
-def main():
-    usage = '%s -i <input file> [-o <output file>]' % sys.argv[0]
-    parser = OptionParser(usage=usage, version='0.1')
-
-    try:
-        parser.add_option('-i', dest='inputFile', help='Input file')
-        parser.add_option('-o', dest='outputFile', help='Output file')
-
-        (args, _) = parser.parse_args()
-
-        if not args.inputFile:
-            parser.error('Missing the input file, -h for help')
-
-    except (OptionError, TypeError) as ex:
-        parser.error(ex)
-
-    if not os.path.isfile(args.inputFile):
-        print('ERROR: the provided input file \'%s\' is not a regular file' % args.inputFile)
-        sys.exit(1)
-
-    f = open(args.inputFile, 'r')
-    data = f.read()
-    f.close()
-
-    if not args.outputFile:
-        args.outputFile = args.inputFile + '.bin'
-
-    f = open(args.outputFile, 'wb')
-    f.write(safechardecode(data))
-    f.close()
-
-if __name__ == '__main__':
-    main()
@@ -76,7 +76,8 @@ class SQLAlchemy(GenericConnector):
                        pass
                elif "invalid literal for int() with base 10: '0b" in traceback.format_exc():
                    raise SqlmapConnectionException("SQLAlchemy connection issue ('https://bitbucket.org/zzzeek/sqlalchemy/issues/3975')")
-                raise
+                else:
+                    pass
            except SqlmapFilePathException:
                raise
            except Exception as ex:
@@ -10,6 +10,7 @@ import re
 from lib.core.common import Backend
 from lib.core.common import Format
 from lib.core.common import randomRange
+from lib.core.common import randomStr
 from lib.core.compat import xrange
 from lib.core.convert import getUnicode
 from lib.core.data import conf
@@ -74,13 +75,14 @@ class Fingerprint(GenericFingerprint):
            ("1.5", ("NULLIF(%d,%d) IS NULL", "EXISTS(SELECT CURRENT_TRANSACTION FROM RDB$DATABASE)")),
            ("2.0", ("EXISTS(SELECT CURRENT_TIME(0) FROM RDB$DATABASE)", "BIT_LENGTH(%d)>0", "CHAR_LENGTH(%d)>0")),
            ("2.1", ("BIN_XOR(%d,%d)=0", "PI()>0.%d", "RAND()<1.%d", "FLOOR(1.%d)>=0")),
-            # TODO: add test for Firebird 2.5
+            ("2.5", ("'%s' SIMILAR TO '%s'",)),  # Reference: https://firebirdsql.org/refdocs/langrefupd25-similar-to.html
+            ("3.0", ("FALSE IS FALSE",)),  # https://www.firebirdsql.org/file/community/conference-2014/pdf/02_fb.2014.whatsnew.30.en.pdf
        )

        for i in xrange(len(table)):
            version, checks = table[i]
            failed = False
-            check = checks[randomRange(0, len(checks) - 1)].replace("%d", getUnicode(randomRange(1, 100)))
+            check = checks[randomRange(0, len(checks) - 1)].replace("%d", getUnicode(randomRange(1, 100))).replace("%s", getUnicode(randomStr()))
            result = inject.checkBooleanExpression(check)

            if result:
@@ -96,7 +96,7 @@ class Fingerprint(GenericFingerprint):
            infoMsg = "actively fingerprinting %s" % DBMS.INFORMIX
            logger.info(infoMsg)

-            for version in ("12.1", "11.7", "11.5"):
+            for version in ("14.1", "12.1", "11.7", "11.5", "10.0"):
                output = inject.checkBooleanExpression("EXISTS(SELECT 1 FROM SYSMASTER:SYSDUAL WHERE DBINFO('VERSION,'FULL') LIKE '%%%s%%')" % version)

                if output:
@@ -52,7 +52,7 @@ class Filesystem(GenericFilesystem):
                else:
                    scrString += " %s" % strLineChar

-                lineAddr += len(lineChar)
+                lineAddr += len(strLineChar) // 2

            fileLines.append(scrString)

@@ -94,7 +94,8 @@ class Fingerprint(GenericFingerprint):
                ("2008", "SYSDATETIME()=SYSDATETIME()"),
                ("2012", "CONCAT(NULL,NULL)=CONCAT(NULL,NULL)"),
                ("2014", "CHARINDEX('12.0.2000',@@version)>0"),
-                ("2016", "ISJSON(NULL) IS NULL")
+                ("2016", "ISJSON(NULL) IS NULL"),
+                ("2017", "TRIM(NULL) IS NULL")
            ):
                result = inject.checkBooleanExpression(check)

@@ -53,7 +53,7 @@ class Fingerprint(GenericFingerprint):
            (50600, 50646),  # MySQL 5.6
            (50700, 50726),  # MySQL 5.7
            (60000, 60014),  # MySQL 6.0
-            (80000, 80015),  # MySQL 8.0
+            (80000, 80017),  # MySQL 8.0
        )

        index = -1
@@ -21,6 +21,7 @@ from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapNoneDataException
+from lib.core.settings import CURRENT_USER
 from lib.request import inject
 from plugins.generic.enumeration import Enumeration as GenericEnumeration

@@ -30,7 +31,7 @@ class Enumeration(GenericEnumeration):

        rootQuery = queries[DBMS.ORACLE].roles

-        if conf.user == "CU":
+        if conf.user == CURRENT_USER:
            infoMsg += " for current user"
            conf.user = self.getCurrentUser()

@@ -55,7 +56,7 @@ class Enumeration(GenericEnumeration):
            values = inject.getValue(query, blind=False, time=False)

            if not values and not query2:
-                infoMsg = "trying with table USER_ROLE_PRIVS"
+                infoMsg = "trying with table 'USER_ROLE_PRIVS'"
                logger.info(infoMsg)

                return self.getRoles(query2=True)
@@ -116,7 +117,7 @@ class Enumeration(GenericEnumeration):

                if not isNumPosStrValue(count):
                    if count != 0 and not query2:
-                        infoMsg = "trying with table USER_SYS_PRIVS"
+                        infoMsg = "trying with table 'USER_SYS_PRIVS'"
                        logger.info(infoMsg)

                        return self.getPrivileges(query2=True)
@@ -105,7 +105,7 @@ class Fingerprint(GenericFingerprint):
            logger.info(infoMsg)

            # Reference: https://en.wikipedia.org/wiki/Oracle_Database
-            for version in ("18c", "12c", "11g", "10g", "9i", "8i"):
+            for version in ("18c", "12c", "11g", "10g", "9i", "8i", "7"):
                number = int(re.search(r"([\d]+)", version).group(1))
                output = inject.checkBooleanExpression("%d=(SELECT SUBSTR((VERSION),1,%d) FROM SYS.PRODUCT_COMPONENT_VERSION WHERE ROWNUM=1)" % (number, 1 if number < 10 else 2))

@@ -10,6 +10,7 @@ import os
 from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import decloakToTemp
+from lib.core.common import flattenValue
 from lib.core.common import isListLike
 from lib.core.common import isStackingAvailable
 from lib.core.common import randomStr
@@ -104,7 +105,7 @@ class Takeover(GenericTakeover):
            output = inject.getValue(query, resumeValue=False)

            if isListLike(output):
-                output = os.linesep.join(output)
+                output = os.linesep.join(flattenValue(output))

            self._cleanupCmd = "DROP TABLE %s" % self.cmdTblName
            inject.goStacked(self._cleanupCmd)
@@ -21,6 +21,8 @@ from lib.core.common import isStackingAvailable
 from lib.core.common import isTechniqueAvailable
 from lib.core.common import readInput
 from lib.core.compat import xrange
+from lib.core.convert import encodeBase64
+from lib.core.convert import encodeHex
 from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.data import conf
@@ -31,7 +33,6 @@ from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapUndefinedMethod
-from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.core.settings import UNICODE_ENCODING
 from lib.request import inject

@@ -41,7 +42,7 @@ class Filesystem(object):
    """

    def __init__(self):
-        self.fileTblName = "%sfile" % TAKEOVER_TABLE_PREFIX
+        self.fileTblName = "%sfile" % conf.tablePrefix
        self.tblField = "data"

    def _checkFileLength(self, localFile, remoteFile, fileRead=False):
@@ -134,8 +135,14 @@ class Filesystem(object):
    def fileContentEncode(self, content, encoding, single, chunkSize=256):
        retVal = []

-        if encoding:
-            content = getText(codecs.encode(content, encoding)).replace("\n", "")
+        if encoding == "hex":
+            content = encodeHex(content)
+        elif encoding == "base64":
+            content = encodeBase64(content)
+        else:
+            content = codecs.encode(content, encoding)
+
+        content = getText(content).replace("\n", "")

        if not single:
            if len(content) > chunkSize:
@@ -10,6 +10,7 @@ import os
 from lib.core.common import Backend
 from lib.core.common import getSafeExString
 from lib.core.common import isStackingAvailable
+from lib.core.common import openFile
 from lib.core.common import readInput
 from lib.core.common import runningAsAdmin
 from lib.core.data import conf
@@ -24,7 +25,6 @@ from lib.core.exception import SqlmapNotVulnerableException
 from lib.core.exception import SqlmapSystemException
 from lib.core.exception import SqlmapUndefinedMethod
 from lib.core.exception import SqlmapUnsupportedDBMSException
-from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.takeover.abstraction import Abstraction
 from lib.takeover.icmpsh import ICMPsh
 from lib.takeover.metasploit import Metasploit
@@ -36,7 +36,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
    """

    def __init__(self):
-        self.cmdTblName = ("%soutput" % TAKEOVER_TABLE_PREFIX)
+        self.cmdTblName = ("%soutput" % conf.tablePrefix)
        self.tblField = "data"

        Abstraction.__init__(self)
@@ -137,7 +137,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):

            if os.path.exists(filename):
                try:
-                    with open(filename, "wb") as f:
+                    with openFile(filename, "wb") as f:
                        f.write("1")
                except IOError as ex:
                    errMsg = "there has been a file opening/writing error "
@@ -39,6 +39,7 @@ from lib.core.enums import EXPECTED
 from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.settings import CURRENT_USER
 from lib.core.threads import getCurrentThreadData
 from lib.request import inject
 from lib.utils.hash import attackCachedUsersPasswords
@@ -153,7 +154,7 @@ class Users(object):

        rootQuery = queries[Backend.getIdentifiedDbms()].passwords

-        if conf.user == "CU":
+        if conf.user == CURRENT_USER:
            infoMsg += " for current user"
            conf.user = self.getCurrentUser()

@@ -362,7 +363,7 @@ class Users(object):

        rootQuery = queries[Backend.getIdentifiedDbms()].privileges

-        if conf.user == "CU":
+        if conf.user == CURRENT_USER:
            infoMsg += " for current user"
            conf.user = self.getCurrentUser()

@@ -410,7 +411,7 @@ class Users(object):
            values = inject.getValue(query, blind=False, time=False)

            if not values and Backend.isDbms(DBMS.ORACLE) and not query2:
-                infoMsg = "trying with table USER_SYS_PRIVS"
+                infoMsg = "trying with table 'USER_SYS_PRIVS'"
                logger.info(infoMsg)

                return self.getPrivileges(query2=True)
@@ -446,7 +447,7 @@ class Users(object):
                            # In MySQL < 5.0 we get Y if the privilege is
                            # True, N otherwise
                            elif Backend.isDbms(DBMS.MYSQL) and not kb.data.has_information_schema:
-                                if privilege.upper() == "Y":
+                                if privilege.upper() == 'Y':
                                    privileges.add(MYSQL_PRIVS[count])

                            # In Firebird we get one letter for each privilege
@@ -465,7 +466,7 @@ class Users(object):
                                    i = 1

                                    for priv in privs:
-                                        if priv.upper() in ("Y", "G"):
+                                        if priv.upper() in ('Y', 'G'):
                                            for position, db2Priv in DB2_PRIVS.items():
                                                if position == i:
                                                    privilege += ", " + db2Priv
@@ -525,7 +526,7 @@ class Users(object):

                    if not isNumPosStrValue(count):
                        if not retrievedUsers and Backend.isDbms(DBMS.ORACLE) and not query2:
-                            infoMsg = "trying with table USER_SYS_PRIVS"
+                            infoMsg = "trying with table 'USER_SYS_PRIVS'"
                            logger.info(infoMsg)

                            return self.getPrivileges(query2=True)
@@ -67,6 +67,10 @@ dropSetCookie = False
 # sqlmap will also test for SQL injection on the HTTP User-Agent value.
 agent =

+# Imitate smartphone through HTTP User-Agent header.
+# Valid: True or False
+mobile = False
+
 # Use randomly selected HTTP User-Agent header value.
 # Valid: True or False
 randomAgent = False
@@ -176,16 +180,19 @@ safeReqFile =
 # Default: 0
 safeFreq = 0

-# Skip URL encoding of payload data
+# Skip URL encoding of payload data.
 # Valid: True or False
 skipUrlEncode = False

-# Parameter used to hold anti-CSRF token
+# Parameter used to hold anti-CSRF token.
 csrfToken = 

 # URL address to visit to extract anti-CSRF token
 csrfUrl = 

+# HTTP method to use during anti-CSRF token page visit.
+csrfMethod =
+
 # Force usage of SSL/HTTPS
 # Valid: True or False
 forceSSL = False
@@ -343,6 +350,10 @@ regexp =
 # code)
 # code = 

+# Conduct thorough tests only if positive heuristic(s).
+# Valid: True or False
+smart = False
+
 # Compare pages based only on the textual content.
 # Valid: True or False
 textOnly = False
@@ -683,6 +694,9 @@ sessionFile =
 # Log all HTTP traffic into a textual file.
 trafficFile = 

+# Set predefined answers (e.g. "quit=N,follow=N").
+answers =
+
 # Never ask for user input, use the default behaviour.
 # Valid: True or False
 batch = False
@@ -693,6 +707,10 @@ binaryFields =
 # Check Internet connection before assessing the target.
 checkInternet = False

+# Clean up the DBMS from sqlmap specific UDF and tables.
+# Valid: True or False
+cleanup = False
+
 # Crawl the website starting from the target URL.
 # Valid: integer
 # Default: 0
@@ -729,6 +747,11 @@ forms = False
 # Valid: True or False
 freshQueries = False

+# Use Google dork results from specified page number.
+# Valid: integer
+# Default: 1
+googlePage = 1
+
 # Use hex conversion during data retrieval.
 # Valid: True or False
 hexConvert = False
@@ -752,15 +775,22 @@ repair = False
 # Example: (google|yahoo)
 scope = 

+# Skip heuristic detection of WAF/IPS protection.
+# Valid: True or False
+skipWaf = False
+
+# Prefix used for temporary tables.
+# Default: sqlmap
+tablePrefix = sqlmap
+
 # Select tests by payloads and/or titles (e.g. ROW)
 testFilter =

 # Skip tests by payloads and/or titles (e.g. BENCHMARK)
 testSkip =

-# Update sqlmap.
-# Valid: True or False
-updateAll = False
+# Web server document root directory (e.g. "/var/www").
+webRoot =


 [Miscellaneous]
@@ -768,9 +798,6 @@ updateAll = False
 # Run host OS command(s) when SQL injection is found.
 alert =

-# Set predefined answers (e.g. "quit=N,follow=N").
-answers =
-
 # Beep on question and/or when SQL injection is found.
 # Valid: True or False
 beep = False
@@ -779,10 +806,6 @@ beep = False
 # Valid: True or False
 checkPayload = False

-# Clean up the DBMS from sqlmap specific UDF and tables.
-# Valid: True or False
-cleanup = False
-
 # Check for missing (optional) sqlmap dependencies.
 # Valid: True or False
 dependencies = False
@@ -791,41 +814,25 @@ dependencies = False
 # Valid: True or False
 disableColoring = False

-# Use Google dork results from specified page number.
-# Valid: integer
-# Default: 1
-googlePage = 1
-
 # Display list of available tamper scripts
 # Valid: True or False
 listTampers = False

-# Imitate smartphone through HTTP User-Agent header.
-# Valid: True or False
-mobile = False
-
 # Work in offline mode (only use session data)
 # Valid: True or False
 offline = False

-# Skip heuristic detection of WAF/IPS protection.
-# Valid: True or False
-skipWaf = False
-
-# Conduct thorough tests only if positive heuristic(s).
-# Valid: True or False
-smart = False
-
 # Local directory for storing temporary files.
 tmpDir =

-# Web server document root directory (e.g. "/var/www").
-webRoot =
-
 # Simple wizard interface for beginner users.
 # Valid: True or False
 wizard = False

+# Update sqlmap.
+# Valid: True or False
+updateAll = False
+
 # Verbosity level.
 # Valid: integer between 0 and 6
 # 0: Show only error and critical messages
@@ -238,7 +238,7 @@ def main():
            dataToStdout(excMsg)
            raise SystemExit

-        elif any(_ in excMsg for _ in ("ImportError", "Can't find file for module")):
+        elif any(_ in excMsg for _ in ("ImportError", "ModuleNotFoundError", "Can't find file for module")):
            errMsg = "invalid runtime environment ('%s')" % excMsg.split("Error: ")[-1].strip()
            logger.critical(errMsg)
            raise SystemExit
@@ -263,6 +263,11 @@ def main():
            logger.critical(errMsg)
            raise SystemExit

+        elif all(_ in excMsg for _ in ("Access is denied", "subprocess", "metasploit")):
+            errMsg = "permission error occurred while running Metasploit"
+            logger.critical(errMsg)
+            raise SystemExit
+
        elif all(_ in excMsg for _ in ("No such file", "_'")):
            errMsg = "corrupted installation detected ('%s'). " % excMsg.strip().split('\n')[-1]
            errMsg += "You should retrieve the latest development version from official GitHub "
@@ -330,6 +335,12 @@ def main():
            logger.critical(errMsg)
            raise SystemExit

+        elif "can't allocate read lock" in excMsg:
+            errMsg = "there has been a problem in regular socket operation "
+            errMsg += "('%s')" % excMsg.strip().split('\n')[-1]
+            logger.critical(errMsg)
+            raise SystemExit
+
        elif "'DictObject' object has no attribute '" in excMsg and all(_ in errMsg for _ in ("(fingerprinted)", "(identified)")):
            errMsg = "there has been a problem in enumeration. "
            errMsg += "Because of a considerable chance of false-positive case "
@@ -10,7 +10,6 @@ import re

 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import zeroDepthSearch
-from lib.core.compat import xrange
 from lib.core.enums import DBMS
 from lib.core.enums import PRIORITY

@@ -35,51 +34,22 @@ def tamper(payload, **kwargs):
    >>> tamper('SELECT CHAR(113)+CHAR(114)+CHAR(115) FROM DUAL')
    'SELECT CONCAT(CHAR(113),CHAR(114),CHAR(115)) FROM DUAL'

-    >>> tamper('SELECT (CHAR(113)+CHAR(114)+CHAR(115)) FROM DUAL')
-    'SELECT CONCAT(CHAR(113),CHAR(114),CHAR(115)) FROM DUAL'
+    >>> tamper('1 UNION ALL SELECT NULL,NULL,CHAR(113)+CHAR(118)+CHAR(112)+CHAR(112)+CHAR(113)+ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32))+CHAR(113)+CHAR(112)+CHAR(107)+CHAR(112)+CHAR(113)-- qtfe')
+    '1 UNION ALL SELECT NULL,NULL,CONCAT(CHAR(113),CHAR(118),CHAR(112),CHAR(112),CHAR(113),ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32)),CHAR(113),CHAR(112),CHAR(107),CHAR(112),CHAR(113))-- qtfe'
    """

    retVal = payload

    if payload:
-        prefix, suffix = '+' * len(re.search(r"\A(\+*)", payload).group(0)), '+' * len(re.search(r"(\+*)\Z", payload).group(0))
-        retVal = retVal.strip('+')
+        match = re.search(r"('[^']+'|CHAR\(\d+\))\+.*(?<=\+)('[^']+'|CHAR\(\d+\))", retVal)
+        if match:
+            part = match.group(0)

-        while True:
-            indexes = zeroDepthSearch(retVal, '+')
+            chars = [char for char in part]
+            for index in zeroDepthSearch(part, '+'):
+                chars[index] = ','

-            if indexes:
-                first, last = 0, 0
-                for i in xrange(1, len(indexes)):
-                    if ' ' in retVal[indexes[0]:indexes[i]]:
-                        break
-                    else:
-                        last = i
-
-                start = retVal[:indexes[first]].rfind(' ') + 1
-                end = (retVal[indexes[last] + 1:].find(' ') + indexes[last] + 1) if ' ' in retVal[indexes[last] + 1:] else len(retVal) - 1
-
-                chars = [char for char in retVal]
-                for index in indexes[first:last + 1]:
-                    chars[index] = ','
-
-                retVal = "%sCONCAT(%s)%s" % (retVal[:start], ''.join(chars)[start:end], retVal[end:])
-            else:
-                match = re.search(r"\((CHAR\(\d+.+\bCHAR\(\d+\))\)", retVal)
-                if match:
-                    part = match.group(0)
-                    indexes = set(zeroDepthSearch(match.group(1), '+'))
-                    if not indexes:
-                        break
-                    chars = [char for char in part]
-                    for i in xrange(1, len(chars)):
-                        if i - 1 in indexes:
-                            chars[i] = ','
-                    replacement = "CONCAT%s" % "".join(chars)
-                    retVal = retVal.replace(part, replacement)
-                else:
-                    break
-
-        retVal = "%s%s%s" % (prefix, retVal, suffix)
+            replacement = "CONCAT(%s)" % "".join(chars)
+            retVal = retVal.replace(part, replacement)

    return retVal
@@ -36,63 +36,29 @@ def tamper(payload, **kwargs):
    >>> tamper('SELECT CHAR(113)+CHAR(114)+CHAR(115) FROM DUAL')
    'SELECT {fn CONCAT({fn CONCAT(CHAR(113),CHAR(114))},CHAR(115))} FROM DUAL'

-    >>> tamper('SELECT (CHAR(113)+CHAR(114)+CHAR(115)) FROM DUAL')
-    'SELECT {fn CONCAT({fn CONCAT(CHAR(113),CHAR(114))},CHAR(115))} FROM DUAL'
+    >>> tamper('1 UNION ALL SELECT NULL,NULL,CHAR(113)+CHAR(118)+CHAR(112)+CHAR(112)+CHAR(113)+ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32))+CHAR(113)+CHAR(112)+CHAR(107)+CHAR(112)+CHAR(113)-- qtfe')
+    '1 UNION ALL SELECT NULL,NULL,{fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT(CHAR(113),CHAR(118))},CHAR(112))},CHAR(112))},CHAR(113))},ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32)))},CHAR(113))},CHAR(112))},CHAR(107))},CHAR(112))},CHAR(113))}-- qtfe'
    """

    retVal = payload

    if payload:
-        prefix, suffix = '+' * len(re.search(r"\A(\+*)", payload).group(0)), '+' * len(re.search(r"(\+*)\Z", payload).group(0))
-        retVal = retVal.strip('+')
+        match = re.search(r"('[^']+'|CHAR\(\d+\))\+.*(?<=\+)('[^']+'|CHAR\(\d+\))", retVal)
+        if match:
+            old = match.group(0)
+            parts = []
+            last = 0

-        while True:
-            indexes = zeroDepthSearch(retVal, '+')
+            for index in zeroDepthSearch(old, '+'):
+                parts.append(old[last:index].strip('+'))
+                last = index

-            if indexes:
-                first, last = 0, 0
-                for i in xrange(1, len(indexes)):
-                    if ' ' in retVal[indexes[0]:indexes[i]]:
-                        break
-                    else:
-                        last = i
+            parts.append(old[last:].strip('+'))
+            replacement = parts[0]

-                start = retVal[:indexes[first]].rfind(' ') + 1
-                end = (retVal[indexes[last] + 1:].find(' ') + indexes[last] + 1) if ' ' in retVal[indexes[last] + 1:] else len(retVal) - 1
+            for i in xrange(1, len(parts)):
+                replacement = "{fn CONCAT(%s,%s)}" % (replacement, parts[i])

-                count = 0
-                chars = [char for char in retVal]
-                for index in indexes[first:last + 1]:
-                    if count == 0:
-                        chars[index] = ','
-                    else:
-                        chars[index] = '\x01'
-                    count += 1
-
-                retVal = "%s%s%s)}%s" % (retVal[:start], "{fn CONCAT(" * count, ''.join(chars)[start:end].replace('\x01', ")},"), retVal[end:])
-            else:
-                match = re.search(r"\((CHAR\(\d+.+\bCHAR\(\d+\))\)", retVal)
-                if match:
-                    part = match.group(0)
-                    indexes = set(zeroDepthSearch(match.group(1), '+'))
-                    if not indexes:
-                        break
-
-                    count = 0
-                    chars = [char for char in part]
-                    for i in xrange(1, len(chars)):
-                        if i - 1 in indexes:
-                            if count == 0:
-                                chars[i] = ','
-                            else:
-                                chars[i] = '\x01'
-                            count += 1
-
-                    replacement = "%s%s}" % (("{fn CONCAT(" * count)[:-1], "".join(chars).replace('\x01', ")},"))
-                    retVal = retVal.replace(part, replacement)
-                else:
-                    break
-
-        retVal = "%s%s%s" % (prefix, retVal, suffix)
+            retVal = retVal.replace(old, replacement)

    return retVal
@@ -17,7 +17,7 @@ def tamper(payload, **kwargs):
    Appends a HTTP header 'X-originating-IP' to bypass Varnish Firewall

    Reference:
-        * http://h30499.www3.hp.com/t5/Fortify-Application-Security/Bypassing-web-application-firewalls-using-HTTP-headers/ba-p/6418366
+        * https://web.archive.org/web/20160815052159/http://community.hpe.com/t5/Protect-Your-Assets/Bypassing-web-application-firewalls-using-HTTP-headers/ba-p/6418366

    Notes:
        Examples:
@@ -16,20 +16,29 @@ def dependencies():
    pass

 def randomIP():
-    numbers = []
+    octets = []

-    while not numbers or numbers[0] in (10, 172, 192):
-        numbers = random.sample(xrange(1, 255), 4)
+    while not octets or octets[0] in (10, 172, 192):
+        octets = random.sample(xrange(1, 255), 4)

-    return '.'.join(str(_) for _ in numbers)
+    return '.'.join(str(_) for _ in octets)

 def tamper(payload, **kwargs):
    """
-    Append a fake HTTP header 'X-Forwarded-For'
+    Append a fake HTTP header 'X-Forwarded-For' (and alike)
    """

    headers = kwargs.get("headers", {})
    headers["X-Forwarded-For"] = randomIP()
    headers["X-Client-Ip"] = randomIP()
    headers["X-Real-Ip"] = randomIP()
+    headers["CF-Connecting-IP"] = randomIP()
+    headers["True-Client-IP"] = randomIP()
+
+    # Reference: https://developer.chrome.com/multidevice/data-compression-for-isps#proxy-connection
+    headers["Via"] = "1.1 Chrome-Compression-Proxy"
+
+    # Reference: https://wordpress.org/support/topic/blocked-country-gaining-access-via-cloudflare/#post-9812007
+    headers["CF-IPCountry"] = random.sample(('GB', 'US', 'FR', 'AU', 'CA', 'NZ', 'BE', 'DK', 'FI', 'IE', 'AT', 'IT', 'LU', 'NL', 'NO', 'PT', 'SE', 'ES', 'CH'), 1)[0]
+
    return payload
@@ -894,7 +894,7 @@ class Tag(PageElement):

    def recursiveChildGenerator(self):
        if not len(self.contents):
-            raise StopIteration
+            return  # Note: https://stackoverflow.com/a/30217723 (PEP 479)
        stopNode = self._lastRecursiveChild().next
        current = self.contents[0]
        while current is not stopNode:
@@ -227,6 +227,9 @@ def unescape(data, entities, encoding=DEFAULT_ENCODING):
    if data is None or "&" not in data:
        return data

+    if isinstance(data, six.string_types):
+        encoding = None
+
    def replace_entities(match, entities=entities, encoding=encoding):
        ent = match.group()
        if ent[1] == "#":
@@ -279,7 +282,6 @@ def get_entitydefs():
            entitydefs["&%s;" % name] = _unichr(codepoint)
    return entitydefs

-
 def issequence(x):
    try:
        x[0]
@@ -184,6 +184,8 @@ class AnsiToWin32(object):
            if not (err.errno == 0 and retry > 0):
                raise
            self._write(text, retry-1)
+        except UnicodeError:
+            self.wrapped.write('?')

    def convert_ansi(self, paramstring, command):
        if self.convert:
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2019 Miroslav Stampar (@stamparm), MIT
+# See the file 'LICENSE' for copying permission
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+pass
@@ -55,7 +55,8 @@
            "name": "360",
            "regex": "<title>493</title>|/wzws-waf-cgi/",
            "signatures": [
-                "9778:RVZXum61OEhCWapBYKcPk4JzWOpohM4JiUcMr2RXg1uQJbX3uhdOnthtOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A5chW1XaTC"
+                "9778:RVZXum61OEhCWapBYKcPk4JzWOpohM4JiUcMr2RXg1uQJbX3uhdOnthtOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A5chW1XaTC",
+                "9ccc:RVZXum61OEhCWapBYKcPk4JzWOpohM4JiUcMr2RXg1uQJbX3uhdOnthtOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A4chW1XaTC"
            ]
        },
        "aesecure": {
@@ -677,6 +678,12 @@
            "signatures": [],
            "note": "Uses Incapsula (Reference: https://www.whitefirdesign.com/blog/2016/11/08/more-evidence-that-sitelocks-trueshield-web-application-firewall-is-really-incapsulas-waf/)"
        },
+        "sniper": {
+            "company": "Wins",
+            "name": "Sniper",
+            "regex": "document\\.title = [^;]+Sniper WAF",
+            "signatures": []
+        },
        "sonicwall": {
            "company": "Dell",
            "name": "SonicWALL",
@@ -777,6 +784,14 @@
                "c02b:RVZXu261OElCWapBYKcPk4JzWOpohM4JiUcMr2RWg1uQJbX3uhdOnthsOj+hXrAB16FcPxJOdLsXo2tKaK99n+i7c4RmkgI2FZjxtDtAeq+c36A4chW1XaTC"
            ]
        },
+        "wapples": {
+            "company": "Penta Security",
+            "name": "Wapples",
+            "regex": "",
+            "signatures": [
+                "60b7:RVZXum60OEhCWKpAYKYPkoJyWOpohM4IiUYMrmRWg1uQJLX2uhZOnthtOj+hXrAA16FcPhJOdLoXomtKaK59nui7c4RmkgI2FZjxtDtAeq+c3qA4chW1XKTC"
+            ]
+        },
        "watchguard": {
            "company": "WatchGuard Technologies",
            "name": "WatchGuard",
@@ -809,6 +824,14 @@
                "17a8:RVZXum60OEhCWKpAYKYPkoJyWOpohM4JiUcMrmRXg1qQJbX3uhdOnthtOj+hXrAB16FcPhJPdLoXomtKaK59nui6c4RmkgI2FZjxtDtAeq+c3qA4chS1XKTC"
            ]
        },
+        "webland": {
+            "company": "WebLand",
+            "name": "WebLand",
+            "regex": "Server: Apache Protected by Webland WAF",
+            "signatures": [
+                "4ba0:RVZXum60OEhCWKpAYKYPkoJzWOpohc4IiUYMr2RWg1uQJLX3uhZOnthsOj6hXrAA16BcPhJOdLoXomtKaK59nui6c4RmkgI2FZjxtDtAeq6c3qA4chS1XKTC"
+            ]
+        },
        "webseal": {
            "company": "IBM",
            "name": "WebSEAL",
@@ -824,7 +847,7 @@
            "signatures": []
        },
        "wordfence": {
-            "company": "Feedjit",
+            "company": "Defiant",
            "name": "Wordfence",
            "regex": "Generated by Wordfence|This response was generated by Wordfence|broke one of the Wordfence (advanced )?blocking rules|: wfWAF|/plugins/wordfence",
            "signatures": [
Author	SHA1	Message	Date
Miroslav Stampar	d72660ef04	Adding a support for option-less URL arg	2019-10-01 15:37:09 +02:00
Miroslav Stampar	4833fb3aa6	Adding autocompletion script	2019-09-30 17:50:25 +02:00
Miroslav Stampar	f437a54561	Update regarding #3944	2019-09-27 21:59:20 +02:00
Miroslav Stampar	f05f84b6e5	Minor bug fix	2019-09-27 21:35:21 +02:00
Miroslav Stampar	871ebfdb70	Fixes #3943	2019-09-27 21:03:21 +02:00
Miroslav Stampar	d34619232f	Implements #3940	2019-09-26 10:36:47 +02:00
Miroslav Stampar	5168daf6ce	Fixes #3939	2019-09-24 10:24:43 +02:00
Miroslav Stampar	95e476d5c9	Fixes #3937	2019-09-22 20:14:43 +02:00
Miroslav Stampar	cc8209d648	Patch related to the #3927	2019-09-16 19:29:38 +02:00
Miroslav Stampar	6ec6e86937	Update regarding #3928	2019-09-16 10:18:51 +02:00
Miroslav Stampar	018908c2b1	Fixes #3925	2019-09-16 10:11:28 +02:00
Miroslav Stampar	9e69d6076d	Fixes #3929	2019-09-16 10:08:10 +02:00
Miroslav Stampar	ad785ea0a2	Fixes #3926	2019-09-16 10:03:19 +02:00
Miroslav Stampar	b51f02c2ca	Patch related to #3918	2019-09-13 16:30:26 +02:00
Miroslav Stampar	ea3de16f24	Implementing PEP 479 (fixes #3924 )	2019-09-13 11:38:26 +02:00
Miroslav Stampar	f29c4e1e07	Minor generalization regarding last commit (#3917 )	2019-09-12 15:29:15 +02:00
Miroslav Stampar	e8871b8a99	Fixes #3917	2019-09-12 15:20:00 +02:00
Miroslav Stampar	8b88bb82d3	Minor refactoring	2019-09-11 14:05:25 +02:00
Miroslav Stampar	30fba849e2	Implements #3916	2019-09-09 13:56:37 +02:00
Miroslav Stampar	617c336813	Minor improvements	2019-09-09 11:45:23 +02:00
Miroslav Stampar	89a5892dd9	Dirty patch for #3915	2019-09-09 11:15:13 +02:00
Miroslav Stampar	af8a2afde1	Cosmetics (trivial)	2019-09-05 11:33:56 +02:00
Miroslav Stampar	ceaf387533	Fixes #3912	2019-09-05 11:22:10 +02:00
Miroslav Stampar	e5a1377c36	Minor update	2019-09-05 11:15:43 +02:00
Miroslav Stampar	bb61b08c83	Minor update of fingerprints	2019-09-03 14:01:37 +02:00
Miroslav Stampar	2d63441cc4	Minor update	2019-09-02 15:58:57 +02:00
Miroslav Stampar	80f6460f72	Minor update for Firebird	2019-09-02 12:22:32 +02:00
Miroslav Stampar	f2c2864ab4	Couple of trivial updates	2019-08-30 14:43:56 +02:00
Miroslav Stampar	9eda11d081	Trivial update	2019-08-29 17:07:16 +02:00
Miroslav Stampar	54e93e53df	Fixes #3898	2019-08-27 13:41:30 +02:00
Miroslav Stampar	f6e1f11711	Fixes #3897	2019-08-27 13:39:18 +02:00
Miroslav Stampar	0a21635e7f	Adding a minor thing (history reasons)	2019-08-26 17:33:03 +02:00
Miroslav Stampar	83aa1ac6a7	Implements #3895	2019-08-26 17:27:32 +02:00
Miroslav Stampar	0aa15a72b0	Minor refactoring	2019-08-26 16:51:17 +02:00
Miroslav Stampar	fecd830622	Minor update	2019-08-26 16:46:21 +02:00
Miroslav Stampar	32a4f6c32f	Initial patch for #3894 (not final)	2019-08-25 13:20:06 +02:00
Miroslav Stampar	9a47b4025b	Trivial update/patch	2019-08-23 09:49:16 +02:00
Miroslav Stampar	1979e7d75b	Fixes #3888	2019-08-23 09:40:49 +02:00
Miroslav Stampar	95e6b6c0af	Implements #3835	2019-08-22 11:41:06 +02:00
Miroslav Stampar	50b8de00bb	Patches #3887	2019-08-22 10:43:38 +02:00
Miroslav Stampar	81289fa7cd	Fixes #3886	2019-08-21 15:29:51 +02:00
Miroslav Stampar	576d81aa49	Minor log message update	2019-08-21 15:19:42 +02:00
Miroslav Stampar	009a3c8391	Minor wording update	2019-08-21 14:08:13 +02:00
Miroslav Stampar	65b0dbd4c4	Patches #3884	2019-08-20 00:35:13 +02:00
Miroslav Stampar	5f411f73e7	Fixes #3883	2019-08-20 00:22:55 +02:00
TaeGeun Moon	dadb33bfdb	Add Korean translation for README (#3881 ) * Add Korean translation for README * Only on main page * Only on main page	2019-08-16 10:28:23 +02:00
Miroslav Stampar	8584c0b021	Fixes #3880	2019-08-16 09:56:03 +02:00
Miroslav Stampar	3f1a8e81b4	Adding support for #3870	2019-08-13 15:22:02 +02:00
Miroslav Stampar	0e14647573	Minor refactoring	2019-08-13 14:59:10 +02:00
Miroslav Stampar	bfe8785ed5	Minor regrouping	2019-08-13 14:55:26 +02:00
Miroslav Stampar	412301bb18	Another trivial patch related to the last commit	2019-08-13 12:29:18 +02:00
Miroslav Stampar	fd4becf389	Minor deprecation patch (drei)	2019-08-13 12:27:28 +02:00
Miroslav Stampar	aa2682ec16	Fixes #3873	2019-08-13 12:23:45 +02:00
Miroslav Stampar	94ef433a37	Minor update for #3874	2019-08-13 11:58:33 +02:00
Miroslav Stampar	15f6796b04	Fixes #3871	2019-08-11 01:32:20 +02:00
Miroslav Stampar	de63238897	Minor patch for --threads and multi Ctrl-C	2019-08-06 02:54:18 +02:00
Miroslav Stampar	1824e5b094	Trivial removal of leftover	2019-08-04 01:05:28 +02:00
Miroslav Stampar	a42a7c88bd	Fixes #3841	2019-08-04 01:05:13 +02:00
Miroslav Stampar	b5063fc25a	Implementation for #3859	2019-08-02 20:29:52 +02:00
Miroslav Stampar	093b36f12d	Minor patch	2019-08-02 19:33:16 +02:00
Miroslav Stampar	a9de51380b	Implementation for #3845	2019-07-31 20:00:51 +02:00
Miroslav Stampar	ec0be6519f	Minor update	2019-07-31 19:53:20 +02:00
Miroslav Stampar	9e8aec37c8	Minor patch	2019-07-30 20:31:22 +02:00
Miroslav Stampar	1f644bd3ae	Fixes #3854	2019-07-30 20:28:56 +02:00
Miroslav Stampar	86abf179f2	Fixes #3850	2019-07-30 20:12:45 +02:00
Miroslav Stampar	4a6ff82273	Patch for #3851	2019-07-24 23:43:08 +02:00
Miroslav Stampar	198ceb8ba1	Minor update regarding the #3834	2019-07-19 12:24:34 +02:00
Miroslav Stampar	0bc5069042	Implements #3834	2019-07-19 12:17:07 +02:00
Miroslav Stampar	bd1ea4fd73	Fixes #3837	2019-07-18 20:32:02 +02:00
Miroslav Stampar	db90ff9c3f	Fixing mess with --common-files --threads>1 (threads in threads - '.shared.' hell)	2019-07-18 14:59:42 +02:00
Miroslav Stampar	b62680b4bc	Minor update (--technique=E --common-files)	2019-07-18 13:26:58 +02:00
Miroslav Stampar	819bf47a11	Minor update (storing force-partial)	2019-07-18 12:16:03 +02:00
Miroslav Stampar	f06e9ecb58	Minor update for #3833	2019-07-18 11:58:40 +02:00
Miroslav Stampar	8fda828bc9	Minor refactoring	2019-07-18 11:27:00 +02:00
Miroslav Stampar	453a6fbc6f	Update for #3831	2019-07-17 13:20:24 +02:00
Miroslav Stampar	9d0f446c24	Dummy patch for #3830	2019-07-17 13:03:48 +02:00
Miroslav Stampar	e6173d7eaa	Minor update	2019-07-17 10:30:14 +02:00
Miroslav Stampar	1b0488160e	Minor update	2019-07-16 15:07:20 +02:00
Miroslav Stampar	86392179c3	Minor patch related to the #3822	2019-07-16 14:02:16 +02:00
Miroslav Stampar	823119a247	Update regarding #3826	2019-07-16 13:37:41 +02:00
Miroslav Stampar	6b88fa3a30	Minor patch	2019-07-16 09:40:59 +02:00
Miroslav Stampar	3c4aadf995	Fixes #3823	2019-07-15 13:08:22 +02:00
Miroslav Stampar	34ed2c51ac	Minor style update	2019-07-12 14:19:25 +02:00
Miroslav Stampar	eb14f296f6	Fixes #3820	2019-07-12 13:41:07 +02:00
Miroslav Stampar	7d43429379	Fixes #3819	2019-07-12 12:18:56 +02:00
Miroslav Stampar	c3a95e81f5	Fixes #3797	2019-07-11 12:40:56 +02:00
Miroslav Stampar	c1ae1b432e	Update regarding #3813	2019-07-11 11:30:21 +02:00
Miroslav Stampar	84d0b346e7	Patch for #3816	2019-07-11 11:13:49 +02:00
Miroslav Stampar	4c8d348e2f	Fixes #3812	2019-07-11 10:56:38 +02:00
Miroslav Stampar	752aed2986	Patch for #3815	2019-07-10 13:49:41 +02:00
Miroslav Stampar	e10a96610d	Minor update	2019-07-10 10:49:05 +02:00
Miroslav Stampar	090cbf75f1	Minor update for #3808	2019-07-08 13:48:13 +02:00
Miroslav Stampar	71b63c9262	Update regarding 3804	2019-07-08 12:25:16 +02:00
Miroslav Stampar	f85abafd79	Patch for #3805	2019-07-08 11:55:04 +02:00
Miroslav Stampar	40bc53537c	Fixes #3806	2019-07-08 11:53:19 +02:00
Miroslav Stampar	b4c00f4172	Update regarding #3802	2019-07-07 16:17:24 +02:00
Miroslav Stampar	d0c48df62c	Fixes #3801	2019-07-07 15:56:54 +02:00
Miroslav Stampar	3f99ec638f	Minor update	2019-07-04 12:03:26 +02:00
Miroslav Stampar	83e1daab96	Fixes #3796	2019-07-04 11:18:55 +02:00
Miroslav Stampar	f9489c3352	Minor patch (fixes #3795 )	2019-07-04 11:07:25 +02:00
Miroslav Stampar	3676cef79b	Preparing for something something	2019-07-03 16:30:18 +02:00
Miroslav Stampar	25f29ca6b0	Minor updates	2019-07-03 10:56:05 +02:00
Miroslav Stampar	a7695dd06f	Fixes #3792	2019-07-03 09:17:25 +02:00
Miroslav Stampar	7b668127fc	Minor update	2019-07-01 10:54:11 +02:00