Minor update for Firebird

* Add Korean translation for README

* Only on main page

* Only on main page

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+dist: trusty
 sudo: false
 git:
     depth: 1
@@ -10,4 +11,4 @@ python:
 script:
     - python -c "import sqlmap; import sqlmapapi"
     - python sqlmap.py --smoke
-    - python sqlmap.py --vuln
+    - python sqlmap.py --vuln

README.md

@@ -63,6 +63,7 @@ Translations
 * [Indonesian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-id-ID.md)
 * [Italian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-it-IT.md)
 * [Japanese](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ja-JP.md)
+* [Korean](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ko-KR.md)
 * [Polish](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-pl-PL.md)
 * [Portuguese](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-pt-BR.md)
 * [Russian](https://github.com/sqlmapproject/sqlmap/blob/master/doc/translations/README-ru-RUS.md)

data/html/index.html

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+
+<!-- http://angrytools.com/bootstrap/editor/ -->
+
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap.min.css" rel="stylesheet">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap-theme.min.css" rel="stylesheet">
+    
+    <!--[if lt IE 9]><script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script><script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script><![endif]-->
+</head>
+<body>
+    <style>
+  #wrapper { width: 100%; }
+
+  #page-wrapper {
+    padding: 0 15px;
+    min-height: 568px;
+    background-color: #fff;
+  }
+
+  @media(min-width:768px) {
+    #page-wrapper {
+      position: inherit;
+      margin: 0 0 0 250px;
+      padding: 0 30px;
+      border-left: 1px solid #e7e7e7;
+    }
+  } 
+
+  .sidebar .sidebar-nav.navbar-collapse { padding-right: 0; padding-left: 0; }
+  .sidebar .sidebar-search { padding: 15px; }
+  .sidebar ul li { border-bottom: 1px solid #e7e7e7; }
+
+  .sidebar ul li a.active { background-color: #eee; }
+
+  .sidebar .arrow { float: right;}
+  .sidebar .fa.arrow:before { content: "f104";}
+  .sidebar .active>a>.fa.arrow:before { content: "f107"; }
+  .sidebar .nav-second-level li,
+  .sidebar .nav-third-level li {
+    border-bottom: 0!important;
+  }
+
+  .sidebar .nav-second-level li a { padding-left: 37px; }
+  .sidebar .nav-third-level li a { padding-left: 52px; }
+
+  @media(min-width:768px) {
+    .sidebar {
+      z-index: 1;
+      position: absolute;
+      width: 250px;
+      margin-top: 51px;
+    }   
+  } 
+</style>
+<div id="wrapper">
+
+  <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <a class="navbar-brand" href="index.html">sqlmap</a>
+    </div>
+
+    <div class="navbar-default sidebar" role="navigation">
+      <div class="sidebar-nav navbar-collapse">
+        <ul class="nav" id="side-menu">                        
+          <li>
+            <a href="#"><i class="glyphicon glyphicon-home"></i> Options<span class="arrow"></span></a>
+            <ul class="nav nav-second-level">
+              <li><a>Target</a></li>
+              <li><a>Request</a></li>
+              <li><a>Optimization</a></li>
+              <li><a>Injection</a></li>
+              <li><a>Detection</a></li>
+              <li><a>Techniques</a></li>
+              <li><a>Fingerprint</a></li>
+              <li><a>Enumeration</a></li>
+              <li><a>Brute force</a></li>
+              <li><a>User-defined function injection</a></li>
+              <li><a>File system access</a></li>
+              <li><a>Operating system access</a></li>
+              <li><a>Windows registry access</a></li>
+              <li><a>General</a></li>
+              <li><a>Miscellaneous</a></li>
+            </ul> 
+          </li> 
+        </ul>
+      </div>
+    </div>             
+  </nav>
+
+  <div id="page-wrapper"> 
+    <div class="row">
+      <h4>DEMO</h4>
+    </div> 
+  </div>
+</div>
+<script>
+  /*
+ * metismenu - v1.0.3
+ * Easy menu jQuery plugin for Twitter Bootstrap 3
+ * https://github.com/onokumus/metisMenu
+ *
+ * Made by Osman Nuri Okumuş
+ * Under MIT License
+*/
+  !function(a,b,c){function d(b,c){this.element=b,this.settings=a.extend({},f,c),this._defaults=f,this._name=e,this.init()}var e="metisMenu",f={toggle:!0};d.prototype={init:function(){var b=a(this.element),c=this.settings.toggle;this.isIE()<=9?(b.find("li.active").has("ul").children("ul").collapse("show"),b.find("li").not(".active").has("ul").children("ul").collapse("hide")):(b.find("li.active").has("ul").children("ul").addClass("collapse in"),b.find("li").not(".active").has("ul").children("ul").addClass("collapse")),b.find("li").has("ul").children("a").on("click",function(b){b.preventDefault(),a(this).parent("li").toggleClass("active").children("ul").collapse("toggle"),c&&a(this).parent("li").siblings().removeClass("active").children("ul.in").collapse("hide")})},isIE:function(){for(var a,b=3,d=c.createElement("div"),e=d.getElementsByTagName("i");d.innerHTML="<!--[if gt IE "+ ++b+"]><i></i><![endif]-->",e[0];)return b>4?b:a}},a.fn[e]=function(b){return this.each(function(){a.data(this,"plugin_"+e)||a.data(this,"plugin_"+e,new d(this,b))})}}(jQuery,window,document);
+
+  $(function() {
+
+    $('#side-menu').metisMenu();
+
+  });
+
+  //Loads the correct sidebar on window load,
+  //collapses the sidebar on window resize.
+  // Sets the min-height of #page-wrapper to window size
+  $(function() {
+    $(window).bind("load resize", function() {
+      topOffset = 50;
+      width = (this.window.innerWidth > 0) ? this.window.innerWidth : this.screen.width;
+      if (width < 768) {
+        $('div.navbar-collapse').addClass('collapse')
+        topOffset = 100; // 2-row-menu
+      } else {
+        $('div.navbar-collapse').removeClass('collapse')
+      }
+
+      height = (this.window.innerHeight > 0) ? this.window.innerHeight : this.screen.height;
+      height = height - topOffset;
+      if (height < 1) height = 1;
+      if (height > topOffset) {
+        $("#page-wrapper").css("min-height", (height) + "px");
+      }
+    })
+  });
+</script>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/js/bootstrap.min.js"></script>
+</body>
+</html>

doc/translations/README-ko-KR.md

@@ -0,0 +1,50 @@
+# sqlmap
+
+[![Build Status](https://api.travis-ci.org/sqlmapproject/sqlmap.svg?branch=master)](https://travis-ci.org/sqlmapproject/sqlmap) [![Python 2.6|2.7|3.x](https://img.shields.io/badge/python-2.6|2.7|3.x-yellow.svg)](https://www.python.org/) [![License](https://img.shields.io/badge/license-GPLv2-red.svg)](https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/LICENSE) [![PyPI version](https://badge.fury.io/py/sqlmap.svg)](https://badge.fury.io/py/sqlmap) [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sqlmapproject/sqlmap.svg?colorB=ff69b4)](https://github.com/sqlmapproject/sqlmap/issues?q=is%3Aissue+is%3Aclosed) [![Twitter](https://img.shields.io/badge/[email protected])](https://twitter.com/sqlmap)
+
+sqlmap은 SQL 인젝션 결함 탐지 및 활용, 데이터베이스 서버 장악 프로세스를 자동화 하는 오픈소스 침투 테스팅 도구입니다. 최고의 침투 테스터, 데이터베이스 핑거프린팅 부터 데이터베이스 데이터 읽기, 대역 외 연결을 통한 기반 파일 시스템 접근 및 명령어 실행에 걸치는 광범위한 스위치들을 위한 강력한 탐지 엔진과 다수의 편리한 기능이 탑재되어 있습니다.
+
+스크린샷
+----
+
+![Screenshot](https://raw.github.com/wiki/sqlmapproject/sqlmap/images/sqlmap_screenshot.png)
+
+또는, wiki에 나와있는 몇몇 기능을 보여주는 [스크린샷 모음](https://github.com/sqlmapproject/sqlmap/wiki/Screenshots) 을 방문하실 수 있습니다.
+
+설치
+----
+
+[여기](https://github.com/sqlmapproject/sqlmap/tarball/master)를 클릭하여 최신 버전의 tarball 파일, 또는 [여기](https://github.com/sqlmapproject/sqlmap/zipball/master)를 클릭하여 최신 zipball 파일을 다운받으실 수 있습니다.
+
+가장 선호되는 방법으로, [Git](https://github.com/sqlmapproject/sqlmap) 저장소를 복제하여 sqlmap을 다운로드 할 수 있습니다:
+
+    git clone --depth 1 https://github.com/sqlmapproject/sqlmap.git sqlmap-dev
+
+sqlmap은 [Python](http://www.python.org/download/) 버전 **2.6**, **2.7** 그리고 **3.x** 을 통해 모든 플랫폼 위에서 사용 가능합니다.
+
+사용법
+----
+
+기본 옵션과 스위치 목록을 보려면 다음 명령어를 사용하세요:
+
+    python sqlmap.py -h
+
+전체 옵션과 스위치 목록을 보려면 다음 명령어를 사용하세요:
+
+    python sqlmap.py -hh
+
+[여기](https://asciinema.org/a/46601)를 통해 사용 샘플들을 확인할 수 있습니다.
+sqlmap의 능력, 지원되는 기능과 모든 옵션과 스위치들의 목록을 예제와 함께 보려면, [사용자 매뉴얼](https://github.com/sqlmapproject/sqlmap/wiki/Usage)을 참고하시길 권장드립니다.
+
+링크
+----
+
+* 홈페이지: http://sqlmap.org
+* 다운로드: [.tar.gz](https://github.com/sqlmapproject/sqlmap/tarball/master) or [.zip](https://github.com/sqlmapproject/sqlmap/zipball/master)
+* RSS 피드 커밋: https://github.com/sqlmapproject/sqlmap/commits/master.atom
+* Issue tracker: https://github.com/sqlmapproject/sqlmap/issues
+* 사용자 매뉴얼: https://github.com/sqlmapproject/sqlmap/wiki
+* 자주 묻는 질문 (FAQ): https://github.com/sqlmapproject/sqlmap/wiki/FAQ
+* 트위터: [@sqlmap](https://twitter.com/sqlmap)
+* 시연 영상: [http://www.youtube.com/user/inquisb/videos](http://www.youtube.com/user/inquisb/videos)
+* 스크린샷: https://github.com/sqlmapproject/sqlmap/wiki/Screenshots

lib/controller/checks.py

@@ -1518,7 +1518,7 @@ def checkConnection(suppressOutput=False):
             warnMsg += "which could interfere with the results of the tests"
             logger.warn(warnMsg)
         elif wasLastResponseHTTPError():
-            if getLastRequestHTTPError() != conf.ignoreCode:
+            if getLastRequestHTTPError() not in (conf.ignoreCode or []):
                 warnMsg = "the web server responded with an HTTP error code (%d) " % getLastRequestHTTPError()
                 warnMsg += "which could interfere with the results of the tests"
                 logger.warn(warnMsg)

lib/controller/controller.py

@@ -422,6 +422,15 @@ def start():
             if not checkConnection(suppressOutput=conf.forms) or not checkString() or not checkRegexp():
                 continue
 
+            if conf.rParam and kb.originalPage:
+                kb.randomPool = dict([_ for _ in kb.randomPool.items() if isinstance(_[1], list)])
+
+                for match in re.finditer(r"(?si)<select[^>]+\bname\s*=\s*[\"']([^\"']+)(.+?)</select>", kb.originalPage):
+                    name, _ = match.groups()
+                    options = tuple(re.findall(r"<option[^>]+\bvalue\s*=\s*[\"']([^\"']+)", _))
+                    if options:
+                        kb.randomPool[name] = options
+
             checkWaf()
 
             if conf.nullConnection:

lib/core/agent.py

@@ -13,6 +13,7 @@ from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
 from lib.core.common import getSQLSnippet
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import isDBMSVersionAtLeast
 from lib.core.common import isNumber
 from lib.core.common import isTechniqueAvailable
@@ -50,6 +51,7 @@ from lib.core.settings import REPLACEMENT_MARKER
 from lib.core.settings import SINGLE_QUOTE_MARKER
 from lib.core.settings import SLEEP_TIME_MARKER
 from lib.core.unescaper import unescaper
+from thirdparty import six
 
 class Agent(object):
     """
@@ -91,7 +93,7 @@ class Agent(object):
         if kb.forceWhere:
             where = kb.forceWhere
         elif where is None and isTechniqueAvailable(getTechnique()):
-            where = kb.injection.data[getTechnique()].where
+            where = getTechniqueData().where
 
         if kb.injection.place is not None:
             place = kb.injection.place
@@ -236,7 +238,7 @@ class Agent(object):
         query = None
 
         if where is None and getTechnique() is not None and getTechnique() in kb.injection.data:
-            where = kb.injection.data[getTechnique()].where
+            where = getTechniqueData().where
 
         # If we are replacing (<where>) the parameter original value with
         # our payload do not prepend with the prefix
@@ -284,8 +286,8 @@ class Agent(object):
         suffix = kb.injection.suffix if kb.injection and suffix is None else suffix
 
         if getTechnique() is not None and getTechnique() in kb.injection.data:
-            where = kb.injection.data[getTechnique()].where if where is None else where
-            comment = kb.injection.data[getTechnique()].comment if comment is None else comment
+            where = getTechniqueData().where if where is None else where
+            comment = getTechniqueData().comment if comment is None else comment
 
         if Backend.getIdentifiedDbms() == DBMS.ACCESS and any((comment or "").startswith(_) for _ in ("--", "[GENERIC_SQL_COMMENT]")):
             comment = queries[DBMS.ACCESS].comment.query
@@ -307,7 +309,7 @@ class Agent(object):
         return re.sub(r";\W*;", ";", expression) if trimEmpty else expression
 
     def cleanupPayload(self, payload, origValue=None):
-        if payload is None:
+        if not isinstance(payload, six.string_types):
             return
 
         replacements = {
@@ -332,6 +334,7 @@ class Agent(object):
 
         if origValue is not None:
             origValue = getUnicode(origValue)
+
             if "[ORIGVALUE]" in payload:
                 payload = getUnicode(payload).replace("[ORIGVALUE]", origValue if origValue.isdigit() else unescaper.escape("'%s'" % origValue))
             if "[ORIGINAL]" in payload:
@@ -350,6 +353,7 @@ class Agent(object):
                     inferenceQuery = inference.query
 
                 payload = payload.replace(INFERENCE_MARKER, inferenceQuery)
+
             elif not kb.testMode:
                 errMsg = "invalid usage of inference payload without "
                 errMsg += "knowledge of underlying DBMS"
@@ -392,7 +396,7 @@ class Agent(object):
         if "hex" in rootQuery:
             hexField = rootQuery.hex.query % field
         else:
-            warnMsg = "switch '--hex' is currently not supported on DBMS %s" % Backend.getIdentifiedDbms()
+            warnMsg = "switch '--hex' is currently not supported on DBMS '%s'" % Backend.getIdentifiedDbms()
             singleTimeWarnMessage(warnMsg)
 
         return hexField
@@ -1006,7 +1010,7 @@ class Agent(object):
                         limitedQuery = "%s WHERE %s " % (limitedQuery, self.nullAndCastField(uniqueField or field))
 
                     limitedQuery += "NOT IN (%s" % (limitStr % num)
-                    limitedQuery += "%s %s ORDER BY %s) ORDER BY %s" % (self.nullAndCastField(uniqueField or field), fromFrom, uniqueField or "1", uniqueField or "1")
+                    limitedQuery += "%s %s ORDER BY %s) ORDER BY %s" % (self.nullAndCastField(uniqueField or field), fromFrom, uniqueField or '1', uniqueField or '1')
                 else:
                     match = re.search(r" ORDER BY (\w+)\Z", query)
                     field = match.group(1) if match else field
@@ -1080,7 +1084,7 @@ class Agent(object):
         Removes payload delimiters from inside the input string
         """
 
-        return value.replace(PAYLOAD_DELIMITER, '') if value else value
+        return value.replace(PAYLOAD_DELIMITER, "") if value else value
 
     def extractPayload(self, value):
         """

lib/core/bigarray.py

@@ -51,6 +51,11 @@ class Cache(object):
 class BigArray(list):
     """
     List-like class used for storing large amounts of data (disk cached)
+
+    >>> _ = BigArray(xrange(100000))
+    >>> _[20] = 0
+    >>> _[100]
+    100
     """
 
     def __init__(self, items=None):

lib/core/common.py

@@ -624,7 +624,7 @@ def paramToDict(place, parameters=None):
                     try:
                         oldValue = value
                         value = decodeBase64(value, binary=False)
-                        parameters = re.sub(r"\b%s\b" % re.escape(oldValue), value, parameters)
+                        parameters = re.sub(r"\b%s(\b|\Z)" % re.escape(oldValue), value, parameters)
                     except:
                         errMsg = "parameter '%s' does not contain " % parameter
                         errMsg += "valid Base64 encoded value ('%s')" % value
@@ -701,7 +701,7 @@ def paramToDict(place, parameters=None):
                             message += "has boundaries. Do you want to inject inside? ('%s') [y/N] " % getUnicode(_)
 
                             if readInput(message, default='N', boolean=True):
-                                testableParameters[parameter] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), (r"\g<1>%s" % re.sub(regex, r"\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter])).replace("\\", r"\\"), parameters)
+                                testableParameters[parameter] = re.sub(r"\b(%s\s*=\s*)%s" % (re.escape(parameter), re.escape(testableParameters[parameter])), (r"\g<1>%s" % re.sub(regex, r"\g<1>%s\g<2>" % BOUNDED_INJECTION_MARKER, testableParameters[parameter].replace("\\", r"\\"))), parameters)
                             break
 
     if conf.testParameter:
@@ -1137,7 +1137,7 @@ def getTechnique():
     Thread-safe getting of currently used technique
     """
 
-    return getCurrentThreadData().technique or kb.technique
+    return getCurrentThreadData().technique or kb.get("technique")
 
 def randomRange(start=0, stop=1000, seed=None):
     """
@@ -1285,36 +1285,43 @@ def banner():
     """
 
     if not any(_ in sys.argv for _ in ("--version", "--api")) and not conf.get("disableBanner"):
-        _ = BANNER
+        result = BANNER
 
         if not IS_TTY or "--disable-coloring" in sys.argv:
-            _ = clearColors(_)
+            result = clearColors(result)
         elif IS_WIN:
             coloramainit()
 
-        dataToStdout(_, forceOutput=True)
+        dataToStdout(result, forceOutput=True)
 
 def parsePasswordHash(password):
     """
     In case of Microsoft SQL Server password hash value is expanded to its components
+
+    >>> pushValue(kb.forcedDbms)
+    >>> kb.forcedDbms = DBMS.MSSQL
+    >>> "salt: 4086ceb6" in parsePasswordHash("0x01004086ceb60c90646a8ab9889fe3ed8e5c150b5460ece8425a")
+    True
+    >>> kb.forcedDbms = popValue()
     """
 
     blank = " " * 8
 
-    if not password or password == " ":
-        password = NULL
+    if isNoneValue(password) or password == " ":
+        retVal = NULL
+    else:
+        retVal = password
 
-    if Backend.isDbms(DBMS.MSSQL) and password != NULL and isHexEncodedString(password):
-        hexPassword = password
-        password = "%s\n" % hexPassword
-        password += "%sheader: %s\n" % (blank, hexPassword[:6])
-        password += "%ssalt: %s\n" % (blank, hexPassword[6:14])
-        password += "%smixedcase: %s\n" % (blank, hexPassword[14:54])
+    if Backend.isDbms(DBMS.MSSQL) and retVal != NULL and isHexEncodedString(password):
+        retVal = "%s\n" % password
+        retVal += "%sheader: %s\n" % (blank, password[:6])
+        retVal += "%ssalt: %s\n" % (blank, password[6:14])
+        retVal += "%smixedcase: %s\n" % (blank, password[14:54])
 
-        if not Backend.isVersionWithin(("2005", "2008")):
-            password += "%suppercase: %s" % (blank, hexPassword[54:])
+        if password[54:]:
+            retVal += "%suppercase: %s" % (blank, password[54:])
 
-    return password
+    return retVal
 
 def cleanQuery(query):
     """
@@ -3222,7 +3229,7 @@ def getTechniqueData(technique=None):
     Returns injection data for technique specified
     """
 
-    return kb.injection.data.get(technique)
+    return kb.injection.data.get(technique if technique is not None else getTechnique())
 
 def isTechniqueAvailable(technique):
     """
@@ -3549,7 +3556,7 @@ def openFile(filename, mode='r', encoding=UNICODE_ENCODING, errors="reversible",
         except IOError:
             errMsg = "there has been a file opening error for filename '%s'. " % filename
             errMsg += "Please check %s permissions on a file " % ("write" if mode and ('w' in mode or 'a' in mode or '+' in mode) else "read")
-            errMsg += "and that it's not locked by another process."
+            errMsg += "and that it's not locked by another process"
             raise SqlmapSystemException(errMsg)
 
 def decodeIntToUnicode(value):
@@ -3575,7 +3582,7 @@ def decodeIntToUnicode(value):
                     # Note: https://github.com/sqlmapproject/sqlmap/issues/1531
                     retVal = getUnicode(raw, conf.encoding or UNICODE_ENCODING)
                 elif Backend.isDbms(DBMS.MSSQL):
-                    retVal = getUnicode(raw, "UTF-16-BE")
+                    retVal = getUnicode(raw, "UTF-16-BE")   # References: https://docs.microsoft.com/en-us/sql/relational-databases/collations/collation-and-unicode-support?view=sql-server-2017 and https://stackoverflow.com/a/14488478
                 elif Backend.getIdentifiedDbms() in (DBMS.PGSQL, DBMS.ORACLE):
                     retVal = _unichr(value)
                 else:
@@ -3662,7 +3669,7 @@ def getLatestRevision():
     """
 
     retVal = None
-    req = _urllib.request.Request(url="https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/lib/core/settings.py")
+    req = _urllib.request.Request(url="https://raw.githubusercontent.com/sqlmapproject/sqlmap/master/lib/core/settings.py", headers={HTTP_HEADER.USER_AGENT: fetchRandomAgent()})
 
     try:
         content = getUnicode(_urllib.request.urlopen(req).read())
@@ -3762,7 +3769,7 @@ def createGithubIssue(errMsg, excMsg):
             logger.info(infoMsg)
 
             try:
-                with open(paths.GITHUB_HISTORY, "a+b") as f:
+                with openFile(paths.GITHUB_HISTORY, "a+b") as f:
                     f.write("%s\n" % key)
             except:
                 pass
@@ -3794,7 +3801,7 @@ def maskSensitiveData(msg):
             retVal = retVal.replace(value, '*' * len(value))
 
     # Just in case (for problematic parameters regarding user encoding)
-    for match in re.finditer(r"(?i)[ -]-(u|url|data|cookie|auth-\w+|proxy)( |=)(.*?)(?= -?-[a-z]|\Z)", retVal):
+    for match in re.finditer(r"(?i)[ -]-(u|url|data|cookie|auth-\w+|proxy|host|referer|headers?|H)( |=)(.*?)(?= -?-[a-z]|\Z)", retVal):
         retVal = retVal.replace(match.group(3), '*' * len(match.group(3)))
 
     # Fail-safe substitutions
@@ -4868,6 +4875,8 @@ def zeroDepthSearch(expression, value):
 
     >>> _ = "SELECT (SELECT id FROM users WHERE 2>1) AS result FROM DUAL"; _[zeroDepthSearch(_, "FROM")[0]:]
     'FROM DUAL'
+    >>> _ = "a(b; c),d;e"; _[zeroDepthSearch(_, "[;, ]")[0]:]
+    ',d;e'
     """
 
     retVal = []
@@ -4878,8 +4887,12 @@ def zeroDepthSearch(expression, value):
             depth += 1
         elif expression[index] == ')':
             depth -= 1
-        elif depth == 0 and expression[index:index + len(value)] == value:
-            retVal.append(index)
+        elif depth == 0:
+            if value.startswith('[') and value.endswith(']'):
+                if re.search(value, expression[index:index + 1]):
+                    retVal.append(index)
+            elif expression[index:index + len(value)] == value:
+                retVal.append(index)
 
     return retVal
 
@@ -4988,7 +5001,7 @@ def parseRequestFile(reqFile, checkParams=True):
             else:
                 scheme, port = None, None
 
-            if not re.search(r"^[\n]*(%s).*?\sHTTP\/" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), request, re.I | re.M):
+            if "HTTP/" not in request:
                 continue
 
             if re.search(r"^[\n]*%s.*?\.(%s)\sHTTP\/" % (HTTPMETHOD.GET, "|".join(CRAWL_EXCLUDE_EXTENSIONS)), request, re.I | re.M):
@@ -5013,7 +5026,7 @@ def parseRequestFile(reqFile, checkParams=True):
 
                 newline = "\r\n" if line.endswith('\r') else '\n'
                 line = line.strip('\r')
-                match = re.search(r"\A(%s) (.+) HTTP/[\d.]+\Z" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), line) if not method else None
+                match = re.search(r"\A([A-Z]+) (.+) HTTP/[\d.]+\Z", line) if not method else None
 
                 if len(line.strip()) == 0 and method and method != HTTPMETHOD.GET and data is None:
                     data = ""

lib/core/compat.py

@@ -13,6 +13,7 @@ import math
 import os
 import random
 import sys
+import time
 import uuid
 
 class WichmannHill(random.Random):
@@ -40,7 +41,6 @@ class WichmannHill(random.Random):
             try:
                 a = int(binascii.hexlify(os.urandom(16)), 16)
             except NotImplementedError:
-                import time
                 a = int(time.time() * 256)  # use fractional seconds
 
         if not isinstance(a, int):
@@ -132,7 +132,6 @@ class WichmannHill(random.Random):
             raise ValueError('seeds must be in range(0, 256)')
         if 0 == x == y == z:
             # Initialize from current time
-            import time
             t = int(time.time() * 256)
             t = int((t & 0xffffff) ^ (t >> 24))
             t, x = divmod(t, 256)
@@ -204,6 +203,7 @@ def round(x, d=0):
     else:
         return float(math.ceil((x * p) - 0.5)) / p
 
+# Reference: https://code.activestate.com/recipes/576653-convert-a-cmp-function-to-a-key-function/
 def cmp_to_key(mycmp):
     """Convert a cmp= function into a key= function"""
     class K(object):

lib/core/convert.py

@@ -164,8 +164,13 @@ def encodeHex(value, binary=True):
     True
     >>> encodeHex("123", binary=False)
     '313233'
+    >>> encodeHex(b"123"[0]) == b"31"
+    True
     """
 
+    if isinstance(value, int):
+        value = six.unichr(value)
+
     if isinstance(value, six.text_type):
         value = value.encode(UNICODE_ENCODING)
 

lib/core/decorators.py

@@ -14,9 +14,11 @@ from lib.core.settings import MAX_CACHE_ITEMS
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
 
-_lock = threading.Lock()
+_cache = {}
+_cache_lock = threading.Lock()
+_method_locks = {}
 
-def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
+def cachedmethod(f):
     """
     Method with a cached content
 
@@ -33,22 +35,24 @@ def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
     Reference: http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/
     """
 
+    _cache[f] = LRUDict(capacity=MAX_CACHE_ITEMS)
+
     @functools.wraps(f)
-    def _(*args, **kwargs):
+    def _f(*args, **kwargs):
         key = int(hashlib.md5("|".join(str(_) for _ in (f, args, kwargs)).encode(UNICODE_ENCODING)).hexdigest(), 16) & 0x7fffffffffffffff
 
         try:
-            with _lock:
-                result = cache[key]
+            with _cache_lock:
+                result = _cache[f][key]
         except KeyError:
             result = f(*args, **kwargs)
 
-            with _lock:
-                cache[key] = result
+            with _cache_lock:
+                _cache[f][key] = result
 
         return result
 
-    return _
+    return _f
 
 def stackedmethod(f):
     """
@@ -76,3 +80,16 @@ def stackedmethod(f):
         return result
 
     return _
+
+def lockedmethod(f):
+    @functools.wraps(f)
+    def _(*args, **kwargs):
+        if f not in _method_locks:
+            _method_locks[f] = threading.RLock()
+
+        with _method_locks[f]:
+            result = f(*args, **kwargs)
+
+        return result
+
+    return _

lib/core/defaults.py

@@ -20,6 +20,7 @@ _defaults = {
     "level": 1,
     "risk": 1,
     "dumpFormat": "CSV",
+    "tablePrefix": "sqlmap",
     "technique": "BEUSTQ",
     "torType": "SOCKS5",
 }

lib/core/dicts.py

@@ -304,7 +304,7 @@ DUMP_DATA_PREPROCESS = {
 
 DEFAULT_DOC_ROOTS = {
     OS.WINDOWS: ("C:/xampp/htdocs/", "C:/wamp/www/", "C:/Inetpub/wwwroot/"),
-    OS.LINUX: ("/var/www/", "/var/www/html", "/usr/local/apache2/htdocs", "/var/www/nginx-default", "/srv/www")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
+    OS.LINUX: ("/var/www/", "/var/www/html", "/var/www/htdocs", "/usr/local/apache2/htdocs", "/usr/local/www/data", "/var/apache2/htdocs", "/var/www/nginx-default", "/srv/www/htdocs")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
 }
 
 PART_RUN_CONTENT_TYPES = {
@@ -334,3 +334,260 @@ PART_RUN_CONTENT_TYPES = {
     "osCmd": CONTENT_TYPE.OS_CMD,
     "regRead": CONTENT_TYPE.REG_READ
 }
+
+# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html
+
+HTML_ENTITIES = {
+    "quot": 34,
+    "amp": 38,
+    "lt": 60,
+    "gt": 62,
+    "nbsp": 160,
+    "iexcl": 161,
+    "cent": 162,
+    "pound": 163,
+    "curren": 164,
+    "yen": 165,
+    "brvbar": 166,
+    "sect": 167,
+    "uml": 168,
+    "copy": 169,
+    "ordf": 170,
+    "laquo": 171,
+    "not": 172,
+    "shy": 173,
+    "reg": 174,
+    "macr": 175,
+    "deg": 176,
+    "plusmn": 177,
+    "sup2": 178,
+    "sup3": 179,
+    "acute": 180,
+    "micro": 181,
+    "para": 182,
+    "middot": 183,
+    "cedil": 184,
+    "sup1": 185,
+    "ordm": 186,
+    "raquo": 187,
+    "frac14": 188,
+    "frac12": 189,
+    "frac34": 190,
+    "iquest": 191,
+    "Agrave": 192,
+    "Aacute": 193,
+    "Acirc": 194,
+    "Atilde": 195,
+    "Auml": 196,
+    "Aring": 197,
+    "AElig": 198,
+    "Ccedil": 199,
+    "Egrave": 200,
+    "Eacute": 201,
+    "Ecirc": 202,
+    "Euml": 203,
+    "Igrave": 204,
+    "Iacute": 205,
+    "Icirc": 206,
+    "Iuml": 207,
+    "ETH": 208,
+    "Ntilde": 209,
+    "Ograve": 210,
+    "Oacute": 211,
+    "Ocirc": 212,
+    "Otilde": 213,
+    "Ouml": 214,
+    "times": 215,
+    "Oslash": 216,
+    "Ugrave": 217,
+    "Uacute": 218,
+    "Ucirc": 219,
+    "Uuml": 220,
+    "Yacute": 221,
+    "THORN": 222,
+    "szlig": 223,
+    "agrave": 224,
+    "aacute": 225,
+    "acirc": 226,
+    "atilde": 227,
+    "auml": 228,
+    "aring": 229,
+    "aelig": 230,
+    "ccedil": 231,
+    "egrave": 232,
+    "eacute": 233,
+    "ecirc": 234,
+    "euml": 235,
+    "igrave": 236,
+    "iacute": 237,
+    "icirc": 238,
+    "iuml": 239,
+    "eth": 240,
+    "ntilde": 241,
+    "ograve": 242,
+    "oacute": 243,
+    "ocirc": 244,
+    "otilde": 245,
+    "ouml": 246,
+    "divide": 247,
+    "oslash": 248,
+    "ugrave": 249,
+    "uacute": 250,
+    "ucirc": 251,
+    "uuml": 252,
+    "yacute": 253,
+    "thorn": 254,
+    "yuml": 255,
+    "OElig": 338,
+    "oelig": 339,
+    "Scaron": 352,
+    "fnof": 402,
+    "scaron": 353,
+    "Yuml": 376,
+    "circ": 710,
+    "tilde": 732,
+    "Alpha": 913,
+    "Beta": 914,
+    "Gamma": 915,
+    "Delta": 916,
+    "Epsilon": 917,
+    "Zeta": 918,
+    "Eta": 919,
+    "Theta": 920,
+    "Iota": 921,
+    "Kappa": 922,
+    "Lambda": 923,
+    "Mu": 924,
+    "Nu": 925,
+    "Xi": 926,
+    "Omicron": 927,
+    "Pi": 928,
+    "Rho": 929,
+    "Sigma": 931,
+    "Tau": 932,
+    "Upsilon": 933,
+    "Phi": 934,
+    "Chi": 935,
+    "Psi": 936,
+    "Omega": 937,
+    "alpha": 945,
+    "beta": 946,
+    "gamma": 947,
+    "delta": 948,
+    "epsilon": 949,
+    "zeta": 950,
+    "eta": 951,
+    "theta": 952,
+    "iota": 953,
+    "kappa": 954,
+    "lambda": 955,
+    "mu": 956,
+    "nu": 957,
+    "xi": 958,
+    "omicron": 959,
+    "pi": 960,
+    "rho": 961,
+    "sigmaf": 962,
+    "sigma": 963,
+    "tau": 964,
+    "upsilon": 965,
+    "phi": 966,
+    "chi": 967,
+    "psi": 968,
+    "omega": 969,
+    "thetasym": 977,
+    "upsih": 978,
+    "piv": 982,
+    "bull": 8226,
+    "hellip": 8230,
+    "prime": 8242,
+    "Prime": 8243,
+    "oline": 8254,
+    "frasl": 8260,
+    "ensp": 8194,
+    "emsp": 8195,
+    "thinsp": 8201,
+    "zwnj": 8204,
+    "zwj": 8205,
+    "lrm": 8206,
+    "rlm": 8207,
+    "ndash": 8211,
+    "mdash": 8212,
+    "lsquo": 8216,
+    "rsquo": 8217,
+    "sbquo": 8218,
+    "ldquo": 8220,
+    "rdquo": 8221,
+    "bdquo": 8222,
+    "dagger": 8224,
+    "Dagger": 8225,
+    "permil": 8240,
+    "lsaquo": 8249,
+    "rsaquo": 8250,
+    "euro": 8364,
+    "weierp": 8472,
+    "image": 8465,
+    "real": 8476,
+    "trade": 8482,
+    "alefsym": 8501,
+    "larr": 8592,
+    "uarr": 8593,
+    "rarr": 8594,
+    "darr": 8595,
+    "harr": 8596,
+    "crarr": 8629,
+    "lArr": 8656,
+    "uArr": 8657,
+    "rArr": 8658,
+    "dArr": 8659,
+    "hArr": 8660,
+    "forall": 8704,
+    "part": 8706,
+    "exist": 8707,
+    "empty": 8709,
+    "nabla": 8711,
+    "isin": 8712,
+    "notin": 8713,
+    "ni": 8715,
+    "prod": 8719,
+    "sum": 8721,
+    "minus": 8722,
+    "lowast": 8727,
+    "radic": 8730,
+    "prop": 8733,
+    "infin": 8734,
+    "ang": 8736,
+    "and": 8743,
+    "or": 8744,
+    "cap": 8745,
+    "cup": 8746,
+    "int": 8747,
+    "there4": 8756,
+    "sim": 8764,
+    "cong": 8773,
+    "asymp": 8776,
+    "ne": 8800,
+    "equiv": 8801,
+    "le": 8804,
+    "ge": 8805,
+    "sub": 8834,
+    "sup": 8835,
+    "nsub": 8836,
+    "sube": 8838,
+    "supe": 8839,
+    "oplus": 8853,
+    "otimes": 8855,
+    "perp": 8869,
+    "sdot": 8901,
+    "lceil": 8968,
+    "rceil": 8969,
+    "lfloor": 8970,
+    "rfloor": 8971,
+    "lang": 9001,
+    "rang": 9002,
+    "loz": 9674,
+    "spades": 9824,
+    "clubs": 9827,
+    "hearts": 9829,
+    "diams": 9830
+}

lib/core/dump.py

@@ -617,9 +617,10 @@ class Dump(object):
                                 warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath)
                                 logger.warn(warnMsg)
 
-                                with open(filepath, "wb") as f:
+                                with openFile(filepath, "w+b", None) as f:
                                     _ = safechardecode(value, True)
                                     f.write(_)
+
                         except magic.MagicException as ex:
                             logger.debug(getSafeExString(ex))
 
@@ -675,28 +676,28 @@ class Dump(object):
             else:
                 colConsiderStr = " '%s' was" % unsafeSQLIdentificatorNaming(column)
 
-            msg = "column%s found in the " % colConsiderStr
-            msg += "following databases:"
-            self._write(msg)
-
-            _ = {}
-
+            found = {}
             for db, tblData in dbs.items():
                 for tbl, colData in tblData.items():
                     for col, dataType in colData.items():
                         if column.lower() in col.lower():
-                            if db in _:
-                                if tbl in _[db]:
-                                    _[db][tbl][col] = dataType
+                            if db in found:
+                                if tbl in found[db]:
+                                    found[db][tbl][col] = dataType
                                 else:
-                                    _[db][tbl] = {col: dataType}
+                                    found[db][tbl] = {col: dataType}
                             else:
-                                _[db] = {}
-                                _[db][tbl] = {col: dataType}
+                                found[db] = {}
+                                found[db][tbl] = {col: dataType}
 
                             continue
 
-            self.dbTableColumns(_)
+            if found:
+                msg = "column%s found in the " % colConsiderStr
+                msg += "following databases:"
+                self._write(msg)
+
+                self.dbTableColumns(found)
 
     def sqlQuery(self, query, queryRes):
         self.string(query, queryRes, content_type=CONTENT_TYPE.SQL_QUERY)

lib/core/enums.py

@@ -130,12 +130,12 @@ class HASH(object):
     MSSQL_NEW = r'(?i)\A0x0200[0-9a-f]{8}[0-9a-f]{128}\Z'
     ORACLE = r'(?i)\As:[0-9a-f]{60}\Z'
     ORACLE_OLD = r'(?i)\A[0-9a-f]{16}\Z'
-    MD5_GENERIC = r'(?i)\A[0-9a-f]{32}\Z'
-    SHA1_GENERIC = r'(?i)\A[0-9a-f]{40}\Z'
+    MD5_GENERIC = r'(?i)\A(0x)?[0-9a-f]{32}\Z'
+    SHA1_GENERIC = r'(?i)\A(0x)?[0-9a-f]{40}\Z'
     SHA224_GENERIC = r'(?i)\A[0-9a-f]{56}\Z'
-    SHA256_GENERIC = r'(?i)\A[0-9a-f]{64}\Z'
+    SHA256_GENERIC = r'(?i)\A(0x)?[0-9a-f]{64}\Z'
     SHA384_GENERIC = r'(?i)\A[0-9a-f]{96}\Z'
-    SHA512_GENERIC = r'(?i)\A[0-9a-f]{128}\Z'
+    SHA512_GENERIC = r'(?i)\A(0x)?[0-9a-f]{128}\Z'
     CRYPT_GENERIC = r'\A(?!\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\Z)(?![0-9]+\Z)[./0-9A-Za-z]{13}\Z'
     JOOMLA = r'\A[0-9a-f]{32}:\w{32}\Z'
     WORDPRESS = r'\A\$P\$[./0-9a-zA-Z]{31}\Z'
@@ -244,8 +244,8 @@ class HASHDB_KEYS(object):
     OS = "OS"
 
 class REDIRECTION(object):
-    YES = "Y"
-    NO = "N"
+    YES = 'Y'
+    NO = 'N'
 
 class PAYLOAD(object):
     SQLINJECTION = {

lib/core/option.py

@@ -293,6 +293,7 @@ def _setRequestFromFile():
     if conf.requestFile:
         for requestFile in re.split(PARAMETER_SPLITTING_REGEX, conf.requestFile):
             requestFile = safeExpandUser(requestFile)
+            url = None
             seen = set()
 
             if not checkFile(requestFile, False):
@@ -311,6 +312,11 @@ def _setRequestFromFile():
                         conf.multipleTargets = True
                     seen.add(url)
 
+            if url is None:
+                errMsg = "specified file '%s' " % requestFile
+                errMsg += "does not contain a valid HTTP request"
+                raise SqlmapDataException(errMsg)
+
     if conf.secondReq:
         conf.secondReq = safeExpandUser(conf.secondReq)
 
@@ -455,12 +461,14 @@ def _findPageForms():
     if conf.url and not checkConnection():
         return
 
+    found = False
     infoMsg = "searching for forms"
     logger.info(infoMsg)
 
     if not any((conf.bulkFile, conf.googleDork, conf.sitemapUrl)):
         page, _, _ = Request.queryPage(content=True)
-        findPageForms(page, conf.url, True, True)
+        if findPageForms(page, conf.url, True, True):
+            found = True
     else:
         if conf.bulkFile:
             targets = getFileItems(conf.bulkFile)
@@ -473,7 +481,8 @@ def _findPageForms():
             try:
                 target = targets[i]
                 page, _, _ = Request.getPage(url=target.strip(), cookie=conf.cookie, crawling=True, raise404=False)
-                findPageForms(page, target, False, True)
+                if findPageForms(page, target, False, True):
+                    found = True
 
                 if conf.verbose in (1, 2):
                     status = '%d/%d links visited (%d%%)' % (i + 1, len(targets), round(100.0 * (i + 1) / len(targets)))
@@ -484,6 +493,10 @@ def _findPageForms():
                 errMsg = "problem occurred while searching for forms at '%s' ('%s')" % (target, getSafeExString(ex))
                 logger.error(errMsg)
 
+    if not found:
+        warnMsg = "no forms found"
+        logger.warn(warnMsg)
+
 def _setDBMSAuthentication():
     """
     Check and set the DBMS authentication credentials to run statements as
@@ -522,26 +535,14 @@ def _setMetasploit():
             errMsg = "sqlmap requires third-party module 'pywin32' "
             errMsg += "in order to use Metasploit functionalities on "
             errMsg += "Windows. You can download it from "
-            errMsg += "'https://sourceforge.net/projects/pywin32/files/pywin32/'"
+            errMsg += "'https://github.com/mhammond/pywin32'"
             raise SqlmapMissingDependence(errMsg)
 
         if not conf.msfPath:
-            def _(key, value):
-                retVal = None
-
-                try:
-                    from six.moves.winreg import ConnectRegistry, OpenKey, QueryValueEx, HKEY_LOCAL_MACHINE
-                    _ = ConnectRegistry(None, HKEY_LOCAL_MACHINE)
-                    _ = OpenKey(_, key)
-                    retVal = QueryValueEx(_, value)[0]
-                except:
-                    logger.debug("unable to identify Metasploit installation path via registry key")
-
-                return retVal
-
-            conf.msfPath = _(r"SOFTWARE\Rapid7\Metasploit", "Location")
-            if conf.msfPath:
-                conf.msfPath = os.path.join(conf.msfPath, "msf3")
+            for candidate in os.environ.get("PATH", "").split(';'):
+                if all(_ in candidate for _ in ("metasploit", "bin")):
+                    conf.msfPath = os.path.dirname(candidate.rstrip('\\'))
+                    break
 
     if conf.osSmb:
         isAdmin = runningAsAdmin()
@@ -1301,6 +1302,9 @@ def _setHTTPExtraHeaders():
 
                 if header and value:
                     conf.httpHeaders.append((header, value))
+            elif headerValue.startswith('@'):
+                checkFile(headerValue[1:])
+                kb.headersFile = headerValue[1:]
             else:
                 errMsg = "invalid header value: %s. Valid header format is 'name:value'" % repr(headerValue).lstrip('u')
                 raise SqlmapSyntaxException(errMsg)
@@ -1453,6 +1457,9 @@ def _createHomeDirectories():
     Creates directories inside sqlmap's home directory
     """
 
+    if conf.get("purge"):
+        return
+
     for context in "output", "history":
         directory = paths["SQLMAP_%s_PATH" % context.upper()]
         try:
@@ -1463,7 +1470,7 @@ def _createHomeDirectories():
             open(_, "w+b").close()
             os.remove(_)
 
-            if conf.outputDir and context == "output":
+            if conf.get("outputDir") and context == "output":
                 warnMsg = "using '%s' as the %s directory" % (directory, context)
                 logger.warn(warnMsg)
         except (OSError, IOError) as ex:
@@ -1557,6 +1564,15 @@ def _cleanupOptions():
     else:
         conf.testParameter = []
 
+    if conf.ignoreCode:
+        try:
+            conf.ignoreCode = [int(_) for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.ignoreCode)]
+        except ValueError:
+            errMsg = "options '--ignore-code' should contain a list of integer values"
+            raise SqlmapSyntaxException(errMsg)
+    else:
+        conf.ignoreCode = []
+
     if conf.paramFilter:
         conf.paramFilter = [_.strip() for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.paramFilter.upper())]
     else:
@@ -1576,8 +1592,19 @@ def _cleanupOptions():
         conf.user = conf.user.replace(" ", "")
 
     if conf.rParam:
-        conf.rParam = conf.rParam.replace(" ", "")
-        conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
+        if all(_ in conf.rParam for _ in ('=', ',')):
+            original = conf.rParam
+            conf.rParam = []
+            for part in original.split(';'):
+                if '=' in part:
+                    left, right = part.split('=', 1)
+                    conf.rParam.append(left)
+                    kb.randomPool[left] = filterNone(_.strip() for _ in right.split(','))
+                else:
+                    conf.rParam.append(part)
+        else:
+            conf.rParam = conf.rParam.replace(" ", "")
+            conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
     else:
         conf.rParam = []
 
@@ -1881,6 +1908,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.forceWhere = None
     kb.futileUnion = None
     kb.heavilyDynamic = False
+    kb.headersFile = None
     kb.headersFp = {}
     kb.heuristicDbms = None
     kb.heuristicExtendedDbms = None
@@ -1901,12 +1929,13 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.lastParserStatus = None
 
     kb.locks = AttribDict()
-    for _ in ("cache", "connError", "count", "handlers", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
+    for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
         kb.locks[_] = threading.Lock()
 
     kb.matchRatio = None
     kb.maxConnectionsFlag = False
     kb.mergeCookies = None
+    kb.multipleCtrlC = False
     kb.negativeLogic = False
     kb.nullConnection = None
     kb.oldMsf = None
@@ -1939,6 +1968,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.processUserMarks = None
     kb.proxyAuthHeader = None
     kb.queryCounter = 0
+    kb.randomPool = {}
     kb.redirectChoice = None
     kb.reflectiveMechanism = True
     kb.reflectiveCounters = {REFLECTIVE_COUNTER.MISS: 0, REFLECTIVE_COUNTER.HIT: 0}

lib/core/optiondict.py

@@ -31,6 +31,7 @@ optDict = {
         "loadCookies": "string",
         "dropSetCookie": "boolean",
         "agent": "string",
+        "mobile": "boolean",
         "randomAgent": "boolean",
         "host": "string",
         "referer": "string",
@@ -38,7 +39,7 @@ optDict = {
         "authType": "string",
         "authCred": "string",
         "authFile": "string",
-        "ignoreCode": "integer",
+        "ignoreCode": "string",
         "ignoreProxy": "boolean",
         "ignoreRedirects": "boolean",
         "ignoreTimeouts": "boolean",
@@ -100,6 +101,7 @@ optDict = {
         "notString": "string",
         "regexp": "string",
         "code": "integer",
+        "smart": "boolean",
         "textOnly": "boolean",
         "titles": "boolean",
     },
@@ -197,10 +199,12 @@ optDict = {
 
     "General": {
         "trafficFile": "string",
+        "answers": "string",
         "batch": "boolean",
         "binaryFields": "string",
         "charset": "string",
         "checkInternet": "boolean",
+        "cleanup": "boolean",
         "crawlDepth": "integer",
         "crawlExclude": "string",
         "csvDel": "string",
@@ -210,6 +214,7 @@ optDict = {
         "flushSession": "boolean",
         "forms": "boolean",
         "freshQueries": "boolean",
+        "googlePage": "integer",
         "harFile": "string",
         "hexConvert": "boolean",
         "outputDir": "string",
@@ -218,28 +223,23 @@ optDict = {
         "repair": "boolean",
         "saveConfig": "string",
         "scope": "string",
+        "skipWaf": "boolean",
         "testFilter": "string",
         "testSkip": "string",
-        "updateAll": "boolean",
+        "webRoot": "string",
     },
 
     "Miscellaneous": {
         "alert": "string",
-        "answers": "string",
         "beep": "boolean",
-        "cleanup": "boolean",
         "dependencies": "boolean",
         "disableColoring": "boolean",
-        "googlePage": "integer",
         "listTampers": "boolean",
-        "mobile": "boolean",
         "offline": "boolean",
         "purge": "boolean",
-        "skipWaf": "boolean",
-        "smart": "boolean",
         "tmpDir": "string",
-        "webRoot": "string",
         "wizard": "boolean",
+        "updateAll": "boolean",
         "verbose": "integer",
     },
 

lib/core/revision.py

@@ -38,7 +38,7 @@ def getRevisionNumber():
     while True:
         if filePath and os.path.isfile(filePath):
             with openFile(filePath, "r") as f:
-                content = f.read()
+                content = getText(f.read())
                 filePath = None
                 if content.startswith("ref: "):
                     filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.7.0"
+VERSION = "1.3.9.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -240,7 +240,7 @@ IS_WIN = PLATFORM == "nt"
 IS_TTY = os.isatty(sys.stdout.fileno())
 
 # DBMS system databases
-MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb")
+MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb", "Resource", "ReportServer", "ReportServerTempDB")
 MYSQL_SYSTEM_DBS = ("information_schema", "mysql", "performance_schema", "sys")
 PGSQL_SYSTEM_DBS = ("information_schema", "pg_catalog", "pg_toast", "pgagent")
 ORACLE_SYSTEM_DBS = ('ANONYMOUS', 'APEX_030200', 'APEX_PUBLIC_USER', 'APPQOSSYS', 'BI', 'CTXSYS', 'DBSNMP', 'DIP', 'EXFSYS', 'FLOWS_%', 'FLOWS_FILES', 'HR', 'IX', 'LBACSYS', 'MDDATA', 'MDSYS', 'MGMT_VIEW', 'OC', 'OE', 'OLAPSYS', 'ORACLE_OCM', 'ORDDATA', 'ORDPLUGINS', 'ORDSYS', 'OUTLN', 'OWBSYS', 'PM', 'SCOTT', 'SH', 'SI_INFORMTN_SCHEMA', 'SPATIAL_CSW_ADMIN_USR', 'SPATIAL_WFS_ADMIN_USR', 'SYS', 'SYSMAN', 'SYSTEM', 'WKPROXY', 'WKSYS', 'WK_TEST', 'WMSYS', 'XDB', 'XS$NULL')
@@ -251,7 +251,7 @@ MAXDB_SYSTEM_DBS = ("SYSINFO", "DOMAIN")
 SYBASE_SYSTEM_DBS = ("master", "model", "sybsystemdb", "sybsystemprocs")
 DB2_SYSTEM_DBS = ("NULLID", "SQLJ", "SYSCAT", "SYSFUN", "SYSIBM", "SYSIBMADM", "SYSIBMINTERNAL", "SYSIBMTS", "SYSPROC", "SYSPUBLIC", "SYSSTAT", "SYSTOOLS")
 HSQLDB_SYSTEM_DBS = ("INFORMATION_SCHEMA", "SYSTEM_LOB")
-H2_SYSTEM_DBS = ("INFORMATION_SCHEMA")
+H2_SYSTEM_DBS = ("INFORMATION_SCHEMA",)
 INFORMIX_SYSTEM_DBS = ("sysmaster", "sysutils", "sysuser", "sysadmin")
 
 MSSQL_ALIASES = ("microsoft sql server", "mssqlserver", "mssql", "ms")
@@ -333,6 +333,9 @@ BLANK = "<blank>"
 # String representation for current database
 CURRENT_DB = "CD"
 
+# String representation for current user
+CURRENT_USER = "CU"
+
 # Name of SQLite file used for storing session data
 SESSION_SQLITE_FILE = "session.sqlite"
 
@@ -540,7 +543,7 @@ BRUTE_TABLE_EXISTS_TEMPLATE = "EXISTS(SELECT %d FROM %s)"
 BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT %s FROM %s)"
 
 # Data inside shellcodeexec to be filled with random string
-SHELLCODEEXEC_RANDOM_STRING_MARKER = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+SHELLCODEEXEC_RANDOM_STRING_MARKER = b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
 
 # Period after last-update to start nagging about the old revision
 LAST_UPDATE_NAGGING_DAYS = 60
@@ -807,9 +810,6 @@ BRUTE_DOC_ROOT_PREFIXES = {
     OS.WINDOWS: ("/xampp", "/Program Files/xampp", "/wamp", "/Program Files/wampp", "/apache", "/Program Files/Apache Group/Apache", "/Program Files/Apache Group/Apache2", "/Program Files/Apache Group/Apache2.2", "/Program Files/Apache Group/Apache2.4", "/Inetpub/wwwroot", "/Inetpub/wwwroot/%TARGET%", "/Inetpub/vhosts/%TARGET%")
 }
 
-# Table prefix to use in "takeover" functionalities (i.e. auxiliary tables used by sqlmap at the vulnerable DBMS)
-TAKEOVER_TABLE_PREFIX = "sqlmap"
-
 # Suffixes used in brute force search for web server document root
 BRUTE_DOC_ROOT_SUFFIXES = ("", "html", "htdocs", "httpdocs", "php", "public", "src", "site", "build", "web", "www", "data", "sites/all", "www/build")
 

lib/core/subprocessng.py

@@ -13,6 +13,7 @@ import subprocess
 import time
 
 from lib.core.compat import buffer
+from lib.core.convert import getBytes
 from lib.core.settings import IS_WIN
 
 if IS_WIN:
@@ -192,6 +193,8 @@ def send_all(p, data):
     if not data:
         return
 
+    data = getBytes(data)
+
     while len(data):
         sent = p.send(data)
         if not isinstance(sent, int):

lib/core/target.py

@@ -111,7 +111,7 @@ def _setRequestParams():
         def process(match, repl):
             retVal = match.group(0)
 
-            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]):
+            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]) and match.group("name") == match.group("name").strip('\\'):
                 retVal = repl
                 while True:
                     _ = re.search(r"\\g<([^>]+)>", retVal)
@@ -121,11 +121,12 @@ def _setRequestParams():
                         break
                 if kb.customInjectionMark in retVal:
                     hintNames.append((retVal.split(kb.customInjectionMark)[0], match.group("name")))
+
             return retVal
 
         if kb.processUserMarks is None and kb.customInjectionMark in conf.data:
-            message = "custom injection marker ('%s') found in option " % kb.customInjectionMark
-            message += "'--data'. Do you want to process it? [Y/n/q] "
+            message = "custom injection marker ('%s') found in POST " % kb.customInjectionMark
+            message += "body. Do you want to process it? [Y/n/q] "
             choice = readInput(message, default='Y').upper()
 
             if choice == 'Q':

lib/core/threads.py

@@ -95,11 +95,13 @@ def exceptionHandledFunction(threadFunction, silent=False):
         kb.threadException = True
         raise
     except Exception as ex:
-        if not silent and kb.get("threadContinue"):
-            errMsg = ex.message if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, ex.message)
+        from lib.core.common import getSafeExString
+
+        if not silent and kb.get("threadContinue") and not isinstance(ex, SqlmapUserQuitException):
+            errMsg = getSafeExString(ex) if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, getSafeExString(ex))
             logger.error("thread %s: '%s'" % (threading.currentThread().getName(), errMsg))
 
-            if conf.get("verbose") > 1 and not isinstance(ex, (SqlmapUserQuitException,)):
+            if conf.get("verbose") > 1 and not isinstance(ex, SqlmapConnectionException):
                 traceback.print_exc()
 
 def setDaemon(thread):
@@ -112,6 +114,7 @@ def setDaemon(thread):
 def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardException=True, threadChoice=False, startThreadMsg=True):
     threads = []
 
+    kb.multipleCtrlC = False
     kb.threadContinue = True
     kb.threadException = False
     kb.technique = ThreadData.technique
@@ -183,6 +186,7 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
                 pass
 
         except KeyboardInterrupt:
+            kb.multipleCtrlC = True
             raise SqlmapThreadException("user aborted (Ctrl+C was pressed multiple times)")
 
         if forwardException:
@@ -193,17 +197,19 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
         kb.threadException = True
         logger.error("thread %s: '%s'" % (threading.currentThread().getName(), ex))
 
-        if conf.get("verbose") > 1:
+        if conf.get("verbose") > 1 and isinstance(ex, SqlmapValueException):
             traceback.print_exc()
 
     except:
-        from lib.core.common import unhandledExceptionMessage
-
         print()
-        kb.threadException = True
-        errMsg = unhandledExceptionMessage()
-        logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
-        traceback.print_exc()
+
+        if not kb.multipleCtrlC:
+            from lib.core.common import unhandledExceptionMessage
+
+            kb.threadException = True
+            errMsg = unhandledExceptionMessage()
+            logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
+            traceback.print_exc()
 
     finally:
         kb.threadContinue = True

lib/core/update.py

@@ -105,6 +105,7 @@ def update():
 
         dataToStdout("\r[%s] [INFO] update in progress" % time.strftime("%X"))
 
+        output = ""
         try:
             process = subprocess.Popen("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=paths.SQLMAP_ROOT_PATH)
             pollProcess(process, True)

lib/core/wordlist.py

@@ -18,7 +18,9 @@ class Wordlist(six.Iterator):
     Iterator for looping over a large dictionaries
 
     >>> from lib.core.option import paths
-    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.string_types)
+    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.binary_type)
+    True
+    >>> isinstance(next(Wordlist(paths.WORDLIST)), six.binary_type)
     True
     """
 
@@ -58,7 +60,7 @@ class Wordlist(six.Iterator):
                     raise SqlmapDataException(errMsg)
                 self.fp = _.open(_.namelist()[0])
             else:
-                self.fp = open(self.current, 'r')
+                self.fp = open(self.current, "rb")
             self.iter = iter(self.fp)
 
         self.index += 1

lib/parse/cmdline.py

@@ -59,6 +59,7 @@ from lib.core.dicts import DEPRECATED_OPTIONS
 from lib.core.enums import AUTOCOMPLETE_TYPE
 from lib.core.exception import SqlmapShellQuitException
 from lib.core.exception import SqlmapSyntaxException
+from lib.core.option import _createHomeDirectories
 from lib.core.settings import BASIC_HELP_ITEMS
 from lib.core.settings import DUMMY_URL
 from lib.core.settings import INFERENCE_UNKNOWN_CHAR
@@ -88,691 +89,695 @@ def cmdLineParser(argv=None):
     parser = ArgumentParser(usage=usage)
 
     try:
-        parser.add_argument("--hh", dest="advancedHelp",
-                          action="store_true",
-                          help="Show advanced help message and exit")
+        parser.add_argument("--hh", dest="advancedHelp", action="store_true",
+            help="Show advanced help message and exit")
 
-        parser.add_argument("--version", dest="showVersion",
-                          action="store_true",
-                          help="Show program's version number and exit")
+        parser.add_argument("--version", dest="showVersion", action="store_true",
+            help="Show program's version number and exit")
 
         parser.add_argument("-v", dest="verbose", type=int,
-                          help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
+            help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
 
         # Target options
-        target = parser.add_argument_group("Target", "At least one of these "
-                             "options has to be provided to define the target(s)")
+        target = parser.add_argument_group("Target", "At least one of these options has to be provided to define the target(s)")
 
-        target.add_argument("-d", dest="direct", help="Connection string "
-                          "for direct database connection")
+        target.add_argument("-d", dest="direct",
+            help="Connection string for direct database connection")
 
-        target.add_argument("-u", "--url", dest="url", help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
+        target.add_argument("-u", "--url", dest="url",
+            help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
 
-        target.add_argument("-l", dest="logFile", help="Parse target(s) from Burp "
-                          "or WebScarab proxy log file")
+        target.add_argument("-l", dest="logFile",
+            help="Parse target(s) from Burp or WebScarab proxy log file")
 
-        target.add_argument("-x", dest="sitemapUrl", help="Parse target(s) from remote sitemap(.xml) file")
+        target.add_argument("-x", dest="sitemapUrl",
+            help="Parse target(s) from remote sitemap(.xml) file")
 
-        target.add_argument("-m", dest="bulkFile", help="Scan multiple targets given "
-                          "in a textual file ")
+        target.add_argument("-m", dest="bulkFile",
+            help="Scan multiple targets given in a textual file ")
 
         target.add_argument("-r", dest="requestFile",
-                          help="Load HTTP request from a file")
+            help="Load HTTP request from a file")
 
         target.add_argument("-g", dest="googleDork",
-                          help="Process Google dork results as target URLs")
+            help="Process Google dork results as target URLs")
 
         target.add_argument("-c", dest="configFile",
-                          help="Load options from a configuration INI file")
+            help="Load options from a configuration INI file")
 
         # Request options
-        request = parser.add_argument_group("Request", "These options can be used "
-                              "to specify how to connect to the target URL")
+        request = parser.add_argument_group("Request", "These options can be used to specify how to connect to the target URL")
 
         request.add_argument("--method", dest="method",
-                           help="Force usage of given HTTP method (e.g. PUT)")
+            help="Force usage of given HTTP method (e.g. PUT)")
 
         request.add_argument("--data", dest="data",
-                           help="Data string to be sent through POST (e.g. \"id=1\")")
+            help="Data string to be sent through POST (e.g. \"id=1\")")
 
         request.add_argument("--param-del", dest="paramDel",
-                           help="Character used for splitting parameter values (e.g. &)")
+            help="Character used for splitting parameter values (e.g. &)")
 
         request.add_argument("--cookie", dest="cookie",
-                           help="HTTP Cookie header value (e.g. \"PHPSESSID=a8d127e..\")")
+            help="HTTP Cookie header value (e.g. \"PHPSESSID=a8d127e..\")")
 
         request.add_argument("--cookie-del", dest="cookieDel",
-                           help="Character used for splitting cookie values (e.g. ;)")
+            help="Character used for splitting cookie values (e.g. ;)")
 
         request.add_argument("--load-cookies", dest="loadCookies",
-                           help="File containing cookies in Netscape/wget format")
+            help="File containing cookies in Netscape/wget format")
 
         request.add_argument("--drop-set-cookie", dest="dropSetCookie", action="store_true",
-                           help="Ignore Set-Cookie header from response")
+            help="Ignore Set-Cookie header from response")
 
         request.add_argument("--user-agent", dest="agent",
-                           help="HTTP User-Agent header value")
+            help="HTTP User-Agent header value")
+
+        request.add_argument("--mobile", dest="mobile", action="store_true",
+            help="Imitate smartphone through HTTP User-Agent header")
 
         request.add_argument("--random-agent", dest="randomAgent", action="store_true",
-                           help="Use randomly selected HTTP User-Agent header value")
+            help="Use randomly selected HTTP User-Agent header value")
 
         request.add_argument("--host", dest="host",
-                           help="HTTP Host header value")
+            help="HTTP Host header value")
 
         request.add_argument("--referer", dest="referer",
-                           help="HTTP Referer header value")
+            help="HTTP Referer header value")
 
         request.add_argument("-H", "--header", dest="header",
-                           help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")")
+            help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")")
 
         request.add_argument("--headers", dest="headers",
-                           help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
+            help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
 
         request.add_argument("--auth-type", dest="authType",
-                           help="HTTP authentication type (Basic, Digest, NTLM or PKI)")
+            help="HTTP authentication type (Basic, Digest, NTLM or PKI)")
 
         request.add_argument("--auth-cred", dest="authCred",
-                           help="HTTP authentication credentials (name:password)")
+            help="HTTP authentication credentials (name:password)")
 
         request.add_argument("--auth-file", dest="authFile",
-                           help="HTTP authentication PEM cert/private key file")
+            help="HTTP authentication PEM cert/private key file")
 
-        request.add_argument("--ignore-code", dest="ignoreCode", type=int,
-                           help="Ignore (problematic) HTTP error code (e.g. 401)")
+        request.add_argument("--ignore-code", dest="ignoreCode",
+            help="Ignore (problematic) HTTP error code (e.g. 401)")
 
         request.add_argument("--ignore-proxy", dest="ignoreProxy", action="store_true",
-                           help="Ignore system default proxy settings")
+            help="Ignore system default proxy settings")
 
         request.add_argument("--ignore-redirects", dest="ignoreRedirects", action="store_true",
-                           help="Ignore redirection attempts")
+            help="Ignore redirection attempts")
 
         request.add_argument("--ignore-timeouts", dest="ignoreTimeouts", action="store_true",
-                           help="Ignore connection timeouts")
+            help="Ignore connection timeouts")
 
         request.add_argument("--proxy", dest="proxy",
-                           help="Use a proxy to connect to the target URL")
+            help="Use a proxy to connect to the target URL")
 
         request.add_argument("--proxy-cred", dest="proxyCred",
-                           help="Proxy authentication credentials (name:password)")
+            help="Proxy authentication credentials (name:password)")
 
         request.add_argument("--proxy-file", dest="proxyFile",
-                           help="Load proxy list from a file")
+            help="Load proxy list from a file")
 
         request.add_argument("--tor", dest="tor", action="store_true",
-                           help="Use Tor anonymity network")
+            help="Use Tor anonymity network")
 
         request.add_argument("--tor-port", dest="torPort",
-                           help="Set Tor proxy port other than default")
+            help="Set Tor proxy port other than default")
 
         request.add_argument("--tor-type", dest="torType",
-                           help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
+            help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
 
         request.add_argument("--check-tor", dest="checkTor", action="store_true",
-                           help="Check to see if Tor is used properly")
+            help="Check to see if Tor is used properly")
 
         request.add_argument("--delay", dest="delay", type=float,
-                           help="Delay in seconds between each HTTP request")
+            help="Delay in seconds between each HTTP request")
 
         request.add_argument("--timeout", dest="timeout", type=float,
-                           help="Seconds to wait before timeout connection (default %d)" % defaults.timeout)
+            help="Seconds to wait before timeout connection (default %d)" % defaults.timeout)
 
         request.add_argument("--retries", dest="retries", type=int,
-                           help="Retries when the connection timeouts (default %d)" % defaults.retries)
+            help="Retries when the connection timeouts (default %d)" % defaults.retries)
 
         request.add_argument("--randomize", dest="rParam",
-                           help="Randomly change value for given parameter(s)")
+            help="Randomly change value for given parameter(s)")
 
         request.add_argument("--safe-url", dest="safeUrl",
-                           help="URL address to visit frequently during testing")
+            help="URL address to visit frequently during testing")
 
         request.add_argument("--safe-post", dest="safePost",
-                           help="POST data to send to a safe URL")
+            help="POST data to send to a safe URL")
 
         request.add_argument("--safe-req", dest="safeReqFile",
-                           help="Load safe HTTP request from a file")
+            help="Load safe HTTP request from a file")
 
         request.add_argument("--safe-freq", dest="safeFreq", type=int,
-                           help="Test requests between two visits to a given safe URL")
+            help="Test requests between two visits to a given safe URL")
 
         request.add_argument("--skip-urlencode", dest="skipUrlEncode", action="store_true",
-                           help="Skip URL encoding of payload data")
+            help="Skip URL encoding of payload data")
 
         request.add_argument("--csrf-token", dest="csrfToken",
-                           help="Parameter used to hold anti-CSRF token")
+            help="Parameter used to hold anti-CSRF token")
 
         request.add_argument("--csrf-url", dest="csrfUrl",
-                           help="URL address to visit for extraction of anti-CSRF token")
+            help="URL address to visit for extraction of anti-CSRF token")
 
         request.add_argument("--force-ssl", dest="forceSSL", action="store_true",
-                           help="Force usage of SSL/HTTPS")
+            help="Force usage of SSL/HTTPS")
 
         request.add_argument("--chunked", dest="chunked", action="store_true",
-                           help="Use HTTP chunked transfer encoded (POST) requests")
+            help="Use HTTP chunked transfer encoded (POST) requests")
 
         request.add_argument("--hpp", dest="hpp", action="store_true",
-                           help="Use HTTP parameter pollution method")
+            help="Use HTTP parameter pollution method")
 
         request.add_argument("--eval", dest="evalCode",
-                           help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
+            help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
 
         # Optimization options
         optimization = parser.add_argument_group("Optimization", "These options can be used to optimize the performance of sqlmap")
 
         optimization.add_argument("-o", dest="optimize", action="store_true",
-                                help="Turn on all optimization switches")
+            help="Turn on all optimization switches")
 
         optimization.add_argument("--predict-output", dest="predictOutput", action="store_true",
-                                help="Predict common queries output")
+            help="Predict common queries output")
 
         optimization.add_argument("--keep-alive", dest="keepAlive", action="store_true",
-                                help="Use persistent HTTP(s) connections")
+            help="Use persistent HTTP(s) connections")
 
         optimization.add_argument("--null-connection", dest="nullConnection", action="store_true",
-                                help="Retrieve page length without actual HTTP response body")
+            help="Retrieve page length without actual HTTP response body")
 
         optimization.add_argument("--threads", dest="threads", type=int,
-                                help="Max number of concurrent HTTP(s) "
-                                "requests (default %d)" % defaults.threads)
+            help="Max number of concurrent HTTP(s) requests (default %d)" % defaults.threads)
 
         # Injection options
         injection = parser.add_argument_group("Injection", "These options can be used to specify which parameters to test for, provide custom injection payloads and optional tampering scripts")
 
         injection.add_argument("-p", dest="testParameter",
-                             help="Testable parameter(s)")
+            help="Testable parameter(s)")
 
         injection.add_argument("--skip", dest="skip",
-                             help="Skip testing for given parameter(s)")
+            help="Skip testing for given parameter(s)")
 
         injection.add_argument("--skip-static", dest="skipStatic", action="store_true",
-                             help="Skip testing parameters that not appear to be dynamic")
+            help="Skip testing parameters that not appear to be dynamic")
 
         injection.add_argument("--param-exclude", dest="paramExclude",
-                             help="Regexp to exclude parameters from testing (e.g. \"ses\")")
+            help="Regexp to exclude parameters from testing (e.g. \"ses\")")
 
         injection.add_argument("--param-filter", dest="paramFilter",
-                             help="Select testable parameter(s) by place (e.g. \"POST\")")
+            help="Select testable parameter(s) by place (e.g. \"POST\")")
 
         injection.add_argument("--dbms", dest="dbms",
-                             help="Force back-end DBMS to provided value")
+            help="Force back-end DBMS to provided value")
 
         injection.add_argument("--dbms-cred", dest="dbmsCred",
-                             help="DBMS authentication credentials (user:password)")
+            help="DBMS authentication credentials (user:password)")
 
         injection.add_argument("--os", dest="os",
-                             help="Force back-end DBMS operating system to provided value")
+            help="Force back-end DBMS operating system to provided value")
 
         injection.add_argument("--invalid-bignum", dest="invalidBignum", action="store_true",
-                             help="Use big numbers for invalidating values")
+            help="Use big numbers for invalidating values")
 
         injection.add_argument("--invalid-logical", dest="invalidLogical", action="store_true",
-                             help="Use logical operations for invalidating values")
+            help="Use logical operations for invalidating values")
 
         injection.add_argument("--invalid-string", dest="invalidString", action="store_true",
-                             help="Use random strings for invalidating values")
+            help="Use random strings for invalidating values")
 
         injection.add_argument("--no-cast", dest="noCast", action="store_true",
-                             help="Turn off payload casting mechanism")
+            help="Turn off payload casting mechanism")
 
         injection.add_argument("--no-escape", dest="noEscape", action="store_true",
-                             help="Turn off string escaping mechanism")
+            help="Turn off string escaping mechanism")
 
         injection.add_argument("--prefix", dest="prefix",
-                             help="Injection payload prefix string")
+            help="Injection payload prefix string")
 
         injection.add_argument("--suffix", dest="suffix",
-                             help="Injection payload suffix string")
+            help="Injection payload suffix string")
 
         injection.add_argument("--tamper", dest="tamper",
-                             help="Use given script(s) for tampering injection data")
+            help="Use given script(s) for tampering injection data")
 
         # Detection options
         detection = parser.add_argument_group("Detection", "These options can be used to customize the detection phase")
 
         detection.add_argument("--level", dest="level", type=int,
-                             help="Level of tests to perform (1-5, default %d)" % defaults.level)
+            help="Level of tests to perform (1-5, default %d)" % defaults.level)
 
         detection.add_argument("--risk", dest="risk", type=int,
-                             help="Risk of tests to perform (1-3, default %d)" % defaults.risk)
+            help="Risk of tests to perform (1-3, default %d)" % defaults.risk)
 
         detection.add_argument("--string", dest="string",
-                             help="String to match when query is evaluated to True")
+            help="String to match when query is evaluated to True")
 
         detection.add_argument("--not-string", dest="notString",
-                             help="String to match when query is evaluated to False")
+            help="String to match when query is evaluated to False")
 
         detection.add_argument("--regexp", dest="regexp",
-                             help="Regexp to match when query is evaluated to True")
+            help="Regexp to match when query is evaluated to True")
 
         detection.add_argument("--code", dest="code", type=int,
-                             help="HTTP code to match when query is evaluated to True")
+            help="HTTP code to match when query is evaluated to True")
+
+        detection.add_argument("--smart", dest="smart", action="store_true",
+            help="Perform thorough tests only if positive heuristic(s)")
 
         detection.add_argument("--text-only", dest="textOnly", action="store_true",
-                             help="Compare pages based only on the textual content")
+            help="Compare pages based only on the textual content")
 
         detection.add_argument("--titles", dest="titles", action="store_true",
-                             help="Compare pages based only on their titles")
+            help="Compare pages based only on their titles")
 
         # Techniques options
         techniques = parser.add_argument_group("Techniques", "These options can be used to tweak testing of specific SQL injection techniques")
 
         techniques.add_argument("--technique", dest="technique",
-                              help="SQL injection techniques to use (default \"%s\")" % defaults.technique)
+            help="SQL injection techniques to use (default \"%s\")" % defaults.technique)
 
         techniques.add_argument("--time-sec", dest="timeSec", type=int,
-                              help="Seconds to delay the DBMS response (default %d)" % defaults.timeSec)
+            help="Seconds to delay the DBMS response (default %d)" % defaults.timeSec)
 
         techniques.add_argument("--union-cols", dest="uCols",
-                              help="Range of columns to test for UNION query SQL injection")
+            help="Range of columns to test for UNION query SQL injection")
 
         techniques.add_argument("--union-char", dest="uChar",
-                              help="Character to use for bruteforcing number of columns")
+            help="Character to use for bruteforcing number of columns")
 
         techniques.add_argument("--union-from", dest="uFrom",
-                              help="Table to use in FROM part of UNION query SQL injection")
+            help="Table to use in FROM part of UNION query SQL injection")
 
         techniques.add_argument("--dns-domain", dest="dnsDomain",
-                              help="Domain name used for DNS exfiltration attack")
+            help="Domain name used for DNS exfiltration attack")
 
         techniques.add_argument("--second-url", dest="secondUrl",
-                              help="Resulting page URL searched for second-order response")
+            help="Resulting page URL searched for second-order response")
 
         techniques.add_argument("--second-req", dest="secondReq",
-                              help="Load second-order HTTP request from file")
+            help="Load second-order HTTP request from file")
 
         # Fingerprint options
         fingerprint = parser.add_argument_group("Fingerprint")
 
         fingerprint.add_argument("-f", "--fingerprint", dest="extensiveFp", action="store_true",
-                               help="Perform an extensive DBMS version fingerprint")
+            help="Perform an extensive DBMS version fingerprint")
 
         # Enumeration options
         enumeration = parser.add_argument_group("Enumeration", "These options can be used to enumerate the back-end database management system information, structure and data contained in the tables. Moreover you can run your own SQL statements")
 
         enumeration.add_argument("-a", "--all", dest="getAll", action="store_true",
-                               help="Retrieve everything")
+            help="Retrieve everything")
 
         enumeration.add_argument("-b", "--banner", dest="getBanner", action="store_true",
-                               help="Retrieve DBMS banner")
+            help="Retrieve DBMS banner")
 
         enumeration.add_argument("--current-user", dest="getCurrentUser", action="store_true",
-                               help="Retrieve DBMS current user")
+            help="Retrieve DBMS current user")
 
         enumeration.add_argument("--current-db", dest="getCurrentDb", action="store_true",
-                               help="Retrieve DBMS current database")
+            help="Retrieve DBMS current database")
 
         enumeration.add_argument("--hostname", dest="getHostname", action="store_true",
-                               help="Retrieve DBMS server hostname")
+            help="Retrieve DBMS server hostname")
 
         enumeration.add_argument("--is-dba", dest="isDba", action="store_true",
-                               help="Detect if the DBMS current user is DBA")
+            help="Detect if the DBMS current user is DBA")
 
         enumeration.add_argument("--users", dest="getUsers", action="store_true",
-                               help="Enumerate DBMS users")
+            help="Enumerate DBMS users")
 
         enumeration.add_argument("--passwords", dest="getPasswordHashes", action="store_true",
-                               help="Enumerate DBMS users password hashes")
+            help="Enumerate DBMS users password hashes")
 
         enumeration.add_argument("--privileges", dest="getPrivileges", action="store_true",
-                               help="Enumerate DBMS users privileges")
+            help="Enumerate DBMS users privileges")
 
         enumeration.add_argument("--roles", dest="getRoles", action="store_true",
-                               help="Enumerate DBMS users roles")
+            help="Enumerate DBMS users roles")
 
         enumeration.add_argument("--dbs", dest="getDbs", action="store_true",
-                               help="Enumerate DBMS databases")
+            help="Enumerate DBMS databases")
 
         enumeration.add_argument("--tables", dest="getTables", action="store_true",
-                               help="Enumerate DBMS database tables")
+            help="Enumerate DBMS database tables")
 
         enumeration.add_argument("--columns", dest="getColumns", action="store_true",
-                               help="Enumerate DBMS database table columns")
+            help="Enumerate DBMS database table columns")
 
         enumeration.add_argument("--schema", dest="getSchema", action="store_true",
-                               help="Enumerate DBMS schema")
+            help="Enumerate DBMS schema")
 
         enumeration.add_argument("--count", dest="getCount", action="store_true",
-                               help="Retrieve number of entries for table(s)")
+            help="Retrieve number of entries for table(s)")
 
         enumeration.add_argument("--dump", dest="dumpTable", action="store_true",
-                               help="Dump DBMS database table entries")
+            help="Dump DBMS database table entries")
 
         enumeration.add_argument("--dump-all", dest="dumpAll", action="store_true",
-                               help="Dump all DBMS databases tables entries")
+            help="Dump all DBMS databases tables entries")
 
         enumeration.add_argument("--search", dest="search", action="store_true",
-                               help="Search column(s), table(s) and/or database name(s)")
+            help="Search column(s), table(s) and/or database name(s)")
 
         enumeration.add_argument("--comments", dest="getComments", action="store_true",
-                               help="Check for DBMS comments during enumeration")
+            help="Check for DBMS comments during enumeration")
 
         enumeration.add_argument("--statements", dest="getStatements", action="store_true",
-                               help="Retrieve SQL statements being run on DBMS")
+            help="Retrieve SQL statements being run on DBMS")
 
         enumeration.add_argument("-D", dest="db",
-                               help="DBMS database to enumerate")
+            help="DBMS database to enumerate")
 
         enumeration.add_argument("-T", dest="tbl",
-                               help="DBMS database table(s) to enumerate")
+            help="DBMS database table(s) to enumerate")
 
         enumeration.add_argument("-C", dest="col",
-                               help="DBMS database table column(s) to enumerate")
+            help="DBMS database table column(s) to enumerate")
 
         enumeration.add_argument("-X", dest="exclude",
-                               help="DBMS database identifier(s) to not enumerate")
+            help="DBMS database identifier(s) to not enumerate")
 
         enumeration.add_argument("-U", dest="user",
-                               help="DBMS user to enumerate")
+            help="DBMS user to enumerate")
 
         enumeration.add_argument("--exclude-sysdbs", dest="excludeSysDbs", action="store_true",
-                               help="Exclude DBMS system databases when enumerating tables")
+            help="Exclude DBMS system databases when enumerating tables")
 
         enumeration.add_argument("--pivot-column", dest="pivotColumn",
-                               help="Pivot column name")
+            help="Pivot column name")
 
         enumeration.add_argument("--where", dest="dumpWhere",
-                               help="Use WHERE condition while table dumping")
+            help="Use WHERE condition while table dumping")
 
         enumeration.add_argument("--start", dest="limitStart", type=int,
-                               help="First dump table entry to retrieve")
+            help="First dump table entry to retrieve")
 
         enumeration.add_argument("--stop", dest="limitStop", type=int,
-                               help="Last dump table entry to retrieve")
+            help="Last dump table entry to retrieve")
 
         enumeration.add_argument("--first", dest="firstChar", type=int,
-                               help="First query output word character to retrieve")
+            help="First query output word character to retrieve")
 
         enumeration.add_argument("--last", dest="lastChar", type=int,
-                               help="Last query output word character to retrieve")
+            help="Last query output word character to retrieve")
 
         enumeration.add_argument("--sql-query", dest="sqlQuery",
-                               help="SQL statement to be executed")
+            help="SQL statement to be executed")
 
         enumeration.add_argument("--sql-shell", dest="sqlShell", action="store_true",
-                               help="Prompt for an interactive SQL shell")
+            help="Prompt for an interactive SQL shell")
 
         enumeration.add_argument("--sql-file", dest="sqlFile",
-                               help="Execute SQL statements from given file(s)")
+            help="Execute SQL statements from given file(s)")
 
         # Brute force options
         brute = parser.add_argument_group("Brute force", "These options can be used to run brute force checks")
 
         brute.add_argument("--common-tables", dest="commonTables", action="store_true",
-                         help="Check existence of common tables")
+            help="Check existence of common tables")
 
         brute.add_argument("--common-columns", dest="commonColumns", action="store_true",
-                         help="Check existence of common columns")
+            help="Check existence of common columns")
 
         brute.add_argument("--common-files", dest="commonFiles", action="store_true",
-                         help="Check existence of common files")
+            help="Check existence of common files")
 
         # User-defined function options
         udf = parser.add_argument_group("User-defined function injection", "These options can be used to create custom user-defined functions")
 
         udf.add_argument("--udf-inject", dest="udfInject", action="store_true",
-                       help="Inject custom user-defined functions")
+            help="Inject custom user-defined functions")
 
         udf.add_argument("--shared-lib", dest="shLib",
-                       help="Local path of the shared library")
+            help="Local path of the shared library")
 
         # File system options
         filesystem = parser.add_argument_group("File system access", "These options can be used to access the back-end database management system underlying file system")
 
         filesystem.add_argument("--file-read", dest="fileRead",
-                              help="Read a file from the back-end DBMS file system")
+            help="Read a file from the back-end DBMS file system")
 
         filesystem.add_argument("--file-write", dest="fileWrite",
-                              help="Write a local file on the back-end DBMS file system")
+            help="Write a local file on the back-end DBMS file system")
 
         filesystem.add_argument("--file-dest", dest="fileDest",
-                              help="Back-end DBMS absolute filepath to write to")
+            help="Back-end DBMS absolute filepath to write to")
 
         # Takeover options
         takeover = parser.add_argument_group("Operating system access", "These options can be used to access the back-end database management system underlying operating system")
 
         takeover.add_argument("--os-cmd", dest="osCmd",
-                            help="Execute an operating system command")
+            help="Execute an operating system command")
 
         takeover.add_argument("--os-shell", dest="osShell", action="store_true",
-                            help="Prompt for an interactive operating system shell")
+            help="Prompt for an interactive operating system shell")
 
         takeover.add_argument("--os-pwn", dest="osPwn", action="store_true",
-                            help="Prompt for an OOB shell, Meterpreter or VNC")
+            help="Prompt for an OOB shell, Meterpreter or VNC")
 
         takeover.add_argument("--os-smbrelay", dest="osSmb", action="store_true",
-                            help="One click prompt for an OOB shell, Meterpreter or VNC")
+            help="One click prompt for an OOB shell, Meterpreter or VNC")
 
         takeover.add_argument("--os-bof", dest="osBof", action="store_true",
-                            help="Stored procedure buffer overflow "
+            help="Stored procedure buffer overflow "
                                  "exploitation")
 
         takeover.add_argument("--priv-esc", dest="privEsc", action="store_true",
-                            help="Database process user privilege escalation")
+            help="Database process user privilege escalation")
 
         takeover.add_argument("--msf-path", dest="msfPath",
-                            help="Local path where Metasploit Framework is installed")
+            help="Local path where Metasploit Framework is installed")
 
         takeover.add_argument("--tmp-path", dest="tmpPath",
-                            help="Remote absolute path of temporary files directory")
+            help="Remote absolute path of temporary files directory")
 
         # Windows registry options
         windows = parser.add_argument_group("Windows registry access", "These options can be used to access the back-end database management system Windows registry")
 
         windows.add_argument("--reg-read", dest="regRead", action="store_true",
-                           help="Read a Windows registry key value")
+            help="Read a Windows registry key value")
 
         windows.add_argument("--reg-add", dest="regAdd", action="store_true",
-                           help="Write a Windows registry key value data")
+            help="Write a Windows registry key value data")
 
         windows.add_argument("--reg-del", dest="regDel", action="store_true",
-                           help="Delete a Windows registry key value")
+            help="Delete a Windows registry key value")
 
         windows.add_argument("--reg-key", dest="regKey",
-                           help="Windows registry key")
+            help="Windows registry key")
 
         windows.add_argument("--reg-value", dest="regVal",
-                           help="Windows registry key value")
+            help="Windows registry key value")
 
         windows.add_argument("--reg-data", dest="regData",
-                           help="Windows registry key value data")
+            help="Windows registry key value data")
 
         windows.add_argument("--reg-type", dest="regType",
-                           help="Windows registry key value type")
+            help="Windows registry key value type")
 
         # General options
         general = parser.add_argument_group("General", "These options can be used to set some general working parameters")
 
         general.add_argument("-s", dest="sessionFile",
-                           help="Load session from a stored (.sqlite) file")
+            help="Load session from a stored (.sqlite) file")
 
         general.add_argument("-t", dest="trafficFile",
-                           help="Log all HTTP traffic into a textual file")
+            help="Log all HTTP traffic into a textual file")
+
+        general.add_argument("--answers", dest="answers",
+            help="Set predefined answers (e.g. \"quit=N,follow=N\")")
 
         general.add_argument("--batch", dest="batch", action="store_true",
-                           help="Never ask for user input, use the default behavior")
+            help="Never ask for user input, use the default behavior")
 
         general.add_argument("--binary-fields", dest="binaryFields",
-                           help="Result fields having binary values (e.g. \"digest\")")
+            help="Result fields having binary values (e.g. \"digest\")")
 
         general.add_argument("--check-internet", dest="checkInternet", action="store_true",
-                           help="Check Internet connection before assessing the target")
+            help="Check Internet connection before assessing the target")
+
+        general.add_argument("--cleanup", dest="cleanup", action="store_true",
+            help="Clean up the DBMS from sqlmap specific UDF and tables")
 
         general.add_argument("--crawl", dest="crawlDepth", type=int,
-                           help="Crawl the website starting from the target URL")
+            help="Crawl the website starting from the target URL")
 
         general.add_argument("--crawl-exclude", dest="crawlExclude",
-                           help="Regexp to exclude pages from crawling (e.g. \"logout\")")
+            help="Regexp to exclude pages from crawling (e.g. \"logout\")")
 
         general.add_argument("--csv-del", dest="csvDel",
-                           help="Delimiting character used in CSV output (default \"%s\")" % defaults.csvDel)
+            help="Delimiting character used in CSV output (default \"%s\")" % defaults.csvDel)
 
         general.add_argument("--charset", dest="charset",
-                           help="Blind SQL injection charset (e.g. \"0123456789abcdef\")")
+            help="Blind SQL injection charset (e.g. \"0123456789abcdef\")")
 
         general.add_argument("--dump-format", dest="dumpFormat",
-                           help="Format of dumped data (CSV (default), HTML or SQLITE)")
+            help="Format of dumped data (CSV (default), HTML or SQLITE)")
 
         general.add_argument("--encoding", dest="encoding",
-                           help="Character encoding used for data retrieval (e.g. GBK)")
+            help="Character encoding used for data retrieval (e.g. GBK)")
 
         general.add_argument("--eta", dest="eta", action="store_true",
-                           help="Display for each output the estimated time of arrival")
+            help="Display for each output the estimated time of arrival")
 
         general.add_argument("--flush-session", dest="flushSession", action="store_true",
-                           help="Flush session files for current target")
+            help="Flush session files for current target")
 
         general.add_argument("--forms", dest="forms", action="store_true",
-                           help="Parse and test forms on target URL")
+            help="Parse and test forms on target URL")
 
         general.add_argument("--fresh-queries", dest="freshQueries", action="store_true",
-                           help="Ignore query results stored in session file")
+            help="Ignore query results stored in session file")
+
+        general.add_argument("--gpage", dest="googlePage", type=int,
+            help="Use Google dork results from specified page number")
 
         general.add_argument("--har", dest="harFile",
-                           help="Log all HTTP traffic into a HAR file")
+            help="Log all HTTP traffic into a HAR file")
 
         general.add_argument("--hex", dest="hexConvert", action="store_true",
-                           help="Use hex conversion during data retrieval")
+            help="Use hex conversion during data retrieval")
 
         general.add_argument("--output-dir", dest="outputDir", action="store",
-                           help="Custom output directory path")
+            help="Custom output directory path")
 
         general.add_argument("--parse-errors", dest="parseErrors", action="store_true",
-                           help="Parse and display DBMS error messages from responses")
+            help="Parse and display DBMS error messages from responses")
 
         general.add_argument("--preprocess", dest="preprocess",
-                           help="Use given script(s) for preprocessing of response data")
+            help="Use given script(s) for preprocessing of response data")
 
         general.add_argument("--repair", dest="repair", action="store_true",
-                           help="Redump entries having unknown character marker (%s)" % INFERENCE_UNKNOWN_CHAR)
+            help="Redump entries having unknown character marker (%s)" % INFERENCE_UNKNOWN_CHAR)
 
         general.add_argument("--save", dest="saveConfig",
-                           help="Save options to a configuration INI file")
+            help="Save options to a configuration INI file")
 
         general.add_argument("--scope", dest="scope",
-                           help="Regexp to filter targets from provided proxy log")
+            help="Regexp to filter targets from provided proxy log")
+
+        general.add_argument("--skip-waf", dest="skipWaf", action="store_true",
+            help="Skip heuristic detection of WAF/IPS protection")
+
+        general.add_argument("--table-prefix", dest="tablePrefix",
+            help="Prefix used for temporary tables (default: \"%s\")" % defaults.tablePrefix)
 
         general.add_argument("--test-filter", dest="testFilter",
-                           help="Select tests by payloads and/or titles (e.g. ROW)")
+            help="Select tests by payloads and/or titles (e.g. ROW)")
 
         general.add_argument("--test-skip", dest="testSkip",
-                           help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
+            help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
 
-        general.add_argument("--update", dest="updateAll", action="store_true",
-                           help="Update sqlmap")
+        general.add_argument("--web-root", dest="webRoot",
+            help="Web server document root directory (e.g. \"/var/www\")")
 
         # Miscellaneous options
-        miscellaneous = parser.add_argument_group("Miscellaneous")
+        miscellaneous = parser.add_argument_group("Miscellaneous", "These options do not fit into any other category")
 
         miscellaneous.add_argument("-z", dest="mnemonics",
-                                 help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
+            help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
 
         miscellaneous.add_argument("--alert", dest="alert",
-                                 help="Run host OS command(s) when SQL injection is found")
-
-        miscellaneous.add_argument("--answers", dest="answers",
-                                 help="Set predefined answers (e.g. \"quit=N,follow=N\")")
+            help="Run host OS command(s) when SQL injection is found")
 
         miscellaneous.add_argument("--beep", dest="beep", action="store_true",
-                                 help="Beep on question and/or when SQL injection is found")
-
-        miscellaneous.add_argument("--cleanup", dest="cleanup", action="store_true",
-                                 help="Clean up the DBMS from sqlmap specific UDF and tables")
+            help="Beep on question and/or when SQL injection is found")
 
         miscellaneous.add_argument("--dependencies", dest="dependencies", action="store_true",
-                                 help="Check for missing (optional) sqlmap dependencies")
+            help="Check for missing (optional) sqlmap dependencies")
 
         miscellaneous.add_argument("--disable-coloring", dest="disableColoring", action="store_true",
-                                 help="Disable console output coloring")
-
-        miscellaneous.add_argument("--gpage", dest="googlePage", type=int,
-                                 help="Use Google dork results from specified page number")
+            help="Disable console output coloring")
 
         miscellaneous.add_argument("--list-tampers", dest="listTampers", action="store_true",
-                                 help="Display list of available tamper scripts")
-
-        miscellaneous.add_argument("--mobile", dest="mobile", action="store_true",
-                                 help="Imitate smartphone through HTTP User-Agent header")
+            help="Display list of available tamper scripts")
 
         miscellaneous.add_argument("--offline", dest="offline", action="store_true",
-                                 help="Work in offline mode (only use session data)")
+            help="Work in offline mode (only use session data)")
 
         miscellaneous.add_argument("--purge", dest="purge", action="store_true",
-                                 help="Safely remove all content from sqlmap data directory")
-
-        miscellaneous.add_argument("--skip-waf", dest="skipWaf", action="store_true",
-                                 help="Skip heuristic detection of WAF/IPS protection")
-
-        miscellaneous.add_argument("--smart", dest="smart", action="store_true",
-                                 help="Conduct thorough tests only if positive heuristic(s)")
+            help="Safely remove all content from sqlmap data directory")
 
         miscellaneous.add_argument("--sqlmap-shell", dest="sqlmapShell", action="store_true",
-                                 help="Prompt for an interactive sqlmap shell")
+            help="Prompt for an interactive sqlmap shell")
 
         miscellaneous.add_argument("--tmp-dir", dest="tmpDir",
-                                 help="Local directory for storing temporary files")
+            help="Local directory for storing temporary files")
 
-        miscellaneous.add_argument("--web-root", dest="webRoot",
-                                 help="Web server document root directory (e.g. \"/var/www\")")
+        miscellaneous.add_argument("--update", dest="updateAll", action="store_true",
+            help="Update sqlmap")
 
         miscellaneous.add_argument("--wizard", dest="wizard", action="store_true",
-                                 help="Simple wizard interface for beginner users")
+            help="Simple wizard interface for beginner users")
 
         # Hidden and/or experimental options
         parser.add_argument("--base64", dest="base64Parameter",
-                          help=SUPPRESS)
-#                          help="Parameter(s) containing Base64 encoded values")
+            help=SUPPRESS)  # "Parameter(s) containing Base64 encoded values"
 
         parser.add_argument("--crack", dest="hashFile",
-                          help=SUPPRESS)
-#                          help="Load and crack hashes from a file (standalone)")
+            help=SUPPRESS)  # "Load and crack hashes from a file (standalone)"
 
         parser.add_argument("--dummy", dest="dummy", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--murphy-rate", dest="murphyRate", type=int,
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--debug", dest="debug", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--disable-precon", dest="disablePrecon", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--disable-stats", dest="disableStats", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--profile", dest="profile", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--force-dbms", dest="forceDbms",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--force-dns", dest="forceDns", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
+
+        parser.add_argument("--force-partial", dest="forcePartial", action="store_true",
+            help=SUPPRESS)
 
         parser.add_argument("--force-pivoting", dest="forcePivoting", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--smoke-test", dest="smokeTest", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--live-test", dest="liveTest", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--vuln-test", dest="vulnTest", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--stop-fail", dest="stopFail", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
-        parser.add_argument("--run-case", dest="runCase", help=SUPPRESS)
+        parser.add_argument("--run-case", dest="runCase",
+            help=SUPPRESS)
 
         # API options
         parser.add_argument("--api", dest="api", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
-        parser.add_argument("--taskid", dest="taskid", help=SUPPRESS)
+        parser.add_argument("--taskid", dest="taskid",
+            help=SUPPRESS)
 
-        parser.add_argument("--database", dest="database", help=SUPPRESS)
+        parser.add_argument("--database", dest="database",
+            help=SUPPRESS)
 
         # Dirty hack to display longer options without breaking into two lines
         if hasattr(parser, "formatter"):
@@ -805,7 +810,7 @@ def cmdLineParser(argv=None):
                     action.option_strings = ["-hh"]
                     break
 
-        ## Dirty hack for inherent help message of switch '-h'
+        # Dirty hack for inherent help message of switch '-h'
         if hasattr(parser, "get_option"):
             option = parser.get_option("-h")
             option.help = option.help.capitalize().replace("this help", "basic help")
@@ -831,6 +836,8 @@ def cmdLineParser(argv=None):
         prompt = "--sqlmap-shell" in argv
 
         if prompt:
+            _createHomeDirectories()
+
             parser.usage = ""
             cmdLineOptions.sqlmapShell = True
 

lib/request/basic.py

@@ -34,6 +34,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.decorators import cachedmethod
+from lib.core.dicts import HTML_ENTITIES
 from lib.core.enums import DBMS
 from lib.core.enums import HTTP_HEADER
 from lib.core.enums import PLACE
@@ -49,7 +50,6 @@ from lib.core.settings import SELECT_FROM_TABLE_REGEX
 from lib.core.settings import VIEWSTATE_REGEX
 from lib.parse.headers import headersParser
 from lib.parse.html import htmlParser
-from lib.utils.htmlentities import htmlEntities
 from thirdparty import six
 from thirdparty.chardet import detect
 from thirdparty.identywaf import identYwaf
@@ -341,7 +341,7 @@ def decodePage(page, contentEncoding, contentType):
             page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page)
 
         # e.g. &
-        page = re.sub(b"&([^;]+);", lambda _: six.int2byte(htmlEntities[getText(_.group(1))]) if htmlEntities.get(getText(_.group(1)), 256) < 256 else _.group(0), page)
+        page = re.sub(b"&([^;]+);", lambda _: six.int2byte(HTML_ENTITIES[getText(_.group(1))]) if HTML_ENTITIES.get(getText(_.group(1)), 256) < 256 else _.group(0), page)
 
         kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
 
@@ -364,7 +364,7 @@ def decodePage(page, contentEncoding, contentType):
             page = re.sub(r"&#(\d+);", _, page)
 
         # e.g. &zeta;
-        page = re.sub(r"&([^;]+);", lambda _: _unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
+        page = re.sub(r"&([^;]+);", lambda _: _unichr(HTML_ENTITIES[_.group(1)]) if HTML_ENTITIES.get(_.group(1), 0) > 255 else _.group(0), page)
 
     return page
 

lib/request/connect.py

@@ -7,6 +7,7 @@ See the file 'LICENSE' for copying permission
 
 import binascii
 import logging
+import random
 import re
 import socket
 import string
@@ -41,6 +42,7 @@ from lib.core.common import getRequestHeader
 from lib.core.common import getSafeExString
 from lib.core.common import isMultiThreadMode
 from lib.core.common import logHTTPTraffic
+from lib.core.common import openFile
 from lib.core.common import popValue
 from lib.core.common import pushValue
 from lib.core.common import randomizeParameterValue
@@ -59,6 +61,7 @@ from lib.core.common import wasLastResponseDelayed
 from lib.core.compat import patchHeaders
 from lib.core.compat import xrange
 from lib.core.convert import getBytes
+from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -425,6 +428,14 @@ class Connect(object):
             if auxHeaders:
                 headers = forgeHeaders(auxHeaders, headers)
 
+            if kb.headersFile:
+                content = openFile(kb.headersFile, "rb").read()
+                for line in content.split("\n"):
+                    line = getText(line.strip())
+                    if ':' in line:
+                        header, value = line.split(':', 1)
+                        headers[header] = value
+
             for key, value in list(headers.items()):
                 del headers[key]
                 if isinstance(value, six.string_types):
@@ -646,7 +657,7 @@ class Connect(object):
             if not multipart:
                 logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
 
-            if ex.code != conf.ignoreCode:
+            if ex.code not in (conf.ignoreCode or []):
                 if ex.code == _http_client.UNAUTHORIZED:
                     errMsg = "not authorized, try to provide right HTTP "
                     errMsg += "authentication type and valid credentials (%d)" % code
@@ -748,21 +759,17 @@ class Connect(object):
 
                 if kb.connErrorCounter >= MAX_CONSECUTIVE_CONNECTION_ERRORS and kb.connErrorChoice is None:
                     message = "there seems to be a continuous problem with connection to the target. "
-                    message += "Are you sure that you want to continue "
-                    message += "with further target testing? [y/N] "
+                    message += "Are you sure that you want to continue? [y/N] "
 
                     kb.connErrorChoice = readInput(message, default='N', boolean=True)
 
-                if kb.connErrorChoice is not None:
-                    if kb.connErrorChoice:
-                        raise SqlmapConnectionException(warnMsg)
-                    else:
-                        raise SqlmapUserQuitException
+                if kb.connErrorChoice is False:
+                    raise SqlmapUserQuitException
 
             if "forcibly closed" in tbMsg:
                 logger.critical(warnMsg)
                 return None, None, None
-            elif ignoreTimeout and any(_ in tbMsg for _ in ("timed out", "IncompleteRead")):
+            elif ignoreTimeout and any(_ in tbMsg for _ in ("timed out", "IncompleteRead", "Interrupted system call")):
                 return None if not conf.ignoreTimeouts else "", None, None
             elif threadData.retriesCount < conf.retries and not kb.threadException:
                 warnMsg += ". sqlmap is going to retry the request"
@@ -1102,7 +1109,8 @@ class Connect(object):
                 match = re.search(r"(\A|\b)%s=(?P<value>[^&;]*)" % re.escape(randomParameter), paramString)
                 if match:
                     origValue = match.group("value")
-                    retVal = re.sub(r"(\A|\b)%s=[^&;]*" % re.escape(randomParameter), "%s=%s" % (randomParameter, randomizeParameterValue(origValue)), paramString)
+                    newValue = randomizeParameterValue(origValue) if randomParameter not in kb.randomPool else random.sample(kb.randomPool[randomParameter], 1)[0]
+                    retVal = re.sub(r"(\A|\b)%s=[^&;]*" % re.escape(randomParameter), "%s=%s" % (randomParameter, newValue), paramString)
                 return retVal
 
             for randomParameter in conf.rParam:

lib/request/direct.py

@@ -25,7 +25,6 @@ from lib.core.enums import CUSTOM_LOGGING
 from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import TIMEOUT_STATE
-from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.core.settings import UNICODE_ENCODING
 from lib.utils.timeout import timeout
 
@@ -54,7 +53,7 @@ def direct(query, content=True):
 
     if not select and "EXEC " not in query.upper():
         timeout(func=conf.dbmsConnector.execute, args=(query,), duration=conf.timeout, default=None)
-    elif not (output and ("%soutput" % TAKEOVER_TABLE_PREFIX) not in query and ("%sfile" % TAKEOVER_TABLE_PREFIX) not in query):
+    elif not (output and ("%soutput" % conf.tablePrefix) not in query and ("%sfile" % conf.tablePrefix) not in query):
         output, state = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
         if state == TIMEOUT_STATE.NORMAL:
             hashDBWrite(query, output, True)

lib/request/dns.py

@@ -84,7 +84,7 @@ class DNSServer(object):
         except:
             pass
         finally:
-            if response and "google" in response:
+            if response and b"google" in response:
                 raise socket.error("another DNS service already running on *:53")
 
     def pop(self, prefix=None, suffix=None):

lib/request/inject.py

@@ -39,6 +39,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import queries
+from lib.core.decorators import lockedmethod
 from lib.core.decorators import stackedmethod
 from lib.core.dicts import FROM_DUMMY_TABLE
 from lib.core.enums import CHARSET_TYPE
@@ -164,7 +165,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
 
     initTechnique(getTechnique())
 
-    query = agent.prefixQuery(kb.injection.data[getTechnique()].vector)
+    query = agent.prefixQuery(getTechniqueData().vector)
     query = agent.suffixQuery(query)
     payload = agent.payload(newValue=query)
     count = None
@@ -312,7 +313,7 @@ def _goBooleanProxy(expression):
     initTechnique(getTechnique())
 
     if conf.dnsDomain:
-        query = agent.prefixQuery(kb.injection.data[getTechnique()].vector)
+        query = agent.prefixQuery(getTechniqueData().vector)
         query = agent.suffixQuery(query)
         payload = agent.payload(newValue=query)
         output = _goDns(payload, expression)
@@ -320,7 +321,7 @@ def _goBooleanProxy(expression):
         if output is not None:
             return output
 
-    vector = kb.injection.data[getTechnique()].vector
+    vector = getTechniqueData().vector
     vector = vector.replace(INFERENCE_MARKER, expression)
     query = agent.prefixQuery(vector)
     query = agent.suffixQuery(query)
@@ -351,6 +352,7 @@ def _goUnion(expression, unpack=True, dump=False):
 
     return output
 
+@lockedmethod
 @stackedmethod
 def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True):
     """

lib/request/rangehandler.py

@@ -24,5 +24,6 @@ class HTTPRangeHandler(_urllib.request.BaseHandler):
 
     def http_error_416(self, req, fp, code, msg, hdrs):
         # HTTP's Range Not Satisfiable error
-        errMsg = "Invalid range"
+        errMsg = "there was a problem while connecting "
+        errMsg += "target ('406 - Range Not Satisfiable')"
         raise SqlmapConnectionException(errMsg)

lib/takeover/metasploit.py

@@ -68,29 +68,11 @@ class Metasploit(object):
         self.payloadConnStr = None
         self.localIP = getLocalIP()
         self.remoteIP = getRemoteIP() or conf.hostname
-        self._msfCli = normalizePath(os.path.join(conf.msfPath, "msfcli"))
-        self._msfConsole = normalizePath(os.path.join(conf.msfPath, "msfconsole"))
-        self._msfEncode = normalizePath(os.path.join(conf.msfPath, "msfencode"))
-        self._msfPayload = normalizePath(os.path.join(conf.msfPath, "msfpayload"))
-        self._msfVenom = normalizePath(os.path.join(conf.msfPath, "msfvenom"))
-
-        if IS_WIN:
-            _ = conf.msfPath
-            while _:
-                if os.path.exists(os.path.join(_, "scripts")):
-                    _ = os.path.join(_, "scripts", "setenv.bat")
-                    break
-                else:
-                    old = _
-                    _ = normalizePath(os.path.join(_, ".."))
-                    if _ == old:
-                        break
-
-            self._msfCli = "%s & ruby %s" % (_, self._msfCli)
-            self._msfConsole = "%s & ruby %s" % (_, self._msfConsole)
-            self._msfEncode = "ruby %s" % self._msfEncode
-            self._msfPayload = "%s & ruby %s" % (_, self._msfPayload)
-            self._msfVenom = "%s & ruby %s" % (_, self._msfVenom)
+        self._msfCli = normalizePath(os.path.join(conf.msfPath, "msfcli%s" % (".bat" if IS_WIN else "")))
+        self._msfConsole = normalizePath(os.path.join(conf.msfPath, "msfconsole%s" % (".bat" if IS_WIN else "")))
+        self._msfEncode = normalizePath(os.path.join(conf.msfPath, "msfencode%s" % (".bat" if IS_WIN else "")))
+        self._msfPayload = normalizePath(os.path.join(conf.msfPath, "msfpayload%s" % (".bat" if IS_WIN else "")))
+        self._msfVenom = normalizePath(os.path.join(conf.msfPath, "msfvenom%s" % (".bat" if IS_WIN else "")))
 
         self._msfPayloadsList = {
             "windows": {
@@ -245,18 +227,15 @@ class Metasploit(object):
 
                     if not choice or choice == "2":
                         _payloadStr = "windows/meterpreter"
-
                         break
 
                     elif choice == "3":
                         _payloadStr = "windows/shell"
-
                         break
 
                     elif choice == "1":
                         if Backend.isDbms(DBMS.PGSQL):
                             logger.warn("beware that the VNC injection might not work")
-
                             break
 
                         elif Backend.isDbms(DBMS.MSSQL) and Backend.isVersionWithin(("2005", "2008")):
@@ -652,7 +631,7 @@ class Metasploit(object):
             self.shellcodeexecLocal = os.path.join(self.shellcodeexecLocal, "windows", "shellcodeexec.x%s.exe_" % "32")
             content = decloak(self.shellcodeexecLocal)
             if SHELLCODEEXEC_RANDOM_STRING_MARKER in content:
-                content = content.replace(SHELLCODEEXEC_RANDOM_STRING_MARKER, randomStr(len(SHELLCODEEXEC_RANDOM_STRING_MARKER)))
+                content = content.replace(SHELLCODEEXEC_RANDOM_STRING_MARKER, getBytes(randomStr(len(SHELLCODEEXEC_RANDOM_STRING_MARKER))))
                 _ = cloak(data=content)
                 handle, self.shellcodeexecLocal = tempfile.mkstemp(suffix="%s.exe_" % "32")
                 os.close(handle)

lib/takeover/web.py

@@ -21,6 +21,7 @@ from lib.core.common import getManualDirectories
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSQLSnippet
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import isTechniqueAvailable
 from lib.core.common import isWindowsDriveLetterPath
 from lib.core.common import normalizePath
@@ -149,7 +150,7 @@ class Web(object):
         query = ""
 
         if isTechniqueAvailable(getTechnique()):
-            where = kb.injection.data[getTechnique()].where
+            where = getTechniqueData().where
 
             if where == PAYLOAD.WHERE.NEGATIVE:
                 randInt = randomInt()

lib/techniques/blind/inference.py

@@ -8,7 +8,6 @@ See the file 'LICENSE' for copying permission
 from __future__ import division
 
 import re
-import threading
 import time
 
 from extra.safe2bin.safe2bin import safecharencode
@@ -23,6 +22,7 @@ from lib.core.common import getCharset
 from lib.core.common import getCounter
 from lib.core.common import getPartRun
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import goGoodSamaritan
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
@@ -161,7 +161,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             length = None
 
         showEta = conf.eta and isinstance(length, int)
-        numThreads = min(conf.threads or 0, length or 0) or 1
+
+        if kb.bruteMode:
+            numThreads = 1
+        else:
+            numThreads = min(conf.threads or 0, length or 0) or 1
 
         if showEta:
             progress = ProgressBar(maxValue=length)
@@ -173,22 +177,20 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             else:
                 numThreads = 1
 
-        if conf.threads == 1 and not timeBasedCompare and not conf.predictOutput:
+        if conf.threads == 1 and not any((timeBasedCompare, conf.predictOutput)):
             warnMsg = "running in a single-thread mode. Please consider "
             warnMsg += "usage of option '--threads' for faster data retrieval"
             singleTimeWarnMessage(warnMsg)
 
-        if conf.verbose in (1, 2) and not showEta and not conf.api:
-            if isinstance(length, int) and conf.threads > 1:
+        if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
+            if isinstance(length, int) and numThreads > 1:
                 dataToStdout("[%s] [INFO] retrieved: %s" % (time.strftime("%X"), "_" * min(length, conf.progressWidth)))
                 dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X"))
             else:
                 dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X"))
 
-        hintlock = threading.Lock()
-
         def tryHint(idx):
-            with hintlock:
+            with kb.locks.hint:
                 hintValue = kb.hintValue
 
             if payload is not None and len(hintValue or "") > 0 and len(hintValue) >= idx:
@@ -207,7 +209,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                 if result:
                     return hintValue[idx - 1]
 
-            with hintlock:
+            with kb.locks.hint:
                 kb.hintValue = ""
 
             return None
@@ -229,10 +231,10 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
             result = not Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
 
-            if result and timeBasedCompare and kb.injection.data[getTechnique()].trueCode:
-                result = threadData.lastCode == kb.injection.data[getTechnique()].trueCode
+            if result and timeBasedCompare and getTechniqueData().trueCode:
+                result = threadData.lastCode == getTechniqueData().trueCode
                 if not result:
-                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, kb.injection.data[getTechnique()].trueCode)
+                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, getTechniqueData().trueCode)
                     singleTimeWarnMessage(warnMsg)
 
             incrementCounter(getTechnique())
@@ -341,8 +343,8 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                     result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
                     incrementCounter(getTechnique())
 
-                    if not timeBasedCompare:
-                        unexpectedCode |= threadData.lastCode not in (kb.injection.data[getTechnique()].falseCode, kb.injection.data[getTechnique()].trueCode)
+                    if not timeBasedCompare and getTechniqueData() is not None:
+                        unexpectedCode |= threadData.lastCode not in (getTechniqueData().falseCode, getTechniqueData().trueCode)
                         if unexpectedCode:
                             warnMsg = "unexpected HTTP code '%s' detected. Will use (extra) validation step in similar cases" % threadData.lastCode
                             singleTimeWarnMessage(warnMsg)
@@ -458,7 +460,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                         return decodeIntToUnicode(candidates[0])
 
         # Go multi-threading (--threads > 1)
-        if conf.threads > 1 and isinstance(length, int) and length > 1:
+        if numThreads > 1 and isinstance(length, int) and length > 1:
             threadData.shared.value = [None] * length
             threadData.shared.index = [firstChar]    # As list for python nested function scoping
             threadData.shared.start = firstChar
@@ -516,7 +518,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                                 if (endCharIndex - startCharIndex == conf.progressWidth) and (endCharIndex < length - 1):
                                     output = output[:-2] + ".."
 
-                                if conf.verbose in (1, 2) and not showEta and not conf.api:
+                                if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
                                     _ = count - firstChar
                                     output += '_' * (min(length, conf.progressWidth) - len(output))
                                     status = ' %d/%d (%d%%)' % (_, length, int(100.0 * _ / length))
@@ -546,7 +548,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                 finalValue = "".join(value)
                 infoMsg = "\r[%s] [INFO] retrieved: %s" % (time.strftime("%X"), filterControlChars(finalValue))
 
-            if conf.verbose in (1, 2) and not showEta and infoMsg and not conf.api:
+            if conf.verbose in (1, 2) and infoMsg and not any((showEta, conf.api, kb.bruteMode)):
                 dataToStdout(infoMsg)
 
         # No multi-threading (--threads = 1)
@@ -570,7 +572,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                         # One-shot query containing equals commonValue
                         testValue = unescaper.escape("'%s'" % commonValue) if "'" not in commonValue else unescaper.escape("%s" % commonValue, quote=False)
 
-                        query = kb.injection.data[getTechnique()].vector
+                        query = getTechniqueData().vector
                         query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)%s%s" % (expressionUnescaped, INFERENCE_EQUALS_CHAR, testValue)))
                         query = agent.suffixQuery(query)
 
@@ -594,7 +596,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                         subquery = queries[Backend.getIdentifiedDbms()].substring.query % (expressionUnescaped, 1, len(commonPattern))
                         testValue = unescaper.escape("'%s'" % commonPattern) if "'" not in commonPattern else unescaper.escape("%s" % commonPattern, quote=False)
 
-                        query = kb.injection.data[getTechnique()].vector
+                        query = getTechniqueData().vector
                         query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)=%s" % (subquery, testValue)))
                         query = agent.suffixQuery(query)
 
@@ -631,7 +633,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
                 if showEta:
                     progress.progress(index)
-                elif conf.verbose in (1, 2) or conf.api:
+                elif (conf.verbose in (1, 2) and not kb.bruteMode) or conf.api:
                     dataToStdout(filterControlChars(val))
 
                 # some DBMSes (e.g. Firebird, DB2, etc.) have issues with trailing spaces
@@ -660,11 +662,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         elif partialValue:
             hashDBWrite(expression, "%s%s" % (PARTIAL_VALUE_MARKER if not conf.hexConvert else PARTIAL_HEX_VALUE_MARKER, partialValue))
 
-    if conf.hexConvert and not abortedFlag and not conf.api:
+    if conf.hexConvert and not any((abortedFlag, conf.api, kb.bruteMode)):
         infoMsg = "\r[%s] [INFO] retrieved: %s  %s\n" % (time.strftime("%X"), filterControlChars(finalValue), " " * retrievedLength)
         dataToStdout(infoMsg)
     else:
-        if conf.verbose in (1, 2) and not showEta and not conf.api:
+        if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
             dataToStdout("\n")
 
         if (conf.verbose in (1, 2) and showEta) or conf.verbose >= 3:

lib/techniques/error/use.py

@@ -22,6 +22,7 @@ from lib.core.common import firstNotNone
 from lib.core.common import getConsoleWidth
 from lib.core.common import getPartRun
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import incrementCounter
@@ -75,7 +76,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
 
     threadData.resumed = retVal is not None and not partialValue
 
-    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.ORACLE)) and kb.errorChunkLength is None and not chunkTest and not kb.testMode:
+    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)) and kb.errorChunkLength is None and not chunkTest and not kb.testMode:
         debugMsg = "searching for error chunk length..."
         logger.debug(debugMsg)
 
@@ -116,7 +117,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                 if field:
                     nulledCastedField = agent.nullAndCastField(field)
 
-                    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.ORACLE)) and not any(_ in field for _ in ("COUNT", "CASE")) and kb.errorChunkLength and not chunkTest:
+                    if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)) and not any(_ in field for _ in ("COUNT", "CASE")) and kb.errorChunkLength and not chunkTest:
                         extendedField = re.search(r"[^ ,]*%s[^ ,]*" % re.escape(field), expression).group(0)
                         if extendedField != field:  # e.g. MIN(surname)
                             nulledCastedField = extendedField.replace(field, nulledCastedField)
@@ -124,7 +125,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                         nulledCastedField = queries[Backend.getIdentifiedDbms()].substring.query % (nulledCastedField, offset, kb.errorChunkLength)
 
                 # Forge the error-based SQL injection request
-                vector = kb.injection.data[getTechnique()].vector
+                vector = getTechniqueData().vector
                 query = agent.prefixQuery(vector)
                 query = agent.suffixQuery(query)
                 injExpression = expression.replace(field, nulledCastedField, 1) if field else expression
@@ -176,7 +177,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                             else:
                                 output = output.rstrip()
 
-                if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.ORACLE)):
+                if any(Backend.isDbms(dbms) for dbms in (DBMS.MYSQL, DBMS.MSSQL, DBMS.SYBASE, DBMS.ORACLE)):
                     if offset == 1:
                         retVal = output
                     else:
@@ -187,7 +188,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                     else:
                         break
 
-                    if output and conf.verbose in (1, 2) and not conf.api:
+                    if output and conf.verbose in (1, 2) and not any((conf.api, kb.bruteMode)):
                         if kb.fileReadMode:
                             dataToStdout(_formatPartialContent(output).replace(r"\n", "\n").replace(r"\t", "\t"))
                         elif offset > 1:
@@ -366,7 +367,7 @@ def errorUse(expression, dump=False):
                     message = "due to huge table size do you want to remove "
                     message += "ORDER BY clause gaining speed over consistency? [y/N] "
 
-                    if readInput(message, default="N", boolean=True):
+                    if readInput(message, default='N', boolean=True):
                         expression = expression[:expression.index(" ORDER BY ")]
 
                 numThreads = min(conf.threads, (stopLimit - startLimit))

lib/techniques/union/test.py

@@ -163,7 +163,7 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
 
     if retVal:
         infoMsg = "target URL appears to be UNION injectable with %d columns" % retVal
-        singleTimeLogMessage(infoMsg, logging.INFO, re.sub(r"\d+", "N", infoMsg))
+        singleTimeLogMessage(infoMsg, logging.INFO, re.sub(r"\d+", 'N', infoMsg))
 
     return retVal
 
@@ -201,7 +201,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
             if content and phrase in content:
                 validPayload = payload
                 kb.unionDuplicates = len(re.findall(phrase, content, re.I)) > 1
-                vector = (position, count, comment, prefix, suffix, kb.uChar, where, kb.unionDuplicates, False)
+                vector = (position, count, comment, prefix, suffix, kb.uChar, where, kb.unionDuplicates, conf.forcePartial)
 
                 if where == PAYLOAD.WHERE.ORIGINAL:
                     # Prepare expression with delimiters
@@ -290,7 +290,7 @@ def _unionTestByCharBruteforce(comment, place, parameter, value, prefix, suffix)
             if not conf.uChar and count > 1 and kb.uChar == NULL:
                 message = "injection not exploitable with NULL values. Do you want to try with a random integer value for option '--union-char'? [Y/n] "
 
-                if not readInput(message, default="Y", boolean=True):
+                if not readInput(message, default='Y', boolean=True):
                     warnMsg += "usage of option '--union-char' "
                     warnMsg += "(e.g. '--union-char=1') "
                 else:

lib/techniques/union/use.py

@@ -241,7 +241,7 @@ def unionUse(expression, unpack=True, dump=False):
     # SQL limiting the query output one entry at a time
     # NOTE: we assume that only queries that get data from a table can
     # return multiple entries
-    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or kb.forcePartialUnion or (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or kb.forcePartialUnion or conf.forcePartial or (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
         expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)
 
         if limitCond:

lib/utils/brute.py

@@ -60,6 +60,7 @@ def _addPageTextWords():
 
     return wordsList
 
+@stackedmethod
 def tableExists(tableFile, regex=None):
     if kb.tableExistsChoice is None and not any(_ for _ in kb.injection.data if _ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) and not conf.direct:
         warnMsg = "it's not recommended to use '%s' and/or '%s' " % (PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.TIME], PAYLOAD.SQLINJECTION[PAYLOAD.TECHNIQUE.STACKED])
@@ -74,15 +75,17 @@ def tableExists(tableFile, regex=None):
 
     result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), randomStr())))
 
-    if conf.db and Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2):
-        conf.db = conf.db.upper()
-
     if result:
         errMsg = "can't use table existence check because of detected invalid results "
         errMsg += "(most likely caused by inability of the used injection "
         errMsg += "to distinguish erroneous results)"
         raise SqlmapDataException(errMsg)
 
+    pushValue(conf.db)
+
+    if conf.db and Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2):
+        conf.db = conf.db.upper()
+
     message = "which common tables (wordlist) file do you want to use?\n"
     message += "[1] default '%s' (press Enter)\n" % tableFile
     message += "[2] custom"
@@ -92,80 +95,88 @@ def tableExists(tableFile, regex=None):
         message = "what's the custom common tables file location?\n"
         tableFile = readInput(message) or tableFile
 
-    infoMsg = "checking table existence using items from '%s'" % tableFile
+    infoMsg = "performing table existence using items from '%s'" % tableFile
     logger.info(infoMsg)
 
     tables = getFileItems(tableFile, lowercase=Backend.getIdentifiedDbms() in (DBMS.ACCESS,), unique=True)
     tables.extend(_addPageTextWords())
     tables = filterListValue(tables, regex)
 
-    threadData = getCurrentThreadData()
-    threadData.shared.count = 0
-    threadData.shared.limit = len(tables)
-    threadData.shared.value = []
-    threadData.shared.unique = set()
+    for conf.db in (conf.db.split(',') if conf.db else [conf.db]):
+        if conf.db:
+            infoMsg = "checking database '%s'" % conf.db
+            logger.info(infoMsg)
 
-    def tableExistsThread():
         threadData = getCurrentThreadData()
+        threadData.shared.count = 0
+        threadData.shared.limit = len(tables)
+        threadData.shared.files = []
+        threadData.shared.unique = set()
 
-        while kb.threadContinue:
-            kb.locks.count.acquire()
-            if threadData.shared.count < threadData.shared.limit:
-                table = safeSQLIdentificatorNaming(tables[threadData.shared.count], True)
-                threadData.shared.count += 1
-                kb.locks.count.release()
-            else:
-                kb.locks.count.release()
-                break
+        def tableExistsThread():
+            threadData = getCurrentThreadData()
 
-            if conf.db and METADB_SUFFIX not in conf.db and Backend.getIdentifiedDbms() not in (DBMS.SQLITE, DBMS.ACCESS, DBMS.FIREBIRD):
-                fullTableName = "%s.%s" % (conf.db, table)
-            else:
-                fullTableName = table
+            while kb.threadContinue:
+                kb.locks.count.acquire()
+                if threadData.shared.count < threadData.shared.limit:
+                    table = safeSQLIdentificatorNaming(tables[threadData.shared.count], True)
+                    threadData.shared.count += 1
+                    kb.locks.count.release()
+                else:
+                    kb.locks.count.release()
+                    break
 
-            result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), fullTableName)))
+                if conf.db and METADB_SUFFIX not in conf.db and Backend.getIdentifiedDbms() not in (DBMS.SQLITE, DBMS.ACCESS, DBMS.FIREBIRD):
+                    fullTableName = "%s.%s" % (conf.db, table)
+                else:
+                    fullTableName = table
 
-            kb.locks.io.acquire()
+                result = inject.checkBooleanExpression("%s" % safeStringFormat(BRUTE_TABLE_EXISTS_TEMPLATE, (randomInt(1), fullTableName)))
 
-            if result and table.lower() not in threadData.shared.unique:
-                threadData.shared.value.append(table)
-                threadData.shared.unique.add(table.lower())
+                kb.locks.io.acquire()
 
-                if conf.verbose in (1, 2) and not conf.api:
-                    clearConsoleLine(True)
-                    infoMsg = "[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), unsafeSQLIdentificatorNaming(table))
-                    dataToStdout(infoMsg, True)
+                if result and table.lower() not in threadData.shared.unique:
+                    threadData.shared.files.append(table)
+                    threadData.shared.unique.add(table.lower())
 
-            if conf.verbose in (1, 2):
-                status = '%d/%d items (%d%%)' % (threadData.shared.count, threadData.shared.limit, round(100.0 * threadData.shared.count / threadData.shared.limit))
-                dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)
+                    if conf.verbose in (1, 2) and not conf.api:
+                        clearConsoleLine(True)
+                        infoMsg = "[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), unsafeSQLIdentificatorNaming(table))
+                        dataToStdout(infoMsg, True)
 
-            kb.locks.io.release()
+                if conf.verbose in (1, 2):
+                    status = '%d/%d items (%d%%)' % (threadData.shared.count, threadData.shared.limit, round(100.0 * threadData.shared.count / threadData.shared.limit))
+                    dataToStdout("\r[%s] [INFO] tried %s" % (time.strftime("%X"), status), True)
 
-    try:
-        runThreads(conf.threads, tableExistsThread, threadChoice=True)
-    except KeyboardInterrupt:
-        warnMsg = "user aborted during table existence "
-        warnMsg += "check. sqlmap will display partial output"
-        logger.warn(warnMsg)
+                kb.locks.io.release()
 
-    clearConsoleLine(True)
-    dataToStdout("\n")
+        try:
+            runThreads(conf.threads, tableExistsThread, threadChoice=True)
+        except KeyboardInterrupt:
+            warnMsg = "user aborted during table existence "
+            warnMsg += "check. sqlmap will display partial output"
+            logger.warn(warnMsg)
 
-    if not threadData.shared.value:
-        warnMsg = "no table(s) found"
-        logger.warn(warnMsg)
-    else:
-        for item in threadData.shared.value:
-            if conf.db not in kb.data.cachedTables:
-                kb.data.cachedTables[conf.db] = [item]
-            else:
-                kb.data.cachedTables[conf.db].append(item)
+        clearConsoleLine(True)
+        dataToStdout("\n")
 
-    for _ in ((conf.db, item) for item in threadData.shared.value):
-        if _ not in kb.brute.tables:
-            kb.brute.tables.append(_)
+        if not threadData.shared.files:
+            warnMsg = "no table(s) found"
+            if conf.db:
+                 warnMsg += "for database '%s'" % conf.db
+            logger.warn(warnMsg)
+        else:
+            for item in threadData.shared.files:
+                if conf.db not in kb.data.cachedTables:
+                    kb.data.cachedTables[conf.db] = [item]
+                else:
+                    kb.data.cachedTables[conf.db].append(item)
 
+        for _ in ((conf.db, item) for item in threadData.shared.files):
+            if _ not in kb.brute.tables:
+                kb.brute.tables.append(_)
+
+    conf.db = popValue()
     hashDBWrite(HASHDB_KEYS.KB_BRUTE_TABLES, kb.brute.tables, True)
 
     return kb.data.cachedTables
@@ -224,7 +235,7 @@ def columnExists(columnFile, regex=None):
     threadData = getCurrentThreadData()
     threadData.shared.count = 0
     threadData.shared.limit = len(columns)
-    threadData.shared.value = []
+    threadData.shared.files = []
 
     def columnExistsThread():
         threadData = getCurrentThreadData()
@@ -244,7 +255,7 @@ def columnExists(columnFile, regex=None):
             kb.locks.io.acquire()
 
             if result:
-                threadData.shared.value.append(column)
+                threadData.shared.files.append(column)
 
                 if conf.verbose in (1, 2) and not conf.api:
                     clearConsoleLine(True)
@@ -269,13 +280,13 @@ def columnExists(columnFile, regex=None):
     clearConsoleLine(True)
     dataToStdout("\n")
 
-    if not threadData.shared.value:
+    if not threadData.shared.files:
         warnMsg = "no column(s) found"
         logger.warn(warnMsg)
     else:
         columns = {}
 
-        for column in threadData.shared.value:
+        for column in threadData.shared.files:
             if Backend.getIdentifiedDbms() in (DBMS.MYSQL,):
                 result = not inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s WHERE %s REGEXP '[^0-9]')", (column, table, column)))
             else:
@@ -327,7 +338,7 @@ def fileExists(pathFile):
     threadData = getCurrentThreadData()
     threadData.shared.count = 0
     threadData.shared.limit = len(paths)
-    threadData.shared.value = []
+    threadData.shared.files = []
 
     def fileExistsThread():
         threadData = getCurrentThreadData()
@@ -350,9 +361,9 @@ def fileExists(pathFile):
             kb.locks.io.acquire()
 
             if not isNoneValue(result):
-                threadData.shared.value.append(result)
+                threadData.shared.files.append(result)
 
-                if conf.verbose in (1, 2) and not conf.api:
+                if not conf.api:
                     clearConsoleLine(True)
                     infoMsg = "[%s] [INFO] retrieved: '%s'\n" % (time.strftime("%X"), path)
                     dataToStdout(infoMsg, True)
@@ -379,10 +390,10 @@ def fileExists(pathFile):
     clearConsoleLine(True)
     dataToStdout("\n")
 
-    if not threadData.shared.value:
+    if not threadData.shared.files:
         warnMsg = "no file(s) found"
         logger.warn(warnMsg)
     else:
-        retVal = threadData.shared.value
+        retVal = threadData.shared.files
 
     return retVal

lib/utils/crawler.py

@@ -92,8 +92,7 @@ def crawl(target):
                         soup = BeautifulSoup(content)
                         tags = soup('a')
 
-                        if not tags:
-                            tags = re.finditer(r'(?i)<a[^>]+href=["\'](?P<href>[^>"\']+)', content)
+                        tags += re.finditer(r'(?i)<a[^>]+href=["\'](?P<href>[^>"\']+)', content)
 
                         for tag in tags:
                             href = tag.get("href") if hasattr(tag, "get") else tag.group("href")

lib/utils/hash.py

@@ -33,6 +33,7 @@ else:
 import base64
 import binascii
 import gc
+import hashlib
 import os
 import re
 import tempfile
@@ -155,7 +156,24 @@ def postgres_passwd(password, username, uppercase=False):
 
     return retVal.upper() if uppercase else retVal.lower()
 
-def mssql_passwd(password, salt, uppercase=False):
+def mssql_new_passwd(password, salt, uppercase=False):  # since version '2012'
+    """
+    Reference(s):
+        http://hashcat.net/forum/thread-1474.html
+        https://sqlity.net/en/2460/sql-password-hash/
+
+    >>> mssql_new_passwd(password='testpass', salt='4086ceb6', uppercase=False)
+    '0x02004086ceb6eb051cdbc5bdae68ffc66c918d4977e592f6bdfc2b444a7214f71fa31c35902c5b7ae773ed5f4c50676d329120ace32ee6bc81c24f70711eb0fc6400e85ebf25'
+    """
+
+    binsalt = decodeHex(salt)
+    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
+
+    retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest())
+
+    return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
+
+def mssql_passwd(password, salt, uppercase=False):  # versions '2005' and '2008'
     """
     Reference(s):
         http://www.leidecker.info/projects/phrasendrescher/mssql.c
@@ -172,7 +190,7 @@ def mssql_passwd(password, salt, uppercase=False):
 
     return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
 
-def mssql_old_passwd(password, salt, uppercase=True):  # prior to version '2005'
+def mssql_old_passwd(password, salt, uppercase=True):  # version '2000' and before
     """
     Reference(s):
         www.exploit-db.com/download_pdf/15537/
@@ -190,22 +208,6 @@ def mssql_old_passwd(password, salt, uppercase=True):  # prior to version '2005'
 
     return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
 
-def mssql_new_passwd(password, salt, uppercase=False):
-    """
-    Reference(s):
-        http://hashcat.net/forum/thread-1474.html
-
-    >>> mssql_new_passwd(password='testpass', salt='4086ceb6', uppercase=False)
-    '0x02004086ceb6eb051cdbc5bdae68ffc66c918d4977e592f6bdfc2b444a7214f71fa31c35902c5b7ae773ed5f4c50676d329120ace32ee6bc81c24f70711eb0fc6400e85ebf25'
-    """
-
-    binsalt = decodeHex(salt)
-    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
-
-    retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest())
-
-    return "0x%s" % (retVal.upper() if uppercase else retVal.lower())
-
 def oracle_passwd(password, salt, uppercase=True):
     """
     Reference(s):
@@ -246,6 +248,18 @@ def oracle_old_passwd(password, username, uppercase=True):  # prior to version '
 
     return retVal.upper() if uppercase else retVal.lower()
 
+def md4_generic_passwd(password, uppercase=False):
+    """
+    >>> md4_generic_passwd(password='testpass', uppercase=False)
+    '5b4d300688f19c8fd65b8d6ccf98e0ae'
+    """
+
+    password = getBytes(password)
+
+    retVal = hashlib.new("md4", password).hexdigest()
+
+    return retVal.upper() if uppercase else retVal.lower()
+
 def md5_generic_passwd(password, uppercase=False):
     """
     >>> md5_generic_passwd(password='testpass', uppercase=False)
@@ -708,6 +722,7 @@ def attackDumpedTable():
                 if hash_:
                     key = hash_ if hash_ not in replacements else replacements[hash_]
                     lut[key.lower()] = password
+                    lut["0x%s" % key.lower()] = password
 
             debugMsg = "post-processing table dump"
             logger.debug(debugMsg)
@@ -942,6 +957,8 @@ def dictionaryAttack(attack_dict):
                         if hash_regex in (HASH.MD5_BASE64, HASH.SHA1_BASE64, HASH.SHA256_BASE64, HASH.SHA512_BASE64):
                             item = [(user, encodeHex(decodeBase64(hash_, binary=True))), {}]
                         elif hash_regex in (HASH.MYSQL, HASH.MYSQL_OLD, HASH.MD5_GENERIC, HASH.SHA1_GENERIC, HASH.SHA224_GENERIC, HASH.SHA256_GENERIC, HASH.SHA384_GENERIC, HASH.SHA512_GENERIC, HASH.APACHE_SHA1):
+                            if hash_.startswith("0x"):  # Reference: https://docs.microsoft.com/en-us/sql/t-sql/functions/hashbytes-transact-sql?view=sql-server-2017
+                                hash_ = hash_[2:]
                             item = [(user, hash_), {}]
                         elif hash_regex in (HASH.SSHA,):
                             item = [(user, hash_), {"salt": decodeBase64(hash_, binary=True)[20:]}]

lib/utils/hashdb.py

@@ -141,6 +141,8 @@ class HashDB(object):
                             self.cursor.execute("INSERT INTO storage VALUES (?, ?)", (hash_, value,))
                         except sqlite3.IntegrityError:
                             self.cursor.execute("UPDATE storage SET value=? WHERE id=?", (value, hash_,))
+                    except UnicodeError:  # e.g. surrogates not allowed (Issue #3851)
+                        break
                     except sqlite3.DatabaseError as ex:
                         if not os.path.exists(self.filepath):
                             debugMsg = "session file '%s' does not exist" % self.filepath

lib/utils/htmlentities.py

@@ -1,263 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
-See the file 'LICENSE' for copying permission
-"""
-
-# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html
-
-htmlEntities = {
-    "quot": 34,
-    "amp": 38,
-    "lt": 60,
-    "gt": 62,
-    "nbsp": 160,
-    "iexcl": 161,
-    "cent": 162,
-    "pound": 163,
-    "curren": 164,
-    "yen": 165,
-    "brvbar": 166,
-    "sect": 167,
-    "uml": 168,
-    "copy": 169,
-    "ordf": 170,
-    "laquo": 171,
-    "not": 172,
-    "shy": 173,
-    "reg": 174,
-    "macr": 175,
-    "deg": 176,
-    "plusmn": 177,
-    "sup2": 178,
-    "sup3": 179,
-    "acute": 180,
-    "micro": 181,
-    "para": 182,
-    "middot": 183,
-    "cedil": 184,
-    "sup1": 185,
-    "ordm": 186,
-    "raquo": 187,
-    "frac14": 188,
-    "frac12": 189,
-    "frac34": 190,
-    "iquest": 191,
-    "Agrave": 192,
-    "Aacute": 193,
-    "Acirc": 194,
-    "Atilde": 195,
-    "Auml": 196,
-    "Aring": 197,
-    "AElig": 198,
-    "Ccedil": 199,
-    "Egrave": 200,
-    "Eacute": 201,
-    "Ecirc": 202,
-    "Euml": 203,
-    "Igrave": 204,
-    "Iacute": 205,
-    "Icirc": 206,
-    "Iuml": 207,
-    "ETH": 208,
-    "Ntilde": 209,
-    "Ograve": 210,
-    "Oacute": 211,
-    "Ocirc": 212,
-    "Otilde": 213,
-    "Ouml": 214,
-    "times": 215,
-    "Oslash": 216,
-    "Ugrave": 217,
-    "Uacute": 218,
-    "Ucirc": 219,
-    "Uuml": 220,
-    "Yacute": 221,
-    "THORN": 222,
-    "szlig": 223,
-    "agrave": 224,
-    "aacute": 225,
-    "acirc": 226,
-    "atilde": 227,
-    "auml": 228,
-    "aring": 229,
-    "aelig": 230,
-    "ccedil": 231,
-    "egrave": 232,
-    "eacute": 233,
-    "ecirc": 234,
-    "euml": 235,
-    "igrave": 236,
-    "iacute": 237,
-    "icirc": 238,
-    "iuml": 239,
-    "eth": 240,
-    "ntilde": 241,
-    "ograve": 242,
-    "oacute": 243,
-    "ocirc": 244,
-    "otilde": 245,
-    "ouml": 246,
-    "divide": 247,
-    "oslash": 248,
-    "ugrave": 249,
-    "uacute": 250,
-    "ucirc": 251,
-    "uuml": 252,
-    "yacute": 253,
-    "thorn": 254,
-    "yuml": 255,
-    "OElig": 338,
-    "oelig": 339,
-    "Scaron": 352,
-    "fnof": 402,
-    "scaron": 353,
-    "Yuml": 376,
-    "circ": 710,
-    "tilde": 732,
-    "Alpha": 913,
-    "Beta": 914,
-    "Gamma": 915,
-    "Delta": 916,
-    "Epsilon": 917,
-    "Zeta": 918,
-    "Eta": 919,
-    "Theta": 920,
-    "Iota": 921,
-    "Kappa": 922,
-    "Lambda": 923,
-    "Mu": 924,
-    "Nu": 925,
-    "Xi": 926,
-    "Omicron": 927,
-    "Pi": 928,
-    "Rho": 929,
-    "Sigma": 931,
-    "Tau": 932,
-    "Upsilon": 933,
-    "Phi": 934,
-    "Chi": 935,
-    "Psi": 936,
-    "Omega": 937,
-    "alpha": 945,
-    "beta": 946,
-    "gamma": 947,
-    "delta": 948,
-    "epsilon": 949,
-    "zeta": 950,
-    "eta": 951,
-    "theta": 952,
-    "iota": 953,
-    "kappa": 954,
-    "lambda": 955,
-    "mu": 956,
-    "nu": 957,
-    "xi": 958,
-    "omicron": 959,
-    "pi": 960,
-    "rho": 961,
-    "sigmaf": 962,
-    "sigma": 963,
-    "tau": 964,
-    "upsilon": 965,
-    "phi": 966,
-    "chi": 967,
-    "psi": 968,
-    "omega": 969,
-    "thetasym": 977,
-    "upsih": 978,
-    "piv": 982,
-    "bull": 8226,
-    "hellip": 8230,
-    "prime": 8242,
-    "Prime": 8243,
-    "oline": 8254,
-    "frasl": 8260,
-    "ensp": 8194,
-    "emsp": 8195,
-    "thinsp": 8201,
-    "zwnj": 8204,
-    "zwj": 8205,
-    "lrm": 8206,
-    "rlm": 8207,
-    "ndash": 8211,
-    "mdash": 8212,
-    "lsquo": 8216,
-    "rsquo": 8217,
-    "sbquo": 8218,
-    "ldquo": 8220,
-    "rdquo": 8221,
-    "bdquo": 8222,
-    "dagger": 8224,
-    "Dagger": 8225,
-    "permil": 8240,
-    "lsaquo": 8249,
-    "rsaquo": 8250,
-    "euro": 8364,
-    "weierp": 8472,
-    "image": 8465,
-    "real": 8476,
-    "trade": 8482,
-    "alefsym": 8501,
-    "larr": 8592,
-    "uarr": 8593,
-    "rarr": 8594,
-    "darr": 8595,
-    "harr": 8596,
-    "crarr": 8629,
-    "lArr": 8656,
-    "uArr": 8657,
-    "rArr": 8658,
-    "dArr": 8659,
-    "hArr": 8660,
-    "forall": 8704,
-    "part": 8706,
-    "exist": 8707,
-    "empty": 8709,
-    "nabla": 8711,
-    "isin": 8712,
-    "notin": 8713,
-    "ni": 8715,
-    "prod": 8719,
-    "sum": 8721,
-    "minus": 8722,
-    "lowast": 8727,
-    "radic": 8730,
-    "prop": 8733,
-    "infin": 8734,
-    "ang": 8736,
-    "and": 8743,
-    "or": 8744,
-    "cap": 8745,
-    "cup": 8746,
-    "int": 8747,
-    "there4": 8756,
-    "sim": 8764,
-    "cong": 8773,
-    "asymp": 8776,
-    "ne": 8800,
-    "equiv": 8801,
-    "le": 8804,
-    "ge": 8805,
-    "sub": 8834,
-    "sup": 8835,
-    "nsub": 8836,
-    "sube": 8838,
-    "supe": 8839,
-    "oplus": 8853,
-    "otimes": 8855,
-    "perp": 8869,
-    "sdot": 8901,
-    "lceil": 8968,
-    "rceil": 8969,
-    "lfloor": 8970,
-    "rfloor": 8971,
-    "lang": 9001,
-    "rang": 9002,
-    "loz": 9674,
-    "spades": 9824,
-    "clubs": 9827,
-    "hearts": 9829,
-    "diams": 9830,
-}

lib/utils/httpd.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+See the file 'LICENSE' for copying permission
+"""
+
+from __future__ import print_function
+
+import mimetypes
+import gzip
+import os
+import re
+import sys
+import threading
+import time
+import traceback
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from lib.core.enums import HTTP_HEADER
+from lib.core.settings import UNICODE_ENCODING
+from lib.core.settings import VERSION_STRING
+from thirdparty import six
+from thirdparty.six.moves import BaseHTTPServer as _BaseHTTPServer
+from thirdparty.six.moves import http_client as _http_client
+from thirdparty.six.moves import socketserver as _socketserver
+from thirdparty.six.moves import urllib as _urllib
+
+HTTP_ADDRESS = "0.0.0.0"
+HTTP_PORT = 8951
+DEBUG = True
+HTML_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "data", "html"))
+DISABLED_CONTENT_EXTENSIONS = (".py", ".pyc", ".md", ".txt", ".bak", ".conf", ".zip", "~")
+
+class ThreadingServer(_socketserver.ThreadingMixIn, _BaseHTTPServer.HTTPServer):
+    def finish_request(self, *args, **kwargs):
+        try:
+            _BaseHTTPServer.HTTPServer.finish_request(self, *args, **kwargs)
+        except Exception:
+            if DEBUG:
+                traceback.print_exc()
+
+class ReqHandler(_BaseHTTPServer.BaseHTTPRequestHandler):
+    def do_GET(self):
+        path, query = self.path.split('?', 1) if '?' in self.path else (self.path, "")
+        params = {}
+        content = None
+
+        if query:
+            params.update(_urllib.parse.parse_qs(query))
+
+        for key in params:
+            if params[key]:
+                params[key] = params[key][-1]
+
+        self.url, self.params = path, params
+
+        if path == '/':
+            path = "index.html"
+
+        path = path.strip('/')
+
+        path = path.replace('/', os.path.sep)
+        path = os.path.abspath(os.path.join(HTML_DIR, path)).strip()
+
+        if not os.path.isfile(path) and os.path.isfile("%s.html" % path):
+            path = "%s.html" % path
+
+        if ".." not in os.path.relpath(path, HTML_DIR) and os.path.isfile(path) and not path.endswith(DISABLED_CONTENT_EXTENSIONS):
+            content = open(path, "rb").read()
+            self.send_response(_http_client.OK)
+            self.send_header(HTTP_HEADER.CONNECTION, "close")
+            self.send_header(HTTP_HEADER.CONTENT_TYPE, mimetypes.guess_type(path)[0] or "application/octet-stream")
+        else:
+            content = ("<!DOCTYPE html><html lang=\"en\"><head><title>404 Not Found</title></head><body><h1>Not Found</h1><p>The requested URL %s was not found on this server.</p></body></html>" % self.path.split('?')[0]).encode(UNICODE_ENCODING)
+            self.send_response(_http_client.NOT_FOUND)
+            self.send_header(HTTP_HEADER.CONNECTION, "close")
+
+        if content is not None:
+            for match in re.finditer(b"<!(\\w+)!>", content):
+                name = match.group(1)
+                _ = getattr(self, "_%s" % name.lower(), None)
+                if _:
+                    content = self._format(content, **{name: _()})
+
+            if "gzip" in self.headers.get(HTTP_HEADER.ACCEPT_ENCODING):
+                self.send_header(HTTP_HEADER.CONTENT_ENCODING, "gzip")
+                _ = six.BytesIO()
+                compress = gzip.GzipFile("", "w+b", 9, _)
+                compress._stream = _
+                compress.write(content)
+                compress.flush()
+                compress.close()
+                content = compress._stream.getvalue()
+
+            self.send_header(HTTP_HEADER.CONTENT_LENGTH, str(len(content)))
+
+        self.end_headers()
+
+        if content:
+            self.wfile.write(content)
+
+        self.wfile.flush()
+
+    def _format(self, content, **params):
+        if content:
+            for key, value in params.items():
+                content = content.replace("<!%s!>" % key, value)
+
+        return content
+
+    def version_string(self):
+        return VERSION_STRING
+
+    def log_message(self, format, *args):
+        return
+
+    def finish(self):
+        try:
+            _BaseHTTPServer.BaseHTTPRequestHandler.finish(self)
+        except Exception:
+            if DEBUG:
+                traceback.print_exc()
+
+def start_httpd():
+    server = ThreadingServer((HTTP_ADDRESS, HTTP_PORT), ReqHandler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.daemon = True
+    thread.start()
+
+    print("[i] running HTTP server at '%s:%d'" % (HTTP_ADDRESS, HTTP_PORT))
+
+if __name__ == "__main__":
+    try:
+        start_httpd()
+
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        pass

plugins/dbms/firebird/fingerprint.py

@@ -10,6 +10,7 @@ import re
 from lib.core.common import Backend
 from lib.core.common import Format
 from lib.core.common import randomRange
+from lib.core.common import randomStr
 from lib.core.compat import xrange
 from lib.core.convert import getUnicode
 from lib.core.data import conf
@@ -74,13 +75,14 @@ class Fingerprint(GenericFingerprint):
             ("1.5", ("NULLIF(%d,%d) IS NULL", "EXISTS(SELECT CURRENT_TRANSACTION FROM RDB$DATABASE)")),
             ("2.0", ("EXISTS(SELECT CURRENT_TIME(0) FROM RDB$DATABASE)", "BIT_LENGTH(%d)>0", "CHAR_LENGTH(%d)>0")),
             ("2.1", ("BIN_XOR(%d,%d)=0", "PI()>0.%d", "RAND()<1.%d", "FLOOR(1.%d)>=0")),
-            # TODO: add test for Firebird 2.5
+            ("2.5", ("'%s' SIMILAR TO '%s'",)),  # Reference: https://firebirdsql.org/refdocs/langrefupd25-similar-to.html
+            ("3.0", ("FALSE IS FALSE",)),  # https://www.firebirdsql.org/file/community/conference-2014/pdf/02_fb.2014.whatsnew.30.en.pdf
         )
 
         for i in xrange(len(table)):
             version, checks = table[i]
             failed = False
-            check = checks[randomRange(0, len(checks) - 1)].replace("%d", getUnicode(randomRange(1, 100)))
+            check = checks[randomRange(0, len(checks) - 1)].replace("%d", getUnicode(randomRange(1, 100))).replace("%s", getUnicode(randomStr()))
             result = inject.checkBooleanExpression(check)
 
             if result:

plugins/dbms/mssqlserver/filesystem.py

@@ -52,7 +52,7 @@ class Filesystem(GenericFilesystem):
                 else:
                     scrString += " %s" % strLineChar
 
-                lineAddr += len(lineChar)
+                lineAddr += len(strLineChar) // 2
 
             fileLines.append(scrString)
 

plugins/dbms/oracle/enumeration.py

@@ -21,6 +21,7 @@ from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapNoneDataException
+from lib.core.settings import CURRENT_USER
 from lib.request import inject
 from plugins.generic.enumeration import Enumeration as GenericEnumeration
 
@@ -30,7 +31,7 @@ class Enumeration(GenericEnumeration):
 
         rootQuery = queries[DBMS.ORACLE].roles
 
-        if conf.user == "CU":
+        if conf.user == CURRENT_USER:
             infoMsg += " for current user"
             conf.user = self.getCurrentUser()
 
@@ -55,7 +56,7 @@ class Enumeration(GenericEnumeration):
             values = inject.getValue(query, blind=False, time=False)
 
             if not values and not query2:
-                infoMsg = "trying with table USER_ROLE_PRIVS"
+                infoMsg = "trying with table 'USER_ROLE_PRIVS'"
                 logger.info(infoMsg)
 
                 return self.getRoles(query2=True)
@@ -116,7 +117,7 @@ class Enumeration(GenericEnumeration):
 
                 if not isNumPosStrValue(count):
                     if count != 0 and not query2:
-                        infoMsg = "trying with table USER_SYS_PRIVS"
+                        infoMsg = "trying with table 'USER_SYS_PRIVS'"
                         logger.info(infoMsg)
 
                         return self.getPrivileges(query2=True)

plugins/generic/filesystem.py

@@ -21,6 +21,8 @@ from lib.core.common import isStackingAvailable
 from lib.core.common import isTechniqueAvailable
 from lib.core.common import readInput
 from lib.core.compat import xrange
+from lib.core.convert import encodeBase64
+from lib.core.convert import encodeHex
 from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.data import conf
@@ -31,7 +33,6 @@ from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapUndefinedMethod
-from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.core.settings import UNICODE_ENCODING
 from lib.request import inject
 
@@ -41,7 +42,7 @@ class Filesystem(object):
     """
 
     def __init__(self):
-        self.fileTblName = "%sfile" % TAKEOVER_TABLE_PREFIX
+        self.fileTblName = "%sfile" % conf.tablePrefix
         self.tblField = "data"
 
     def _checkFileLength(self, localFile, remoteFile, fileRead=False):
@@ -134,8 +135,14 @@ class Filesystem(object):
     def fileContentEncode(self, content, encoding, single, chunkSize=256):
         retVal = []
 
-        if encoding:
-            content = getText(codecs.encode(content, encoding)).replace("\n", "")
+        if encoding == "hex":
+            content = encodeHex(content)
+        elif encoding == "base64":
+            content = encodeBase64(content)
+        else:
+            content = codecs.encode(content, encoding)
+
+        content = getText(content).replace("\n", "")
 
         if not single:
             if len(content) > chunkSize:

plugins/generic/takeover.py

@@ -10,6 +10,7 @@ import os
 from lib.core.common import Backend
 from lib.core.common import getSafeExString
 from lib.core.common import isStackingAvailable
+from lib.core.common import openFile
 from lib.core.common import readInput
 from lib.core.common import runningAsAdmin
 from lib.core.data import conf
@@ -24,7 +25,6 @@ from lib.core.exception import SqlmapNotVulnerableException
 from lib.core.exception import SqlmapSystemException
 from lib.core.exception import SqlmapUndefinedMethod
 from lib.core.exception import SqlmapUnsupportedDBMSException
-from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.takeover.abstraction import Abstraction
 from lib.takeover.icmpsh import ICMPsh
 from lib.takeover.metasploit import Metasploit
@@ -36,7 +36,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
     """
 
     def __init__(self):
-        self.cmdTblName = ("%soutput" % TAKEOVER_TABLE_PREFIX)
+        self.cmdTblName = ("%soutput" % conf.tablePrefix)
         self.tblField = "data"
 
         Abstraction.__init__(self)
@@ -137,7 +137,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
 
             if os.path.exists(filename):
                 try:
-                    with open(filename, "wb") as f:
+                    with openFile(filename, "wb") as f:
                         f.write("1")
                 except IOError as ex:
                     errMsg = "there has been a file opening/writing error "

plugins/generic/users.py

@@ -39,6 +39,7 @@ from lib.core.enums import EXPECTED
 from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.settings import CURRENT_USER
 from lib.core.threads import getCurrentThreadData
 from lib.request import inject
 from lib.utils.hash import attackCachedUsersPasswords
@@ -153,7 +154,7 @@ class Users(object):
 
         rootQuery = queries[Backend.getIdentifiedDbms()].passwords
 
-        if conf.user == "CU":
+        if conf.user == CURRENT_USER:
             infoMsg += " for current user"
             conf.user = self.getCurrentUser()
 
@@ -362,7 +363,7 @@ class Users(object):
 
         rootQuery = queries[Backend.getIdentifiedDbms()].privileges
 
-        if conf.user == "CU":
+        if conf.user == CURRENT_USER:
             infoMsg += " for current user"
             conf.user = self.getCurrentUser()
 
@@ -410,7 +411,7 @@ class Users(object):
             values = inject.getValue(query, blind=False, time=False)
 
             if not values and Backend.isDbms(DBMS.ORACLE) and not query2:
-                infoMsg = "trying with table USER_SYS_PRIVS"
+                infoMsg = "trying with table 'USER_SYS_PRIVS'"
                 logger.info(infoMsg)
 
                 return self.getPrivileges(query2=True)
@@ -446,7 +447,7 @@ class Users(object):
                             # In MySQL < 5.0 we get Y if the privilege is
                             # True, N otherwise
                             elif Backend.isDbms(DBMS.MYSQL) and not kb.data.has_information_schema:
-                                if privilege.upper() == "Y":
+                                if privilege.upper() == 'Y':
                                     privileges.add(MYSQL_PRIVS[count])
 
                             # In Firebird we get one letter for each privilege
@@ -465,7 +466,7 @@ class Users(object):
                                     i = 1
 
                                     for priv in privs:
-                                        if priv.upper() in ("Y", "G"):
+                                        if priv.upper() in ('Y', 'G'):
                                             for position, db2Priv in DB2_PRIVS.items():
                                                 if position == i:
                                                     privilege += ", " + db2Priv
@@ -525,7 +526,7 @@ class Users(object):
 
                     if not isNumPosStrValue(count):
                         if not retrievedUsers and Backend.isDbms(DBMS.ORACLE) and not query2:
-                            infoMsg = "trying with table USER_SYS_PRIVS"
+                            infoMsg = "trying with table 'USER_SYS_PRIVS'"
                             logger.info(infoMsg)
 
                             return self.getPrivileges(query2=True)

sqlmap.conf

@@ -67,6 +67,10 @@ dropSetCookie = False
 # sqlmap will also test for SQL injection on the HTTP User-Agent value.
 agent =
 
+# Imitate smartphone through HTTP User-Agent header.
+# Valid: True or False
+mobile = False
+
 # Use randomly selected HTTP User-Agent header value.
 # Valid: True or False
 randomAgent = False
@@ -343,6 +347,10 @@ regexp =
 # code)
 # code = 
 
+# Conduct thorough tests only if positive heuristic(s).
+# Valid: True or False
+smart = False
+
 # Compare pages based only on the textual content.
 # Valid: True or False
 textOnly = False
@@ -683,6 +691,9 @@ sessionFile =
 # Log all HTTP traffic into a textual file.
 trafficFile = 
 
+# Set predefined answers (e.g. "quit=N,follow=N").
+answers =
+
 # Never ask for user input, use the default behaviour.
 # Valid: True or False
 batch = False
@@ -693,6 +704,10 @@ binaryFields =
 # Check Internet connection before assessing the target.
 checkInternet = False
 
+# Clean up the DBMS from sqlmap specific UDF and tables.
+# Valid: True or False
+cleanup = False
+
 # Crawl the website starting from the target URL.
 # Valid: integer
 # Default: 0
@@ -729,6 +744,11 @@ forms = False
 # Valid: True or False
 freshQueries = False
 
+# Use Google dork results from specified page number.
+# Valid: integer
+# Default: 1
+googlePage = 1
+
 # Use hex conversion during data retrieval.
 # Valid: True or False
 hexConvert = False
@@ -752,15 +772,22 @@ repair = False
 # Example: (google|yahoo)
 scope = 
 
+# Skip heuristic detection of WAF/IPS protection.
+# Valid: True or False
+skipWaf = False
+
+# Prefix used for temporary tables.
+# Default: sqlmap
+tablePrefix = sqlmap
+
 # Select tests by payloads and/or titles (e.g. ROW)
 testFilter =
 
 # Skip tests by payloads and/or titles (e.g. BENCHMARK)
 testSkip =
 
-# Update sqlmap.
-# Valid: True or False
-updateAll = False
+# Web server document root directory (e.g. "/var/www").
+webRoot =
 
 
 [Miscellaneous]
@@ -768,9 +795,6 @@ updateAll = False
 # Run host OS command(s) when SQL injection is found.
 alert =
 
-# Set predefined answers (e.g. "quit=N,follow=N").
-answers =
-
 # Beep on question and/or when SQL injection is found.
 # Valid: True or False
 beep = False
@@ -779,10 +803,6 @@ beep = False
 # Valid: True or False
 checkPayload = False
 
-# Clean up the DBMS from sqlmap specific UDF and tables.
-# Valid: True or False
-cleanup = False
-
 # Check for missing (optional) sqlmap dependencies.
 # Valid: True or False
 dependencies = False
@@ -791,41 +811,25 @@ dependencies = False
 # Valid: True or False
 disableColoring = False
 
-# Use Google dork results from specified page number.
-# Valid: integer
-# Default: 1
-googlePage = 1
-
 # Display list of available tamper scripts
 # Valid: True or False
 listTampers = False
 
-# Imitate smartphone through HTTP User-Agent header.
-# Valid: True or False
-mobile = False
-
 # Work in offline mode (only use session data)
 # Valid: True or False
 offline = False
 
-# Skip heuristic detection of WAF/IPS protection.
-# Valid: True or False
-skipWaf = False
-
-# Conduct thorough tests only if positive heuristic(s).
-# Valid: True or False
-smart = False
-
 # Local directory for storing temporary files.
 tmpDir =
 
-# Web server document root directory (e.g. "/var/www").
-webRoot =
-
 # Simple wizard interface for beginner users.
 # Valid: True or False
 wizard = False
 
+# Update sqlmap.
+# Valid: True or False
+updateAll = False
+
 # Verbosity level.
 # Valid: integer between 0 and 6
 # 0: Show only error and critical messages

sqlmap.py

@@ -238,7 +238,7 @@ def main():
             dataToStdout(excMsg)
             raise SystemExit
 
-        elif any(_ in excMsg for _ in ("ImportError", "Can't find file for module")):
+        elif any(_ in excMsg for _ in ("ImportError", "ModuleNotFoundError", "Can't find file for module")):
             errMsg = "invalid runtime environment ('%s')" % excMsg.split("Error: ")[-1].strip()
             logger.critical(errMsg)
             raise SystemExit
@@ -330,6 +330,12 @@ def main():
             logger.critical(errMsg)
             raise SystemExit
 
+        elif "can't allocate read lock" in excMsg:
+            errMsg = "there has been a problem in regular socket operation "
+            errMsg += "('%s')" % excMsg.strip().split('\n')[-1]
+            logger.critical(errMsg)
+            raise SystemExit
+
         elif "'DictObject' object has no attribute '" in excMsg and all(_ in errMsg for _ in ("(fingerprinted)", "(identified)")):
             errMsg = "there has been a problem in enumeration. "
             errMsg += "Because of a considerable chance of false-positive case "

tamper/plus2concat.py

@@ -10,7 +10,6 @@ import re
 
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import zeroDepthSearch
-from lib.core.compat import xrange
 from lib.core.enums import DBMS
 from lib.core.enums import PRIORITY
 
@@ -35,51 +34,22 @@ def tamper(payload, **kwargs):
     >>> tamper('SELECT CHAR(113)+CHAR(114)+CHAR(115) FROM DUAL')
     'SELECT CONCAT(CHAR(113),CHAR(114),CHAR(115)) FROM DUAL'
 
-    >>> tamper('SELECT (CHAR(113)+CHAR(114)+CHAR(115)) FROM DUAL')
-    'SELECT CONCAT(CHAR(113),CHAR(114),CHAR(115)) FROM DUAL'
+    >>> tamper('1 UNION ALL SELECT NULL,NULL,CHAR(113)+CHAR(118)+CHAR(112)+CHAR(112)+CHAR(113)+ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32))+CHAR(113)+CHAR(112)+CHAR(107)+CHAR(112)+CHAR(113)-- qtfe')
+    '1 UNION ALL SELECT NULL,NULL,CONCAT(CHAR(113),CHAR(118),CHAR(112),CHAR(112),CHAR(113),ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32)),CHAR(113),CHAR(112),CHAR(107),CHAR(112),CHAR(113))-- qtfe'
     """
 
     retVal = payload
 
     if payload:
-        prefix, suffix = '+' * len(re.search(r"\A(\+*)", payload).group(0)), '+' * len(re.search(r"(\+*)\Z", payload).group(0))
-        retVal = retVal.strip('+')
+        match = re.search(r"('[^']+'|CHAR\(\d+\))\+.*(?<=\+)('[^']+'|CHAR\(\d+\))", retVal)
+        if match:
+            part = match.group(0)
 
-        while True:
-            indexes = zeroDepthSearch(retVal, '+')
+            chars = [char for char in part]
+            for index in zeroDepthSearch(part, '+'):
+                chars[index] = ','
 
-            if indexes:
-                first, last = 0, 0
-                for i in xrange(1, len(indexes)):
-                    if ' ' in retVal[indexes[0]:indexes[i]]:
-                        break
-                    else:
-                        last = i
-
-                start = retVal[:indexes[first]].rfind(' ') + 1
-                end = (retVal[indexes[last] + 1:].find(' ') + indexes[last] + 1) if ' ' in retVal[indexes[last] + 1:] else len(retVal) - 1
-
-                chars = [char for char in retVal]
-                for index in indexes[first:last + 1]:
-                    chars[index] = ','
-
-                retVal = "%sCONCAT(%s)%s" % (retVal[:start], ''.join(chars)[start:end], retVal[end:])
-            else:
-                match = re.search(r"\((CHAR\(\d+.+\bCHAR\(\d+\))\)", retVal)
-                if match:
-                    part = match.group(0)
-                    indexes = set(zeroDepthSearch(match.group(1), '+'))
-                    if not indexes:
-                        break
-                    chars = [char for char in part]
-                    for i in xrange(1, len(chars)):
-                        if i - 1 in indexes:
-                            chars[i] = ','
-                    replacement = "CONCAT%s" % "".join(chars)
-                    retVal = retVal.replace(part, replacement)
-                else:
-                    break
-
-        retVal = "%s%s%s" % (prefix, retVal, suffix)
+            replacement = "CONCAT(%s)" % "".join(chars)
+            retVal = retVal.replace(part, replacement)
 
     return retVal

tamper/plus2fnconcat.py

@@ -36,63 +36,29 @@ def tamper(payload, **kwargs):
     >>> tamper('SELECT CHAR(113)+CHAR(114)+CHAR(115) FROM DUAL')
     'SELECT {fn CONCAT({fn CONCAT(CHAR(113),CHAR(114))},CHAR(115))} FROM DUAL'
 
-    >>> tamper('SELECT (CHAR(113)+CHAR(114)+CHAR(115)) FROM DUAL')
-    'SELECT {fn CONCAT({fn CONCAT(CHAR(113),CHAR(114))},CHAR(115))} FROM DUAL'
+    >>> tamper('1 UNION ALL SELECT NULL,NULL,CHAR(113)+CHAR(118)+CHAR(112)+CHAR(112)+CHAR(113)+ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32))+CHAR(113)+CHAR(112)+CHAR(107)+CHAR(112)+CHAR(113)-- qtfe')
+    '1 UNION ALL SELECT NULL,NULL,{fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT(CHAR(113),CHAR(118))},CHAR(112))},CHAR(112))},CHAR(113))},ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32)))},CHAR(113))},CHAR(112))},CHAR(107))},CHAR(112))},CHAR(113))}-- qtfe'
     """
 
     retVal = payload
 
     if payload:
-        prefix, suffix = '+' * len(re.search(r"\A(\+*)", payload).group(0)), '+' * len(re.search(r"(\+*)\Z", payload).group(0))
-        retVal = retVal.strip('+')
+        match = re.search(r"('[^']+'|CHAR\(\d+\))\+.*(?<=\+)('[^']+'|CHAR\(\d+\))", retVal)
+        if match:
+            old = match.group(0)
+            parts = []
+            last = 0
 
-        while True:
-            indexes = zeroDepthSearch(retVal, '+')
+            for index in zeroDepthSearch(old, '+'):
+                parts.append(old[last:index].strip('+'))
+                last = index
 
-            if indexes:
-                first, last = 0, 0
-                for i in xrange(1, len(indexes)):
-                    if ' ' in retVal[indexes[0]:indexes[i]]:
-                        break
-                    else:
-                        last = i
+            parts.append(old[last:].strip('+'))
+            replacement = parts[0]
 
-                start = retVal[:indexes[first]].rfind(' ') + 1
-                end = (retVal[indexes[last] + 1:].find(' ') + indexes[last] + 1) if ' ' in retVal[indexes[last] + 1:] else len(retVal) - 1
+            for i in xrange(1, len(parts)):
+                replacement = "{fn CONCAT(%s,%s)}" % (replacement, parts[i])
 
-                count = 0
-                chars = [char for char in retVal]
-                for index in indexes[first:last + 1]:
-                    if count == 0:
-                        chars[index] = ','
-                    else:
-                        chars[index] = '\x01'
-                    count += 1
-
-                retVal = "%s%s%s)}%s" % (retVal[:start], "{fn CONCAT(" * count, ''.join(chars)[start:end].replace('\x01', ")},"), retVal[end:])
-            else:
-                match = re.search(r"\((CHAR\(\d+.+\bCHAR\(\d+\))\)", retVal)
-                if match:
-                    part = match.group(0)
-                    indexes = set(zeroDepthSearch(match.group(1), '+'))
-                    if not indexes:
-                        break
-
-                    count = 0
-                    chars = [char for char in part]
-                    for i in xrange(1, len(chars)):
-                        if i - 1 in indexes:
-                            if count == 0:
-                                chars[i] = ','
-                            else:
-                                chars[i] = '\x01'
-                            count += 1
-
-                    replacement = "%s%s}" % (("{fn CONCAT(" * count)[:-1], "".join(chars).replace('\x01', ")},"))
-                    retVal = retVal.replace(part, replacement)
-                else:
-                    break
-
-        retVal = "%s%s%s" % (prefix, retVal, suffix)
+            retVal = retVal.replace(old, replacement)
 
     return retVal

tamper/varnish.py

@@ -17,7 +17,7 @@ def tamper(payload, **kwargs):
     Appends a HTTP header 'X-originating-IP' to bypass Varnish Firewall
 
     Reference:
-        * http://h30499.www3.hp.com/t5/Fortify-Application-Security/Bypassing-web-application-firewalls-using-HTTP-headers/ba-p/6418366
+        * https://web.archive.org/web/20160815052159/http://community.hpe.com/t5/Protect-Your-Assets/Bypassing-web-application-firewalls-using-HTTP-headers/ba-p/6418366
 
     Notes:
         Examples:

thirdparty/colorama/ansitowin32.py

@@ -184,6 +184,8 @@ class AnsiToWin32(object):
             if not (err.errno == 0 and retry > 0):
                 raise
             self._write(text, retry-1)
+        except UnicodeError:
+            self.wrapped.write('?')
 
     def convert_ansi(self, paramstring, command):
         if self.convert:

thirdparty/identywaf/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2019 Miroslav Stampar (@stamparm), MIT
+# See the file 'LICENSE' for copying permission
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+pass

thirdparty/identywaf/data.json

@@ -55,7 +55,8 @@
             "name": "360",
             "regex": "<title>493</title>|/wzws-waf-cgi/",
             "signatures": [
-                "9778:RVZXum61OEhCWapBYKcPk4JzWOpohM4JiUcMr2RXg1uQJbX3uhdOnthtOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A5chW1XaTC"
+                "9778:RVZXum61OEhCWapBYKcPk4JzWOpohM4JiUcMr2RXg1uQJbX3uhdOnthtOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A5chW1XaTC",
+                "9ccc:RVZXum61OEhCWapBYKcPk4JzWOpohM4JiUcMr2RXg1uQJbX3uhdOnthtOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A4chW1XaTC"
             ]
         },
         "aesecure": {
@@ -677,6 +678,12 @@
             "signatures": [],
             "note": "Uses Incapsula (Reference: https://www.whitefirdesign.com/blog/2016/11/08/more-evidence-that-sitelocks-trueshield-web-application-firewall-is-really-incapsulas-waf/)"
         },
+        "sniper": {
+            "company": "Wins",
+            "name": "Sniper",
+            "regex": "document\\.title = [^;]+Sniper WAF",
+            "signatures": []
+        },
         "sonicwall": {
             "company": "Dell",
             "name": "SonicWALL",
@@ -777,6 +784,14 @@
                 "c02b:RVZXu261OElCWapBYKcPk4JzWOpohM4JiUcMr2RWg1uQJbX3uhdOnthsOj+hXrAB16FcPxJOdLsXo2tKaK99n+i7c4RmkgI2FZjxtDtAeq+c36A4chW1XaTC"
             ]
         },
+        "wapples": {
+            "company": "Penta Security",
+            "name": "Wapples",
+            "regex": "",
+            "signatures": [
+                "60b7:RVZXum60OEhCWKpAYKYPkoJyWOpohM4IiUYMrmRWg1uQJLX2uhZOnthtOj+hXrAA16FcPhJOdLoXomtKaK59nui7c4RmkgI2FZjxtDtAeq+c3qA4chW1XKTC"
+            ]
+        },
         "watchguard": {
             "company": "WatchGuard Technologies",
             "name": "WatchGuard",
@@ -809,6 +824,14 @@
                 "17a8:RVZXum60OEhCWKpAYKYPkoJyWOpohM4JiUcMrmRXg1qQJbX3uhdOnthtOj+hXrAB16FcPhJPdLoXomtKaK59nui6c4RmkgI2FZjxtDtAeq+c3qA4chS1XKTC"
             ]
         },
+        "webland": {
+            "company": "WebLand",
+            "name": "WebLand",
+            "regex": "Server: Apache Protected by Webland WAF",
+            "signatures": [
+                "4ba0:RVZXum60OEhCWKpAYKYPkoJzWOpohc4IiUYMr2RWg1uQJLX3uhZOnthsOj6hXrAA16BcPhJOdLoXomtKaK59nui6c4RmkgI2FZjxtDtAeq6c3qA4chS1XKTC"
+            ]
+        },
         "webseal": {
             "company": "IBM",
             "name": "WebSEAL",
@@ -824,7 +847,7 @@
             "signatures": []
         },
         "wordfence": {
-            "company": "Feedjit",
+            "company": "Defiant",
             "name": "Wordfence",
             "regex": "Generated by Wordfence|This response was generated by Wordfence|broke one of the Wordfence (advanced )?blocking rules|: wfWAF|/plugins/wordfence",
             "signatures": [