Minor patch

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+dist: trusty
 sudo: false
 git:
     depth: 1
@@ -10,4 +11,4 @@ python:
 script:
     - python -c "import sqlmap; import sqlmapapi"
     - python sqlmap.py --smoke
-    - python sqlmap.py --vuln
+    - python sqlmap.py --vuln

data/html/index.html

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+
+<!-- http://angrytools.com/bootstrap/editor/ -->
+
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap.min.css" rel="stylesheet">
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/css/bootstrap-theme.min.css" rel="stylesheet">
+    
+    <!--[if lt IE 9]><script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script><script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script><![endif]-->
+</head>
+<body>
+    <style>
+  #wrapper { width: 100%; }
+
+  #page-wrapper {
+    padding: 0 15px;
+    min-height: 568px;
+    background-color: #fff;
+  }
+
+  @media(min-width:768px) {
+    #page-wrapper {
+      position: inherit;
+      margin: 0 0 0 250px;
+      padding: 0 30px;
+      border-left: 1px solid #e7e7e7;
+    }
+  } 
+
+  .sidebar .sidebar-nav.navbar-collapse { padding-right: 0; padding-left: 0; }
+  .sidebar .sidebar-search { padding: 15px; }
+  .sidebar ul li { border-bottom: 1px solid #e7e7e7; }
+
+  .sidebar ul li a.active { background-color: #eee; }
+
+  .sidebar .arrow { float: right;}
+  .sidebar .fa.arrow:before { content: "f104";}
+  .sidebar .active>a>.fa.arrow:before { content: "f107"; }
+  .sidebar .nav-second-level li,
+  .sidebar .nav-third-level li {
+    border-bottom: 0!important;
+  }
+
+  .sidebar .nav-second-level li a { padding-left: 37px; }
+  .sidebar .nav-third-level li a { padding-left: 52px; }
+
+  @media(min-width:768px) {
+    .sidebar {
+      z-index: 1;
+      position: absolute;
+      width: 250px;
+      margin-top: 51px;
+    }   
+  } 
+</style>
+<div id="wrapper">
+
+  <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+        <span class="sr-only">Toggle navigation</span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+        <span class="icon-bar"></span>
+      </button>
+      <a class="navbar-brand" href="index.html">sqlmap</a>
+    </div>
+
+    <div class="navbar-default sidebar" role="navigation">
+      <div class="sidebar-nav navbar-collapse">
+        <ul class="nav" id="side-menu">                        
+          <li>
+            <a href="#"><i class="glyphicon glyphicon-home"></i> Options<span class="arrow"></span></a>
+            <ul class="nav nav-second-level">
+              <li><a>Target</a></li>
+              <li><a>Request</a></li>
+              <li><a>Optimization</a></li>
+              <li><a>Injection</a></li>
+              <li><a>Detection</a></li>
+              <li><a>Techniques</a></li>
+              <li><a>Fingerprint</a></li>
+              <li><a>Enumeration</a></li>
+              <li><a>Brute force</a></li>
+              <li><a>User-defined function injection</a></li>
+              <li><a>File system access</a></li>
+              <li><a>Operating system access</a></li>
+              <li><a>Windows registry access</a></li>
+              <li><a>General</a></li>
+              <li><a>Miscellaneous</a></li>
+            </ul> 
+          </li> 
+        </ul>
+      </div>
+    </div>             
+  </nav>
+
+  <div id="page-wrapper"> 
+    <div class="row">
+      <h4>DEMO</h4>
+    </div> 
+  </div>
+</div>
+<script>
+  /*
+ * metismenu - v1.0.3
+ * Easy menu jQuery plugin for Twitter Bootstrap 3
+ * https://github.com/onokumus/metisMenu
+ *
+ * Made by Osman Nuri Okumuş
+ * Under MIT License
+*/
+  !function(a,b,c){function d(b,c){this.element=b,this.settings=a.extend({},f,c),this._defaults=f,this._name=e,this.init()}var e="metisMenu",f={toggle:!0};d.prototype={init:function(){var b=a(this.element),c=this.settings.toggle;this.isIE()<=9?(b.find("li.active").has("ul").children("ul").collapse("show"),b.find("li").not(".active").has("ul").children("ul").collapse("hide")):(b.find("li.active").has("ul").children("ul").addClass("collapse in"),b.find("li").not(".active").has("ul").children("ul").addClass("collapse")),b.find("li").has("ul").children("a").on("click",function(b){b.preventDefault(),a(this).parent("li").toggleClass("active").children("ul").collapse("toggle"),c&&a(this).parent("li").siblings().removeClass("active").children("ul.in").collapse("hide")})},isIE:function(){for(var a,b=3,d=c.createElement("div"),e=d.getElementsByTagName("i");d.innerHTML="<!--[if gt IE "+ ++b+"]><i></i><![endif]-->",e[0];)return b>4?b:a}},a.fn[e]=function(b){return this.each(function(){a.data(this,"plugin_"+e)||a.data(this,"plugin_"+e,new d(this,b))})}}(jQuery,window,document);
+
+  $(function() {
+
+    $('#side-menu').metisMenu();
+
+  });
+
+  //Loads the correct sidebar on window load,
+  //collapses the sidebar on window resize.
+  // Sets the min-height of #page-wrapper to window size
+  $(function() {
+    $(window).bind("load resize", function() {
+      topOffset = 50;
+      width = (this.window.innerWidth > 0) ? this.window.innerWidth : this.screen.width;
+      if (width < 768) {
+        $('div.navbar-collapse').addClass('collapse')
+        topOffset = 100; // 2-row-menu
+      } else {
+        $('div.navbar-collapse').removeClass('collapse')
+      }
+
+      height = (this.window.innerHeight > 0) ? this.window.innerHeight : this.screen.height;
+      height = height - topOffset;
+      if (height < 1) height = 1;
+      if (height > topOffset) {
+        $("#page-wrapper").css("min-height", (height) + "px");
+      }
+    })
+  });
+</script>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js"></script>
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.0/js/bootstrap.min.js"></script>
+</body>
+</html>

lib/controller/checks.py

@@ -1518,7 +1518,7 @@ def checkConnection(suppressOutput=False):
             warnMsg += "which could interfere with the results of the tests"
             logger.warn(warnMsg)
         elif wasLastResponseHTTPError():
-            if getLastRequestHTTPError() != conf.ignoreCode:
+            if getLastRequestHTTPError() not in (conf.ignoreCode or []):
                 warnMsg = "the web server responded with an HTTP error code (%d) " % getLastRequestHTTPError()
                 warnMsg += "which could interfere with the results of the tests"
                 logger.warn(warnMsg)

lib/core/agent.py

@@ -13,6 +13,7 @@ from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
 from lib.core.common import getSQLSnippet
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import isDBMSVersionAtLeast
 from lib.core.common import isNumber
 from lib.core.common import isTechniqueAvailable
@@ -91,7 +92,7 @@ class Agent(object):
         if kb.forceWhere:
             where = kb.forceWhere
         elif where is None and isTechniqueAvailable(getTechnique()):
-            where = kb.injection.data[getTechnique()].where
+            where = getTechniqueData().where
 
         if kb.injection.place is not None:
             place = kb.injection.place
@@ -236,7 +237,7 @@ class Agent(object):
         query = None
 
         if where is None and getTechnique() is not None and getTechnique() in kb.injection.data:
-            where = kb.injection.data[getTechnique()].where
+            where = getTechniqueData().where
 
         # If we are replacing (<where>) the parameter original value with
         # our payload do not prepend with the prefix
@@ -284,8 +285,8 @@ class Agent(object):
         suffix = kb.injection.suffix if kb.injection and suffix is None else suffix
 
         if getTechnique() is not None and getTechnique() in kb.injection.data:
-            where = kb.injection.data[getTechnique()].where if where is None else where
-            comment = kb.injection.data[getTechnique()].comment if comment is None else comment
+            where = getTechniqueData().where if where is None else where
+            comment = getTechniqueData().comment if comment is None else comment
 
         if Backend.getIdentifiedDbms() == DBMS.ACCESS and any((comment or "").startswith(_) for _ in ("--", "[GENERIC_SQL_COMMENT]")):
             comment = queries[DBMS.ACCESS].comment.query

lib/core/bigarray.py

@@ -51,6 +51,11 @@ class Cache(object):
 class BigArray(list):
     """
     List-like class used for storing large amounts of data (disk cached)
+
+    >>> _ = BigArray(xrange(100000))
+    >>> _[20] = 0
+    >>> _[100]
+    100
     """
 
     def __init__(self, items=None):

lib/core/common.py

@@ -1137,7 +1137,7 @@ def getTechnique():
     Thread-safe getting of currently used technique
     """
 
-    return getCurrentThreadData().technique or kb.technique
+    return getCurrentThreadData().technique or kb.get("technique")
 
 def randomRange(start=0, stop=1000, seed=None):
     """
@@ -3222,7 +3222,7 @@ def getTechniqueData(technique=None):
     Returns injection data for technique specified
     """
 
-    return kb.injection.data.get(technique)
+    return kb.injection.data.get(technique if technique is not None else getTechnique())
 
 def isTechniqueAvailable(technique):
     """
@@ -3549,7 +3549,7 @@ def openFile(filename, mode='r', encoding=UNICODE_ENCODING, errors="reversible",
         except IOError:
             errMsg = "there has been a file opening error for filename '%s'. " % filename
             errMsg += "Please check %s permissions on a file " % ("write" if mode and ('w' in mode or 'a' in mode or '+' in mode) else "read")
-            errMsg += "and that it's not locked by another process."
+            errMsg += "and that it's not locked by another process"
             raise SqlmapSystemException(errMsg)
 
 def decodeIntToUnicode(value):
@@ -3762,7 +3762,7 @@ def createGithubIssue(errMsg, excMsg):
             logger.info(infoMsg)
 
             try:
-                with open(paths.GITHUB_HISTORY, "a+b") as f:
+                with openFile(paths.GITHUB_HISTORY, "a+b") as f:
                     f.write("%s\n" % key)
             except:
                 pass
@@ -4868,6 +4868,8 @@ def zeroDepthSearch(expression, value):
 
     >>> _ = "SELECT (SELECT id FROM users WHERE 2>1) AS result FROM DUAL"; _[zeroDepthSearch(_, "FROM")[0]:]
     'FROM DUAL'
+    >>> _ = "a(b; c),d;e"; _[zeroDepthSearch(_, "[;, ]")[0]:]
+    ',d;e'
     """
 
     retVal = []
@@ -4878,8 +4880,13 @@ def zeroDepthSearch(expression, value):
             depth += 1
         elif expression[index] == ')':
             depth -= 1
-        elif depth == 0 and expression[index:index + len(value)] == value:
-            retVal.append(index)
+        elif depth == 0:
+            found = False
+            if value.startswith('[') and value.endswith(']'):
+                if re.search(value, expression[index:index + 1]):
+                    retVal.append(index)
+            elif expression[index:index + len(value)] == value:
+                retVal.append(index)
 
     return retVal
 
@@ -4988,7 +4995,7 @@ def parseRequestFile(reqFile, checkParams=True):
             else:
                 scheme, port = None, None
 
-            if not re.search(r"^[\n]*(%s).*?\sHTTP\/" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), request, re.I | re.M):
+            if "HTTP/" not in request:
                 continue
 
             if re.search(r"^[\n]*%s.*?\.(%s)\sHTTP\/" % (HTTPMETHOD.GET, "|".join(CRAWL_EXCLUDE_EXTENSIONS)), request, re.I | re.M):
@@ -5013,7 +5020,7 @@ def parseRequestFile(reqFile, checkParams=True):
 
                 newline = "\r\n" if line.endswith('\r') else '\n'
                 line = line.strip('\r')
-                match = re.search(r"\A(%s) (.+) HTTP/[\d.]+\Z" % "|".join(getPublicTypeMembers(HTTPMETHOD, True)), line) if not method else None
+                match = re.search(r"\A([A-Z]+) (.+) HTTP/[\d.]+\Z", line) if not method else None
 
                 if len(line.strip()) == 0 and method and method != HTTPMETHOD.GET and data is None:
                     data = ""

lib/core/convert.py

@@ -164,8 +164,13 @@ def encodeHex(value, binary=True):
     True
     >>> encodeHex("123", binary=False)
     '313233'
+    >>> encodeHex(b"123"[0]) == b"31"
+    True
     """
 
+    if isinstance(value, int):
+        value = six.unichr(value)
+
     if isinstance(value, six.text_type):
         value = value.encode(UNICODE_ENCODING)
 

lib/core/decorators.py

@@ -14,9 +14,11 @@ from lib.core.settings import MAX_CACHE_ITEMS
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.threads import getCurrentThreadData
 
-_lock = threading.Lock()
+_cache = {}
+_cache_lock = threading.Lock()
+_method_locks = {}
 
-def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
+def cachedmethod(f):
     """
     Method with a cached content
 
@@ -33,22 +35,24 @@ def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
     Reference: http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/
     """
 
+    _cache[f] = LRUDict(capacity=MAX_CACHE_ITEMS)
+
     @functools.wraps(f)
-    def _(*args, **kwargs):
+    def _f(*args, **kwargs):
         key = int(hashlib.md5("|".join(str(_) for _ in (f, args, kwargs)).encode(UNICODE_ENCODING)).hexdigest(), 16) & 0x7fffffffffffffff
 
         try:
-            with _lock:
-                result = cache[key]
+            with _cache_lock:
+                result = _cache[f][key]
         except KeyError:
             result = f(*args, **kwargs)
 
-            with _lock:
-                cache[key] = result
+            with _cache_lock:
+                _cache[f][key] = result
 
         return result
 
-    return _
+    return _f
 
 def stackedmethod(f):
     """
@@ -76,3 +80,16 @@ def stackedmethod(f):
         return result
 
     return _
+
+def lockedmethod(f):
+    @functools.wraps(f)
+    def _(*args, **kwargs):
+        if f not in _method_locks:
+            _method_locks[f] = threading.Lock()
+
+        with _method_locks[f]:
+            result = f(*args, **kwargs)
+
+        return result
+
+    return _

lib/core/dicts.py

@@ -304,7 +304,7 @@ DUMP_DATA_PREPROCESS = {
 
 DEFAULT_DOC_ROOTS = {
     OS.WINDOWS: ("C:/xampp/htdocs/", "C:/wamp/www/", "C:/Inetpub/wwwroot/"),
-    OS.LINUX: ("/var/www/", "/var/www/html", "/usr/local/apache2/htdocs", "/var/www/nginx-default", "/srv/www")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
+    OS.LINUX: ("/var/www/", "/var/www/html", "/var/www/htdocs", "/usr/local/apache2/htdocs", "/usr/local/www/data", "/var/apache2/htdocs", "/var/www/nginx-default", "/srv/www/htdocs")  # Reference: https://wiki.apache.org/httpd/DistrosDefaultLayout
 }
 
 PART_RUN_CONTENT_TYPES = {
@@ -334,3 +334,260 @@ PART_RUN_CONTENT_TYPES = {
     "osCmd": CONTENT_TYPE.OS_CMD,
     "regRead": CONTENT_TYPE.REG_READ
 }
+
+# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html
+
+HTML_ENTITIES = {
+    "quot": 34,
+    "amp": 38,
+    "lt": 60,
+    "gt": 62,
+    "nbsp": 160,
+    "iexcl": 161,
+    "cent": 162,
+    "pound": 163,
+    "curren": 164,
+    "yen": 165,
+    "brvbar": 166,
+    "sect": 167,
+    "uml": 168,
+    "copy": 169,
+    "ordf": 170,
+    "laquo": 171,
+    "not": 172,
+    "shy": 173,
+    "reg": 174,
+    "macr": 175,
+    "deg": 176,
+    "plusmn": 177,
+    "sup2": 178,
+    "sup3": 179,
+    "acute": 180,
+    "micro": 181,
+    "para": 182,
+    "middot": 183,
+    "cedil": 184,
+    "sup1": 185,
+    "ordm": 186,
+    "raquo": 187,
+    "frac14": 188,
+    "frac12": 189,
+    "frac34": 190,
+    "iquest": 191,
+    "Agrave": 192,
+    "Aacute": 193,
+    "Acirc": 194,
+    "Atilde": 195,
+    "Auml": 196,
+    "Aring": 197,
+    "AElig": 198,
+    "Ccedil": 199,
+    "Egrave": 200,
+    "Eacute": 201,
+    "Ecirc": 202,
+    "Euml": 203,
+    "Igrave": 204,
+    "Iacute": 205,
+    "Icirc": 206,
+    "Iuml": 207,
+    "ETH": 208,
+    "Ntilde": 209,
+    "Ograve": 210,
+    "Oacute": 211,
+    "Ocirc": 212,
+    "Otilde": 213,
+    "Ouml": 214,
+    "times": 215,
+    "Oslash": 216,
+    "Ugrave": 217,
+    "Uacute": 218,
+    "Ucirc": 219,
+    "Uuml": 220,
+    "Yacute": 221,
+    "THORN": 222,
+    "szlig": 223,
+    "agrave": 224,
+    "aacute": 225,
+    "acirc": 226,
+    "atilde": 227,
+    "auml": 228,
+    "aring": 229,
+    "aelig": 230,
+    "ccedil": 231,
+    "egrave": 232,
+    "eacute": 233,
+    "ecirc": 234,
+    "euml": 235,
+    "igrave": 236,
+    "iacute": 237,
+    "icirc": 238,
+    "iuml": 239,
+    "eth": 240,
+    "ntilde": 241,
+    "ograve": 242,
+    "oacute": 243,
+    "ocirc": 244,
+    "otilde": 245,
+    "ouml": 246,
+    "divide": 247,
+    "oslash": 248,
+    "ugrave": 249,
+    "uacute": 250,
+    "ucirc": 251,
+    "uuml": 252,
+    "yacute": 253,
+    "thorn": 254,
+    "yuml": 255,
+    "OElig": 338,
+    "oelig": 339,
+    "Scaron": 352,
+    "fnof": 402,
+    "scaron": 353,
+    "Yuml": 376,
+    "circ": 710,
+    "tilde": 732,
+    "Alpha": 913,
+    "Beta": 914,
+    "Gamma": 915,
+    "Delta": 916,
+    "Epsilon": 917,
+    "Zeta": 918,
+    "Eta": 919,
+    "Theta": 920,
+    "Iota": 921,
+    "Kappa": 922,
+    "Lambda": 923,
+    "Mu": 924,
+    "Nu": 925,
+    "Xi": 926,
+    "Omicron": 927,
+    "Pi": 928,
+    "Rho": 929,
+    "Sigma": 931,
+    "Tau": 932,
+    "Upsilon": 933,
+    "Phi": 934,
+    "Chi": 935,
+    "Psi": 936,
+    "Omega": 937,
+    "alpha": 945,
+    "beta": 946,
+    "gamma": 947,
+    "delta": 948,
+    "epsilon": 949,
+    "zeta": 950,
+    "eta": 951,
+    "theta": 952,
+    "iota": 953,
+    "kappa": 954,
+    "lambda": 955,
+    "mu": 956,
+    "nu": 957,
+    "xi": 958,
+    "omicron": 959,
+    "pi": 960,
+    "rho": 961,
+    "sigmaf": 962,
+    "sigma": 963,
+    "tau": 964,
+    "upsilon": 965,
+    "phi": 966,
+    "chi": 967,
+    "psi": 968,
+    "omega": 969,
+    "thetasym": 977,
+    "upsih": 978,
+    "piv": 982,
+    "bull": 8226,
+    "hellip": 8230,
+    "prime": 8242,
+    "Prime": 8243,
+    "oline": 8254,
+    "frasl": 8260,
+    "ensp": 8194,
+    "emsp": 8195,
+    "thinsp": 8201,
+    "zwnj": 8204,
+    "zwj": 8205,
+    "lrm": 8206,
+    "rlm": 8207,
+    "ndash": 8211,
+    "mdash": 8212,
+    "lsquo": 8216,
+    "rsquo": 8217,
+    "sbquo": 8218,
+    "ldquo": 8220,
+    "rdquo": 8221,
+    "bdquo": 8222,
+    "dagger": 8224,
+    "Dagger": 8225,
+    "permil": 8240,
+    "lsaquo": 8249,
+    "rsaquo": 8250,
+    "euro": 8364,
+    "weierp": 8472,
+    "image": 8465,
+    "real": 8476,
+    "trade": 8482,
+    "alefsym": 8501,
+    "larr": 8592,
+    "uarr": 8593,
+    "rarr": 8594,
+    "darr": 8595,
+    "harr": 8596,
+    "crarr": 8629,
+    "lArr": 8656,
+    "uArr": 8657,
+    "rArr": 8658,
+    "dArr": 8659,
+    "hArr": 8660,
+    "forall": 8704,
+    "part": 8706,
+    "exist": 8707,
+    "empty": 8709,
+    "nabla": 8711,
+    "isin": 8712,
+    "notin": 8713,
+    "ni": 8715,
+    "prod": 8719,
+    "sum": 8721,
+    "minus": 8722,
+    "lowast": 8727,
+    "radic": 8730,
+    "prop": 8733,
+    "infin": 8734,
+    "ang": 8736,
+    "and": 8743,
+    "or": 8744,
+    "cap": 8745,
+    "cup": 8746,
+    "int": 8747,
+    "there4": 8756,
+    "sim": 8764,
+    "cong": 8773,
+    "asymp": 8776,
+    "ne": 8800,
+    "equiv": 8801,
+    "le": 8804,
+    "ge": 8805,
+    "sub": 8834,
+    "sup": 8835,
+    "nsub": 8836,
+    "sube": 8838,
+    "supe": 8839,
+    "oplus": 8853,
+    "otimes": 8855,
+    "perp": 8869,
+    "sdot": 8901,
+    "lceil": 8968,
+    "rceil": 8969,
+    "lfloor": 8970,
+    "rfloor": 8971,
+    "lang": 9001,
+    "rang": 9002,
+    "loz": 9674,
+    "spades": 9824,
+    "clubs": 9827,
+    "hearts": 9829,
+    "diams": 9830
+}

lib/core/dump.py

@@ -617,9 +617,10 @@ class Dump(object):
                                 warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath)
                                 logger.warn(warnMsg)
 
-                                with open(filepath, "wb") as f:
+                                with openFile(filepath, "w+b", None) as f:
                                     _ = safechardecode(value, True)
                                     f.write(_)
+
                         except magic.MagicException as ex:
                             logger.debug(getSafeExString(ex))
 

lib/core/option.py

@@ -293,6 +293,7 @@ def _setRequestFromFile():
     if conf.requestFile:
         for requestFile in re.split(PARAMETER_SPLITTING_REGEX, conf.requestFile):
             requestFile = safeExpandUser(requestFile)
+            url = None
             seen = set()
 
             if not checkFile(requestFile, False):
@@ -311,6 +312,11 @@ def _setRequestFromFile():
                         conf.multipleTargets = True
                     seen.add(url)
 
+            if url is None:
+                errMsg = "specified file '%s' " % requestFile
+                errMsg += "does not contain a valid HTTP request"
+                raise SqlmapDataException(errMsg)
+
     if conf.secondReq:
         conf.secondReq = safeExpandUser(conf.secondReq)
 
@@ -455,12 +461,14 @@ def _findPageForms():
     if conf.url and not checkConnection():
         return
 
+    found = False
     infoMsg = "searching for forms"
     logger.info(infoMsg)
 
     if not any((conf.bulkFile, conf.googleDork, conf.sitemapUrl)):
         page, _, _ = Request.queryPage(content=True)
-        findPageForms(page, conf.url, True, True)
+        if findPageForms(page, conf.url, True, True):
+            found = True
     else:
         if conf.bulkFile:
             targets = getFileItems(conf.bulkFile)
@@ -473,7 +481,8 @@ def _findPageForms():
             try:
                 target = targets[i]
                 page, _, _ = Request.getPage(url=target.strip(), cookie=conf.cookie, crawling=True, raise404=False)
-                findPageForms(page, target, False, True)
+                if findPageForms(page, target, False, True):
+                    found = True
 
                 if conf.verbose in (1, 2):
                     status = '%d/%d links visited (%d%%)' % (i + 1, len(targets), round(100.0 * (i + 1) / len(targets)))
@@ -484,6 +493,10 @@ def _findPageForms():
                 errMsg = "problem occurred while searching for forms at '%s' ('%s')" % (target, getSafeExString(ex))
                 logger.error(errMsg)
 
+    if not found:
+        warnMsg = "no forms found"
+        logger.warn(warnMsg)
+
 def _setDBMSAuthentication():
     """
     Check and set the DBMS authentication credentials to run statements as
@@ -522,26 +535,14 @@ def _setMetasploit():
             errMsg = "sqlmap requires third-party module 'pywin32' "
             errMsg += "in order to use Metasploit functionalities on "
             errMsg += "Windows. You can download it from "
-            errMsg += "'https://sourceforge.net/projects/pywin32/files/pywin32/'"
+            errMsg += "'https://github.com/mhammond/pywin32'"
             raise SqlmapMissingDependence(errMsg)
 
         if not conf.msfPath:
-            def _(key, value):
-                retVal = None
-
-                try:
-                    from six.moves.winreg import ConnectRegistry, OpenKey, QueryValueEx, HKEY_LOCAL_MACHINE
-                    _ = ConnectRegistry(None, HKEY_LOCAL_MACHINE)
-                    _ = OpenKey(_, key)
-                    retVal = QueryValueEx(_, value)[0]
-                except:
-                    logger.debug("unable to identify Metasploit installation path via registry key")
-
-                return retVal
-
-            conf.msfPath = _(r"SOFTWARE\Rapid7\Metasploit", "Location")
-            if conf.msfPath:
-                conf.msfPath = os.path.join(conf.msfPath, "msf3")
+            for candidate in os.environ.get("PATH", "").split(';'):
+                if all(_ in candidate for _ in ("metasploit", "bin")):
+                    conf.msfPath = os.path.dirname(candidate.rstrip('\\'))
+                    break
 
     if conf.osSmb:
         isAdmin = runningAsAdmin()
@@ -1453,6 +1454,9 @@ def _createHomeDirectories():
     Creates directories inside sqlmap's home directory
     """
 
+    if conf.get("purge"):
+        return
+
     for context in "output", "history":
         directory = paths["SQLMAP_%s_PATH" % context.upper()]
         try:
@@ -1463,7 +1467,7 @@ def _createHomeDirectories():
             open(_, "w+b").close()
             os.remove(_)
 
-            if conf.outputDir and context == "output":
+            if conf.get("outputDir") and context == "output":
                 warnMsg = "using '%s' as the %s directory" % (directory, context)
                 logger.warn(warnMsg)
         except (OSError, IOError) as ex:
@@ -1557,6 +1561,15 @@ def _cleanupOptions():
     else:
         conf.testParameter = []
 
+    if conf.ignoreCode:
+        try:
+            conf.ignoreCode = [int(_) for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.ignoreCode)]
+        except ValueError:
+            errMsg = "options '--ignore-code' should contain a list of integer values"
+            raise SqlmapSyntaxException(errMsg)
+    else:
+        conf.ignoreCode = []
+
     if conf.paramFilter:
         conf.paramFilter = [_.strip() for _ in re.split(PARAMETER_SPLITTING_REGEX, conf.paramFilter.upper())]
     else:
@@ -1576,8 +1589,19 @@ def _cleanupOptions():
         conf.user = conf.user.replace(" ", "")
 
     if conf.rParam:
-        conf.rParam = conf.rParam.replace(" ", "")
-        conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
+        if all(_ in conf.rParam for _ in ('=', ',')):
+            original = conf.rParam
+            conf.rParam = []
+            for part in original.split(';'):
+                if '=' in part:
+                    left, right = part.split('=', 1)
+                    conf.rParam.append(left)
+                    kb.randomPool[left] = filterNone(_.strip() for _ in right.split(','))
+                else:
+                    conf.rParam.append(part)
+        else:
+            conf.rParam = conf.rParam.replace(" ", "")
+            conf.rParam = re.split(PARAMETER_SPLITTING_REGEX, conf.rParam)
     else:
         conf.rParam = []
 
@@ -1939,6 +1963,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.processUserMarks = None
     kb.proxyAuthHeader = None
     kb.queryCounter = 0
+    kb.randomPool = {}
     kb.redirectChoice = None
     kb.reflectiveMechanism = True
     kb.reflectiveCounters = {REFLECTIVE_COUNTER.MISS: 0, REFLECTIVE_COUNTER.HIT: 0}

lib/core/optiondict.py

@@ -38,7 +38,7 @@ optDict = {
         "authType": "string",
         "authCred": "string",
         "authFile": "string",
-        "ignoreCode": "integer",
+        "ignoreCode": "string",
         "ignoreProxy": "boolean",
         "ignoreRedirects": "boolean",
         "ignoreTimeouts": "boolean",

lib/core/revision.py

@@ -38,7 +38,7 @@ def getRevisionNumber():
     while True:
         if filePath and os.path.isfile(filePath):
             with openFile(filePath, "r") as f:
-                content = f.read()
+                content = getText(f.read())
                 filePath = None
                 if content.startswith("ref: "):
                     filePath = os.path.join(_, ".git", content.replace("ref: ", "")).strip()

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3.7.0"
+VERSION = "1.3.8.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/target.py

@@ -111,7 +111,7 @@ def _setRequestParams():
         def process(match, repl):
             retVal = match.group(0)
 
-            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]):
+            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]) and match.group("name") == match.group("name").strip('\\'):
                 retVal = repl
                 while True:
                     _ = re.search(r"\\g<([^>]+)>", retVal)
@@ -121,6 +121,7 @@ def _setRequestParams():
                         break
                 if kb.customInjectionMark in retVal:
                     hintNames.append((retVal.split(kb.customInjectionMark)[0], match.group("name")))
+
             return retVal
 
         if kb.processUserMarks is None and kb.customInjectionMark in conf.data:

lib/core/threads.py

@@ -95,11 +95,13 @@ def exceptionHandledFunction(threadFunction, silent=False):
         kb.threadException = True
         raise
     except Exception as ex:
+        from lib.core.common import getSafeExString
+
         if not silent and kb.get("threadContinue"):
-            errMsg = ex.message if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, ex.message)
+            errMsg = getSafeExString(ex) if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, getSafeExString(ex))
             logger.error("thread %s: '%s'" % (threading.currentThread().getName(), errMsg))
 
-            if conf.get("verbose") > 1 and not isinstance(ex, (SqlmapUserQuitException,)):
+            if conf.get("verbose") > 1 and not isinstance(ex, (SqlmapUserQuitException, SqlmapConnectionException)):
                 traceback.print_exc()
 
 def setDaemon(thread):
@@ -193,7 +195,7 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
         kb.threadException = True
         logger.error("thread %s: '%s'" % (threading.currentThread().getName(), ex))
 
-        if conf.get("verbose") > 1:
+        if conf.get("verbose") > 1 and isinstance(ex, SqlmapValueException):
             traceback.print_exc()
 
     except:

lib/core/wordlist.py

@@ -18,7 +18,9 @@ class Wordlist(six.Iterator):
     Iterator for looping over a large dictionaries
 
     >>> from lib.core.option import paths
-    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.string_types)
+    >>> isinstance(next(Wordlist(paths.SMALL_DICT)), six.binary_type)
+    True
+    >>> isinstance(next(Wordlist(paths.WORDLIST)), six.binary_type)
     True
     """
 
@@ -58,7 +60,7 @@ class Wordlist(six.Iterator):
                     raise SqlmapDataException(errMsg)
                 self.fp = _.open(_.namelist()[0])
             else:
-                self.fp = open(self.current, 'r')
+                self.fp = open(self.current, "rb")
             self.iter = iter(self.fp)
 
         self.index += 1

lib/parse/cmdline.py

@@ -59,6 +59,7 @@ from lib.core.dicts import DEPRECATED_OPTIONS
 from lib.core.enums import AUTOCOMPLETE_TYPE
 from lib.core.exception import SqlmapShellQuitException
 from lib.core.exception import SqlmapSyntaxException
+from lib.core.option import _createHomeDirectories
 from lib.core.settings import BASIC_HELP_ITEMS
 from lib.core.settings import DUMMY_URL
 from lib.core.settings import INFERENCE_UNKNOWN_CHAR
@@ -88,691 +89,692 @@ def cmdLineParser(argv=None):
     parser = ArgumentParser(usage=usage)
 
     try:
-        parser.add_argument("--hh", dest="advancedHelp",
-                          action="store_true",
-                          help="Show advanced help message and exit")
+        parser.add_argument("--hh", dest="advancedHelp", action="store_true",
+            help="Show advanced help message and exit")
 
-        parser.add_argument("--version", dest="showVersion",
-                          action="store_true",
-                          help="Show program's version number and exit")
+        parser.add_argument("--version", dest="showVersion", action="store_true",
+            help="Show program's version number and exit")
 
         parser.add_argument("-v", dest="verbose", type=int,
-                          help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
+            help="Verbosity level: 0-6 (default %d)" % defaults.verbose)
 
         # Target options
-        target = parser.add_argument_group("Target", "At least one of these "
-                             "options has to be provided to define the target(s)")
+        target = parser.add_argument_group("Target", "At least one of these options has to be provided to define the target(s)")
 
-        target.add_argument("-d", dest="direct", help="Connection string "
-                          "for direct database connection")
+        target.add_argument("-d", dest="direct",
+            help="Connection string for direct database connection")
 
-        target.add_argument("-u", "--url", dest="url", help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
+        target.add_argument("-u", "--url", dest="url",
+            help="Target URL (e.g. \"http://www.site.com/vuln.php?id=1\")")
 
-        target.add_argument("-l", dest="logFile", help="Parse target(s) from Burp "
-                          "or WebScarab proxy log file")
+        target.add_argument("-l", dest="logFile",
+            help="Parse target(s) from Burp or WebScarab proxy log file")
 
-        target.add_argument("-x", dest="sitemapUrl", help="Parse target(s) from remote sitemap(.xml) file")
+        target.add_argument("-x", dest="sitemapUrl",
+            help="Parse target(s) from remote sitemap(.xml) file")
 
-        target.add_argument("-m", dest="bulkFile", help="Scan multiple targets given "
-                          "in a textual file ")
+        target.add_argument("-m", dest="bulkFile",
+            help="Scan multiple targets given in a textual file ")
 
         target.add_argument("-r", dest="requestFile",
-                          help="Load HTTP request from a file")
+            help="Load HTTP request from a file")
 
         target.add_argument("-g", dest="googleDork",
-                          help="Process Google dork results as target URLs")
+            help="Process Google dork results as target URLs")
 
         target.add_argument("-c", dest="configFile",
-                          help="Load options from a configuration INI file")
+            help="Load options from a configuration INI file")
 
         # Request options
-        request = parser.add_argument_group("Request", "These options can be used "
-                              "to specify how to connect to the target URL")
+        request = parser.add_argument_group("Request", "These options can be used to specify how to connect to the target URL")
 
         request.add_argument("--method", dest="method",
-                           help="Force usage of given HTTP method (e.g. PUT)")
+            help="Force usage of given HTTP method (e.g. PUT)")
 
         request.add_argument("--data", dest="data",
-                           help="Data string to be sent through POST (e.g. \"id=1\")")
+            help="Data string to be sent through POST (e.g. \"id=1\")")
 
         request.add_argument("--param-del", dest="paramDel",
-                           help="Character used for splitting parameter values (e.g. &)")
+            help="Character used for splitting parameter values (e.g. &)")
 
         request.add_argument("--cookie", dest="cookie",
-                           help="HTTP Cookie header value (e.g. \"PHPSESSID=a8d127e..\")")
+            help="HTTP Cookie header value (e.g. \"PHPSESSID=a8d127e..\")")
 
         request.add_argument("--cookie-del", dest="cookieDel",
-                           help="Character used for splitting cookie values (e.g. ;)")
+            help="Character used for splitting cookie values (e.g. ;)")
 
         request.add_argument("--load-cookies", dest="loadCookies",
-                           help="File containing cookies in Netscape/wget format")
+            help="File containing cookies in Netscape/wget format")
 
         request.add_argument("--drop-set-cookie", dest="dropSetCookie", action="store_true",
-                           help="Ignore Set-Cookie header from response")
+            help="Ignore Set-Cookie header from response")
 
         request.add_argument("--user-agent", dest="agent",
-                           help="HTTP User-Agent header value")
+            help="HTTP User-Agent header value")
 
         request.add_argument("--random-agent", dest="randomAgent", action="store_true",
-                           help="Use randomly selected HTTP User-Agent header value")
+            help="Use randomly selected HTTP User-Agent header value")
 
         request.add_argument("--host", dest="host",
-                           help="HTTP Host header value")
+            help="HTTP Host header value")
 
         request.add_argument("--referer", dest="referer",
-                           help="HTTP Referer header value")
+            help="HTTP Referer header value")
 
         request.add_argument("-H", "--header", dest="header",
-                           help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")")
+            help="Extra header (e.g. \"X-Forwarded-For: 127.0.0.1\")")
 
         request.add_argument("--headers", dest="headers",
-                           help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
+            help="Extra headers (e.g. \"Accept-Language: fr\\nETag: 123\")")
 
         request.add_argument("--auth-type", dest="authType",
-                           help="HTTP authentication type (Basic, Digest, NTLM or PKI)")
+            help="HTTP authentication type (Basic, Digest, NTLM or PKI)")
 
         request.add_argument("--auth-cred", dest="authCred",
-                           help="HTTP authentication credentials (name:password)")
+            help="HTTP authentication credentials (name:password)")
 
         request.add_argument("--auth-file", dest="authFile",
-                           help="HTTP authentication PEM cert/private key file")
+            help="HTTP authentication PEM cert/private key file")
 
-        request.add_argument("--ignore-code", dest="ignoreCode", type=int,
-                           help="Ignore (problematic) HTTP error code (e.g. 401)")
+        request.add_argument("--ignore-code", dest="ignoreCode",
+            help="Ignore (problematic) HTTP error code (e.g. 401)")
 
         request.add_argument("--ignore-proxy", dest="ignoreProxy", action="store_true",
-                           help="Ignore system default proxy settings")
+            help="Ignore system default proxy settings")
 
         request.add_argument("--ignore-redirects", dest="ignoreRedirects", action="store_true",
-                           help="Ignore redirection attempts")
+            help="Ignore redirection attempts")
 
         request.add_argument("--ignore-timeouts", dest="ignoreTimeouts", action="store_true",
-                           help="Ignore connection timeouts")
+            help="Ignore connection timeouts")
 
         request.add_argument("--proxy", dest="proxy",
-                           help="Use a proxy to connect to the target URL")
+            help="Use a proxy to connect to the target URL")
 
         request.add_argument("--proxy-cred", dest="proxyCred",
-                           help="Proxy authentication credentials (name:password)")
+            help="Proxy authentication credentials (name:password)")
 
         request.add_argument("--proxy-file", dest="proxyFile",
-                           help="Load proxy list from a file")
+            help="Load proxy list from a file")
 
         request.add_argument("--tor", dest="tor", action="store_true",
-                           help="Use Tor anonymity network")
+            help="Use Tor anonymity network")
 
         request.add_argument("--tor-port", dest="torPort",
-                           help="Set Tor proxy port other than default")
+            help="Set Tor proxy port other than default")
 
         request.add_argument("--tor-type", dest="torType",
-                           help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
+            help="Set Tor proxy type (HTTP, SOCKS4 or SOCKS5 (default))")
 
         request.add_argument("--check-tor", dest="checkTor", action="store_true",
-                           help="Check to see if Tor is used properly")
+            help="Check to see if Tor is used properly")
 
         request.add_argument("--delay", dest="delay", type=float,
-                           help="Delay in seconds between each HTTP request")
+            help="Delay in seconds between each HTTP request")
 
         request.add_argument("--timeout", dest="timeout", type=float,
-                           help="Seconds to wait before timeout connection (default %d)" % defaults.timeout)
+            help="Seconds to wait before timeout connection (default %d)" % defaults.timeout)
 
         request.add_argument("--retries", dest="retries", type=int,
-                           help="Retries when the connection timeouts (default %d)" % defaults.retries)
+            help="Retries when the connection timeouts (default %d)" % defaults.retries)
 
         request.add_argument("--randomize", dest="rParam",
-                           help="Randomly change value for given parameter(s)")
+            help="Randomly change value for given parameter(s)")
 
         request.add_argument("--safe-url", dest="safeUrl",
-                           help="URL address to visit frequently during testing")
+            help="URL address to visit frequently during testing")
 
         request.add_argument("--safe-post", dest="safePost",
-                           help="POST data to send to a safe URL")
+            help="POST data to send to a safe URL")
 
         request.add_argument("--safe-req", dest="safeReqFile",
-                           help="Load safe HTTP request from a file")
+            help="Load safe HTTP request from a file")
 
         request.add_argument("--safe-freq", dest="safeFreq", type=int,
-                           help="Test requests between two visits to a given safe URL")
+            help="Test requests between two visits to a given safe URL")
 
         request.add_argument("--skip-urlencode", dest="skipUrlEncode", action="store_true",
-                           help="Skip URL encoding of payload data")
+            help="Skip URL encoding of payload data")
 
         request.add_argument("--csrf-token", dest="csrfToken",
-                           help="Parameter used to hold anti-CSRF token")
+            help="Parameter used to hold anti-CSRF token")
 
         request.add_argument("--csrf-url", dest="csrfUrl",
-                           help="URL address to visit for extraction of anti-CSRF token")
+            help="URL address to visit for extraction of anti-CSRF token")
 
         request.add_argument("--force-ssl", dest="forceSSL", action="store_true",
-                           help="Force usage of SSL/HTTPS")
+            help="Force usage of SSL/HTTPS")
 
         request.add_argument("--chunked", dest="chunked", action="store_true",
-                           help="Use HTTP chunked transfer encoded (POST) requests")
+            help="Use HTTP chunked transfer encoded (POST) requests")
 
         request.add_argument("--hpp", dest="hpp", action="store_true",
-                           help="Use HTTP parameter pollution method")
+            help="Use HTTP parameter pollution method")
 
         request.add_argument("--eval", dest="evalCode",
-                           help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
+            help="Evaluate provided Python code before the request (e.g. \"import hashlib;id2=hashlib.md5(id).hexdigest()\")")
 
         # Optimization options
         optimization = parser.add_argument_group("Optimization", "These options can be used to optimize the performance of sqlmap")
 
         optimization.add_argument("-o", dest="optimize", action="store_true",
-                                help="Turn on all optimization switches")
+            help="Turn on all optimization switches")
 
         optimization.add_argument("--predict-output", dest="predictOutput", action="store_true",
-                                help="Predict common queries output")
+            help="Predict common queries output")
 
         optimization.add_argument("--keep-alive", dest="keepAlive", action="store_true",
-                                help="Use persistent HTTP(s) connections")
+            help="Use persistent HTTP(s) connections")
 
         optimization.add_argument("--null-connection", dest="nullConnection", action="store_true",
-                                help="Retrieve page length without actual HTTP response body")
+            help="Retrieve page length without actual HTTP response body")
 
         optimization.add_argument("--threads", dest="threads", type=int,
-                                help="Max number of concurrent HTTP(s) "
-                                "requests (default %d)" % defaults.threads)
+            help="Max number of concurrent HTTP(s) requests (default %d)" % defaults.threads)
 
         # Injection options
         injection = parser.add_argument_group("Injection", "These options can be used to specify which parameters to test for, provide custom injection payloads and optional tampering scripts")
 
         injection.add_argument("-p", dest="testParameter",
-                             help="Testable parameter(s)")
+            help="Testable parameter(s)")
 
         injection.add_argument("--skip", dest="skip",
-                             help="Skip testing for given parameter(s)")
+            help="Skip testing for given parameter(s)")
 
         injection.add_argument("--skip-static", dest="skipStatic", action="store_true",
-                             help="Skip testing parameters that not appear to be dynamic")
+            help="Skip testing parameters that not appear to be dynamic")
 
         injection.add_argument("--param-exclude", dest="paramExclude",
-                             help="Regexp to exclude parameters from testing (e.g. \"ses\")")
+            help="Regexp to exclude parameters from testing (e.g. \"ses\")")
 
         injection.add_argument("--param-filter", dest="paramFilter",
-                             help="Select testable parameter(s) by place (e.g. \"POST\")")
+            help="Select testable parameter(s) by place (e.g. \"POST\")")
 
         injection.add_argument("--dbms", dest="dbms",
-                             help="Force back-end DBMS to provided value")
+            help="Force back-end DBMS to provided value")
 
         injection.add_argument("--dbms-cred", dest="dbmsCred",
-                             help="DBMS authentication credentials (user:password)")
+            help="DBMS authentication credentials (user:password)")
 
         injection.add_argument("--os", dest="os",
-                             help="Force back-end DBMS operating system to provided value")
+            help="Force back-end DBMS operating system to provided value")
 
         injection.add_argument("--invalid-bignum", dest="invalidBignum", action="store_true",
-                             help="Use big numbers for invalidating values")
+            help="Use big numbers for invalidating values")
 
         injection.add_argument("--invalid-logical", dest="invalidLogical", action="store_true",
-                             help="Use logical operations for invalidating values")
+            help="Use logical operations for invalidating values")
 
         injection.add_argument("--invalid-string", dest="invalidString", action="store_true",
-                             help="Use random strings for invalidating values")
+            help="Use random strings for invalidating values")
 
         injection.add_argument("--no-cast", dest="noCast", action="store_true",
-                             help="Turn off payload casting mechanism")
+            help="Turn off payload casting mechanism")
 
         injection.add_argument("--no-escape", dest="noEscape", action="store_true",
-                             help="Turn off string escaping mechanism")
+            help="Turn off string escaping mechanism")
 
         injection.add_argument("--prefix", dest="prefix",
-                             help="Injection payload prefix string")
+            help="Injection payload prefix string")
 
         injection.add_argument("--suffix", dest="suffix",
-                             help="Injection payload suffix string")
+            help="Injection payload suffix string")
 
         injection.add_argument("--tamper", dest="tamper",
-                             help="Use given script(s) for tampering injection data")
+            help="Use given script(s) for tampering injection data")
 
         # Detection options
         detection = parser.add_argument_group("Detection", "These options can be used to customize the detection phase")
 
         detection.add_argument("--level", dest="level", type=int,
-                             help="Level of tests to perform (1-5, default %d)" % defaults.level)
+            help="Level of tests to perform (1-5, default %d)" % defaults.level)
 
         detection.add_argument("--risk", dest="risk", type=int,
-                             help="Risk of tests to perform (1-3, default %d)" % defaults.risk)
+            help="Risk of tests to perform (1-3, default %d)" % defaults.risk)
 
         detection.add_argument("--string", dest="string",
-                             help="String to match when query is evaluated to True")
+            help="String to match when query is evaluated to True")
 
         detection.add_argument("--not-string", dest="notString",
-                             help="String to match when query is evaluated to False")
+            help="String to match when query is evaluated to False")
 
         detection.add_argument("--regexp", dest="regexp",
-                             help="Regexp to match when query is evaluated to True")
+            help="Regexp to match when query is evaluated to True")
 
         detection.add_argument("--code", dest="code", type=int,
-                             help="HTTP code to match when query is evaluated to True")
+            help="HTTP code to match when query is evaluated to True")
 
         detection.add_argument("--text-only", dest="textOnly", action="store_true",
-                             help="Compare pages based only on the textual content")
+            help="Compare pages based only on the textual content")
 
         detection.add_argument("--titles", dest="titles", action="store_true",
-                             help="Compare pages based only on their titles")
+            help="Compare pages based only on their titles")
 
         # Techniques options
         techniques = parser.add_argument_group("Techniques", "These options can be used to tweak testing of specific SQL injection techniques")
 
         techniques.add_argument("--technique", dest="technique",
-                              help="SQL injection techniques to use (default \"%s\")" % defaults.technique)
+            help="SQL injection techniques to use (default \"%s\")" % defaults.technique)
 
         techniques.add_argument("--time-sec", dest="timeSec", type=int,
-                              help="Seconds to delay the DBMS response (default %d)" % defaults.timeSec)
+            help="Seconds to delay the DBMS response (default %d)" % defaults.timeSec)
 
         techniques.add_argument("--union-cols", dest="uCols",
-                              help="Range of columns to test for UNION query SQL injection")
+            help="Range of columns to test for UNION query SQL injection")
 
         techniques.add_argument("--union-char", dest="uChar",
-                              help="Character to use for bruteforcing number of columns")
+            help="Character to use for bruteforcing number of columns")
 
         techniques.add_argument("--union-from", dest="uFrom",
-                              help="Table to use in FROM part of UNION query SQL injection")
+            help="Table to use in FROM part of UNION query SQL injection")
 
         techniques.add_argument("--dns-domain", dest="dnsDomain",
-                              help="Domain name used for DNS exfiltration attack")
+            help="Domain name used for DNS exfiltration attack")
 
         techniques.add_argument("--second-url", dest="secondUrl",
-                              help="Resulting page URL searched for second-order response")
+            help="Resulting page URL searched for second-order response")
 
         techniques.add_argument("--second-req", dest="secondReq",
-                              help="Load second-order HTTP request from file")
+            help="Load second-order HTTP request from file")
 
         # Fingerprint options
         fingerprint = parser.add_argument_group("Fingerprint")
 
         fingerprint.add_argument("-f", "--fingerprint", dest="extensiveFp", action="store_true",
-                               help="Perform an extensive DBMS version fingerprint")
+            help="Perform an extensive DBMS version fingerprint")
 
         # Enumeration options
         enumeration = parser.add_argument_group("Enumeration", "These options can be used to enumerate the back-end database management system information, structure and data contained in the tables. Moreover you can run your own SQL statements")
 
         enumeration.add_argument("-a", "--all", dest="getAll", action="store_true",
-                               help="Retrieve everything")
+            help="Retrieve everything")
 
         enumeration.add_argument("-b", "--banner", dest="getBanner", action="store_true",
-                               help="Retrieve DBMS banner")
+            help="Retrieve DBMS banner")
 
         enumeration.add_argument("--current-user", dest="getCurrentUser", action="store_true",
-                               help="Retrieve DBMS current user")
+            help="Retrieve DBMS current user")
 
         enumeration.add_argument("--current-db", dest="getCurrentDb", action="store_true",
-                               help="Retrieve DBMS current database")
+            help="Retrieve DBMS current database")
 
         enumeration.add_argument("--hostname", dest="getHostname", action="store_true",
-                               help="Retrieve DBMS server hostname")
+            help="Retrieve DBMS server hostname")
 
         enumeration.add_argument("--is-dba", dest="isDba", action="store_true",
-                               help="Detect if the DBMS current user is DBA")
+            help="Detect if the DBMS current user is DBA")
 
         enumeration.add_argument("--users", dest="getUsers", action="store_true",
-                               help="Enumerate DBMS users")
+            help="Enumerate DBMS users")
 
         enumeration.add_argument("--passwords", dest="getPasswordHashes", action="store_true",
-                               help="Enumerate DBMS users password hashes")
+            help="Enumerate DBMS users password hashes")
 
         enumeration.add_argument("--privileges", dest="getPrivileges", action="store_true",
-                               help="Enumerate DBMS users privileges")
+            help="Enumerate DBMS users privileges")
 
         enumeration.add_argument("--roles", dest="getRoles", action="store_true",
-                               help="Enumerate DBMS users roles")
+            help="Enumerate DBMS users roles")
 
         enumeration.add_argument("--dbs", dest="getDbs", action="store_true",
-                               help="Enumerate DBMS databases")
+            help="Enumerate DBMS databases")
 
         enumeration.add_argument("--tables", dest="getTables", action="store_true",
-                               help="Enumerate DBMS database tables")
+            help="Enumerate DBMS database tables")
 
         enumeration.add_argument("--columns", dest="getColumns", action="store_true",
-                               help="Enumerate DBMS database table columns")
+            help="Enumerate DBMS database table columns")
 
         enumeration.add_argument("--schema", dest="getSchema", action="store_true",
-                               help="Enumerate DBMS schema")
+            help="Enumerate DBMS schema")
 
         enumeration.add_argument("--count", dest="getCount", action="store_true",
-                               help="Retrieve number of entries for table(s)")
+            help="Retrieve number of entries for table(s)")
 
         enumeration.add_argument("--dump", dest="dumpTable", action="store_true",
-                               help="Dump DBMS database table entries")
+            help="Dump DBMS database table entries")
 
         enumeration.add_argument("--dump-all", dest="dumpAll", action="store_true",
-                               help="Dump all DBMS databases tables entries")
+            help="Dump all DBMS databases tables entries")
 
         enumeration.add_argument("--search", dest="search", action="store_true",
-                               help="Search column(s), table(s) and/or database name(s)")
+            help="Search column(s), table(s) and/or database name(s)")
 
         enumeration.add_argument("--comments", dest="getComments", action="store_true",
-                               help="Check for DBMS comments during enumeration")
+            help="Check for DBMS comments during enumeration")
 
         enumeration.add_argument("--statements", dest="getStatements", action="store_true",
-                               help="Retrieve SQL statements being run on DBMS")
+            help="Retrieve SQL statements being run on DBMS")
 
         enumeration.add_argument("-D", dest="db",
-                               help="DBMS database to enumerate")
+            help="DBMS database to enumerate")
 
         enumeration.add_argument("-T", dest="tbl",
-                               help="DBMS database table(s) to enumerate")
+            help="DBMS database table(s) to enumerate")
 
         enumeration.add_argument("-C", dest="col",
-                               help="DBMS database table column(s) to enumerate")
+            help="DBMS database table column(s) to enumerate")
 
         enumeration.add_argument("-X", dest="exclude",
-                               help="DBMS database identifier(s) to not enumerate")
+            help="DBMS database identifier(s) to not enumerate")
 
         enumeration.add_argument("-U", dest="user",
-                               help="DBMS user to enumerate")
+            help="DBMS user to enumerate")
 
         enumeration.add_argument("--exclude-sysdbs", dest="excludeSysDbs", action="store_true",
-                               help="Exclude DBMS system databases when enumerating tables")
+            help="Exclude DBMS system databases when enumerating tables")
 
         enumeration.add_argument("--pivot-column", dest="pivotColumn",
-                               help="Pivot column name")
+            help="Pivot column name")
 
         enumeration.add_argument("--where", dest="dumpWhere",
-                               help="Use WHERE condition while table dumping")
+            help="Use WHERE condition while table dumping")
 
         enumeration.add_argument("--start", dest="limitStart", type=int,
-                               help="First dump table entry to retrieve")
+            help="First dump table entry to retrieve")
 
         enumeration.add_argument("--stop", dest="limitStop", type=int,
-                               help="Last dump table entry to retrieve")
+            help="Last dump table entry to retrieve")
 
         enumeration.add_argument("--first", dest="firstChar", type=int,
-                               help="First query output word character to retrieve")
+            help="First query output word character to retrieve")
 
         enumeration.add_argument("--last", dest="lastChar", type=int,
-                               help="Last query output word character to retrieve")
+            help="Last query output word character to retrieve")
 
         enumeration.add_argument("--sql-query", dest="sqlQuery",
-                               help="SQL statement to be executed")
+            help="SQL statement to be executed")
 
         enumeration.add_argument("--sql-shell", dest="sqlShell", action="store_true",
-                               help="Prompt for an interactive SQL shell")
+            help="Prompt for an interactive SQL shell")
 
         enumeration.add_argument("--sql-file", dest="sqlFile",
-                               help="Execute SQL statements from given file(s)")
+            help="Execute SQL statements from given file(s)")
 
         # Brute force options
         brute = parser.add_argument_group("Brute force", "These options can be used to run brute force checks")
 
         brute.add_argument("--common-tables", dest="commonTables", action="store_true",
-                         help="Check existence of common tables")
+            help="Check existence of common tables")
 
         brute.add_argument("--common-columns", dest="commonColumns", action="store_true",
-                         help="Check existence of common columns")
+            help="Check existence of common columns")
 
         brute.add_argument("--common-files", dest="commonFiles", action="store_true",
-                         help="Check existence of common files")
+            help="Check existence of common files")
 
         # User-defined function options
         udf = parser.add_argument_group("User-defined function injection", "These options can be used to create custom user-defined functions")
 
         udf.add_argument("--udf-inject", dest="udfInject", action="store_true",
-                       help="Inject custom user-defined functions")
+            help="Inject custom user-defined functions")
 
         udf.add_argument("--shared-lib", dest="shLib",
-                       help="Local path of the shared library")
+            help="Local path of the shared library")
 
         # File system options
         filesystem = parser.add_argument_group("File system access", "These options can be used to access the back-end database management system underlying file system")
 
         filesystem.add_argument("--file-read", dest="fileRead",
-                              help="Read a file from the back-end DBMS file system")
+            help="Read a file from the back-end DBMS file system")
 
         filesystem.add_argument("--file-write", dest="fileWrite",
-                              help="Write a local file on the back-end DBMS file system")
+            help="Write a local file on the back-end DBMS file system")
 
         filesystem.add_argument("--file-dest", dest="fileDest",
-                              help="Back-end DBMS absolute filepath to write to")
+            help="Back-end DBMS absolute filepath to write to")
 
         # Takeover options
         takeover = parser.add_argument_group("Operating system access", "These options can be used to access the back-end database management system underlying operating system")
 
         takeover.add_argument("--os-cmd", dest="osCmd",
-                            help="Execute an operating system command")
+            help="Execute an operating system command")
 
         takeover.add_argument("--os-shell", dest="osShell", action="store_true",
-                            help="Prompt for an interactive operating system shell")
+            help="Prompt for an interactive operating system shell")
 
         takeover.add_argument("--os-pwn", dest="osPwn", action="store_true",
-                            help="Prompt for an OOB shell, Meterpreter or VNC")
+            help="Prompt for an OOB shell, Meterpreter or VNC")
 
         takeover.add_argument("--os-smbrelay", dest="osSmb", action="store_true",
-                            help="One click prompt for an OOB shell, Meterpreter or VNC")
+            help="One click prompt for an OOB shell, Meterpreter or VNC")
 
         takeover.add_argument("--os-bof", dest="osBof", action="store_true",
-                            help="Stored procedure buffer overflow "
+            help="Stored procedure buffer overflow "
                                  "exploitation")
 
         takeover.add_argument("--priv-esc", dest="privEsc", action="store_true",
-                            help="Database process user privilege escalation")
+            help="Database process user privilege escalation")
 
         takeover.add_argument("--msf-path", dest="msfPath",
-                            help="Local path where Metasploit Framework is installed")
+            help="Local path where Metasploit Framework is installed")
 
         takeover.add_argument("--tmp-path", dest="tmpPath",
-                            help="Remote absolute path of temporary files directory")
+            help="Remote absolute path of temporary files directory")
 
         # Windows registry options
         windows = parser.add_argument_group("Windows registry access", "These options can be used to access the back-end database management system Windows registry")
 
         windows.add_argument("--reg-read", dest="regRead", action="store_true",
-                           help="Read a Windows registry key value")
+            help="Read a Windows registry key value")
 
         windows.add_argument("--reg-add", dest="regAdd", action="store_true",
-                           help="Write a Windows registry key value data")
+            help="Write a Windows registry key value data")
 
         windows.add_argument("--reg-del", dest="regDel", action="store_true",
-                           help="Delete a Windows registry key value")
+            help="Delete a Windows registry key value")
 
         windows.add_argument("--reg-key", dest="regKey",
-                           help="Windows registry key")
+            help="Windows registry key")
 
         windows.add_argument("--reg-value", dest="regVal",
-                           help="Windows registry key value")
+            help="Windows registry key value")
 
         windows.add_argument("--reg-data", dest="regData",
-                           help="Windows registry key value data")
+            help="Windows registry key value data")
 
         windows.add_argument("--reg-type", dest="regType",
-                           help="Windows registry key value type")
+            help="Windows registry key value type")
 
         # General options
         general = parser.add_argument_group("General", "These options can be used to set some general working parameters")
 
         general.add_argument("-s", dest="sessionFile",
-                           help="Load session from a stored (.sqlite) file")
+            help="Load session from a stored (.sqlite) file")
 
         general.add_argument("-t", dest="trafficFile",
-                           help="Log all HTTP traffic into a textual file")
+            help="Log all HTTP traffic into a textual file")
 
         general.add_argument("--batch", dest="batch", action="store_true",
-                           help="Never ask for user input, use the default behavior")
+            help="Never ask for user input, use the default behavior")
 
         general.add_argument("--binary-fields", dest="binaryFields",
-                           help="Result fields having binary values (e.g. \"digest\")")
+            help="Result fields having binary values (e.g. \"digest\")")
 
         general.add_argument("--check-internet", dest="checkInternet", action="store_true",
-                           help="Check Internet connection before assessing the target")
+            help="Check Internet connection before assessing the target")
 
         general.add_argument("--crawl", dest="crawlDepth", type=int,
-                           help="Crawl the website starting from the target URL")
+            help="Crawl the website starting from the target URL")
 
         general.add_argument("--crawl-exclude", dest="crawlExclude",
-                           help="Regexp to exclude pages from crawling (e.g. \"logout\")")
+            help="Regexp to exclude pages from crawling (e.g. \"logout\")")
 
         general.add_argument("--csv-del", dest="csvDel",
-                           help="Delimiting character used in CSV output (default \"%s\")" % defaults.csvDel)
+            help="Delimiting character used in CSV output (default \"%s\")" % defaults.csvDel)
 
         general.add_argument("--charset", dest="charset",
-                           help="Blind SQL injection charset (e.g. \"0123456789abcdef\")")
+            help="Blind SQL injection charset (e.g. \"0123456789abcdef\")")
 
         general.add_argument("--dump-format", dest="dumpFormat",
-                           help="Format of dumped data (CSV (default), HTML or SQLITE)")
+            help="Format of dumped data (CSV (default), HTML or SQLITE)")
 
         general.add_argument("--encoding", dest="encoding",
-                           help="Character encoding used for data retrieval (e.g. GBK)")
+            help="Character encoding used for data retrieval (e.g. GBK)")
 
         general.add_argument("--eta", dest="eta", action="store_true",
-                           help="Display for each output the estimated time of arrival")
+            help="Display for each output the estimated time of arrival")
 
         general.add_argument("--flush-session", dest="flushSession", action="store_true",
-                           help="Flush session files for current target")
+            help="Flush session files for current target")
 
         general.add_argument("--forms", dest="forms", action="store_true",
-                           help="Parse and test forms on target URL")
+            help="Parse and test forms on target URL")
 
         general.add_argument("--fresh-queries", dest="freshQueries", action="store_true",
-                           help="Ignore query results stored in session file")
+            help="Ignore query results stored in session file")
 
         general.add_argument("--har", dest="harFile",
-                           help="Log all HTTP traffic into a HAR file")
+            help="Log all HTTP traffic into a HAR file")
 
         general.add_argument("--hex", dest="hexConvert", action="store_true",
-                           help="Use hex conversion during data retrieval")
+            help="Use hex conversion during data retrieval")
 
         general.add_argument("--output-dir", dest="outputDir", action="store",
-                           help="Custom output directory path")
+            help="Custom output directory path")
 
         general.add_argument("--parse-errors", dest="parseErrors", action="store_true",
-                           help="Parse and display DBMS error messages from responses")
+            help="Parse and display DBMS error messages from responses")
 
         general.add_argument("--preprocess", dest="preprocess",
-                           help="Use given script(s) for preprocessing of response data")
+            help="Use given script(s) for preprocessing of response data")
 
         general.add_argument("--repair", dest="repair", action="store_true",
-                           help="Redump entries having unknown character marker (%s)" % INFERENCE_UNKNOWN_CHAR)
+            help="Redump entries having unknown character marker (%s)" % INFERENCE_UNKNOWN_CHAR)
 
         general.add_argument("--save", dest="saveConfig",
-                           help="Save options to a configuration INI file")
+            help="Save options to a configuration INI file")
 
         general.add_argument("--scope", dest="scope",
-                           help="Regexp to filter targets from provided proxy log")
+            help="Regexp to filter targets from provided proxy log")
 
         general.add_argument("--test-filter", dest="testFilter",
-                           help="Select tests by payloads and/or titles (e.g. ROW)")
+            help="Select tests by payloads and/or titles (e.g. ROW)")
 
         general.add_argument("--test-skip", dest="testSkip",
-                           help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
+            help="Skip tests by payloads and/or titles (e.g. BENCHMARK)")
 
         general.add_argument("--update", dest="updateAll", action="store_true",
-                           help="Update sqlmap")
+            help="Update sqlmap")
 
         # Miscellaneous options
         miscellaneous = parser.add_argument_group("Miscellaneous")
 
         miscellaneous.add_argument("-z", dest="mnemonics",
-                                 help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
+            help="Use short mnemonics (e.g. \"flu,bat,ban,tec=EU\")")
 
         miscellaneous.add_argument("--alert", dest="alert",
-                                 help="Run host OS command(s) when SQL injection is found")
+            help="Run host OS command(s) when SQL injection is found")
 
         miscellaneous.add_argument("--answers", dest="answers",
-                                 help="Set predefined answers (e.g. \"quit=N,follow=N\")")
+            help="Set predefined answers (e.g. \"quit=N,follow=N\")")
 
         miscellaneous.add_argument("--beep", dest="beep", action="store_true",
-                                 help="Beep on question and/or when SQL injection is found")
+            help="Beep on question and/or when SQL injection is found")
 
         miscellaneous.add_argument("--cleanup", dest="cleanup", action="store_true",
-                                 help="Clean up the DBMS from sqlmap specific UDF and tables")
+            help="Clean up the DBMS from sqlmap specific UDF and tables")
 
         miscellaneous.add_argument("--dependencies", dest="dependencies", action="store_true",
-                                 help="Check for missing (optional) sqlmap dependencies")
+            help="Check for missing (optional) sqlmap dependencies")
 
         miscellaneous.add_argument("--disable-coloring", dest="disableColoring", action="store_true",
-                                 help="Disable console output coloring")
+            help="Disable console output coloring")
 
         miscellaneous.add_argument("--gpage", dest="googlePage", type=int,
-                                 help="Use Google dork results from specified page number")
+            help="Use Google dork results from specified page number")
 
         miscellaneous.add_argument("--list-tampers", dest="listTampers", action="store_true",
-                                 help="Display list of available tamper scripts")
+            help="Display list of available tamper scripts")
 
         miscellaneous.add_argument("--mobile", dest="mobile", action="store_true",
-                                 help="Imitate smartphone through HTTP User-Agent header")
+            help="Imitate smartphone through HTTP User-Agent header")
 
         miscellaneous.add_argument("--offline", dest="offline", action="store_true",
-                                 help="Work in offline mode (only use session data)")
+            help="Work in offline mode (only use session data)")
 
         miscellaneous.add_argument("--purge", dest="purge", action="store_true",
-                                 help="Safely remove all content from sqlmap data directory")
+            help="Safely remove all content from sqlmap data directory")
 
         miscellaneous.add_argument("--skip-waf", dest="skipWaf", action="store_true",
-                                 help="Skip heuristic detection of WAF/IPS protection")
+            help="Skip heuristic detection of WAF/IPS protection")
 
         miscellaneous.add_argument("--smart", dest="smart", action="store_true",
-                                 help="Conduct thorough tests only if positive heuristic(s)")
+            help="Conduct thorough tests only if positive heuristic(s)")
 
         miscellaneous.add_argument("--sqlmap-shell", dest="sqlmapShell", action="store_true",
-                                 help="Prompt for an interactive sqlmap shell")
+            help="Prompt for an interactive sqlmap shell")
 
         miscellaneous.add_argument("--tmp-dir", dest="tmpDir",
-                                 help="Local directory for storing temporary files")
+            help="Local directory for storing temporary files")
 
         miscellaneous.add_argument("--web-root", dest="webRoot",
-                                 help="Web server document root directory (e.g. \"/var/www\")")
+            help="Web server document root directory (e.g. \"/var/www\")")
 
         miscellaneous.add_argument("--wizard", dest="wizard", action="store_true",
-                                 help="Simple wizard interface for beginner users")
+            help="Simple wizard interface for beginner users")
 
         # Hidden and/or experimental options
         parser.add_argument("--base64", dest="base64Parameter",
-                          help=SUPPRESS)
-#                          help="Parameter(s) containing Base64 encoded values")
+            help=SUPPRESS)  # "Parameter(s) containing Base64 encoded values"
 
         parser.add_argument("--crack", dest="hashFile",
-                          help=SUPPRESS)
-#                          help="Load and crack hashes from a file (standalone)")
+            help=SUPPRESS)  # "Load and crack hashes from a file (standalone)"
 
         parser.add_argument("--dummy", dest="dummy", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--murphy-rate", dest="murphyRate", type=int,
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--debug", dest="debug", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--disable-precon", dest="disablePrecon", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--disable-stats", dest="disableStats", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--profile", dest="profile", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--force-dbms", dest="forceDbms",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--force-dns", dest="forceDns", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
+
+        parser.add_argument("--force-partial", dest="forcePartial", action="store_true",
+            help=SUPPRESS)
 
         parser.add_argument("--force-pivoting", dest="forcePivoting", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--smoke-test", dest="smokeTest", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--live-test", dest="liveTest", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--vuln-test", dest="vulnTest", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
         parser.add_argument("--stop-fail", dest="stopFail", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
-        parser.add_argument("--run-case", dest="runCase", help=SUPPRESS)
+        parser.add_argument("--run-case", dest="runCase",
+            help=SUPPRESS)
 
         # API options
         parser.add_argument("--api", dest="api", action="store_true",
-                          help=SUPPRESS)
+            help=SUPPRESS)
 
-        parser.add_argument("--taskid", dest="taskid", help=SUPPRESS)
+        parser.add_argument("--taskid", dest="taskid",
+            help=SUPPRESS)
 
-        parser.add_argument("--database", dest="database", help=SUPPRESS)
+        parser.add_argument("--database", dest="database",
+            help=SUPPRESS)
 
         # Dirty hack to display longer options without breaking into two lines
         if hasattr(parser, "formatter"):
@@ -805,7 +807,7 @@ def cmdLineParser(argv=None):
                     action.option_strings = ["-hh"]
                     break
 
-        ## Dirty hack for inherent help message of switch '-h'
+        # Dirty hack for inherent help message of switch '-h'
         if hasattr(parser, "get_option"):
             option = parser.get_option("-h")
             option.help = option.help.capitalize().replace("this help", "basic help")
@@ -831,6 +833,8 @@ def cmdLineParser(argv=None):
         prompt = "--sqlmap-shell" in argv
 
         if prompt:
+            _createHomeDirectories()
+
             parser.usage = ""
             cmdLineOptions.sqlmapShell = True
 

lib/request/basic.py

@@ -34,6 +34,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.decorators import cachedmethod
+from lib.core.dicts import HTML_ENTITIES
 from lib.core.enums import DBMS
 from lib.core.enums import HTTP_HEADER
 from lib.core.enums import PLACE
@@ -49,7 +50,6 @@ from lib.core.settings import SELECT_FROM_TABLE_REGEX
 from lib.core.settings import VIEWSTATE_REGEX
 from lib.parse.headers import headersParser
 from lib.parse.html import htmlParser
-from lib.utils.htmlentities import htmlEntities
 from thirdparty import six
 from thirdparty.chardet import detect
 from thirdparty.identywaf import identYwaf
@@ -341,7 +341,7 @@ def decodePage(page, contentEncoding, contentType):
             page = re.sub(b"%([0-9a-fA-F]{2})", lambda _: decodeHex(_.group(1)), page)
 
         # e.g. &
-        page = re.sub(b"&([^;]+);", lambda _: six.int2byte(htmlEntities[getText(_.group(1))]) if htmlEntities.get(getText(_.group(1)), 256) < 256 else _.group(0), page)
+        page = re.sub(b"&([^;]+);", lambda _: six.int2byte(HTML_ENTITIES[getText(_.group(1))]) if HTML_ENTITIES.get(getText(_.group(1)), 256) < 256 else _.group(0), page)
 
         kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
 
@@ -364,7 +364,7 @@ def decodePage(page, contentEncoding, contentType):
             page = re.sub(r"&#(\d+);", _, page)
 
         # e.g. &zeta;
-        page = re.sub(r"&([^;]+);", lambda _: _unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
+        page = re.sub(r"&([^;]+);", lambda _: _unichr(HTML_ENTITIES[_.group(1)]) if HTML_ENTITIES.get(_.group(1), 0) > 255 else _.group(0), page)
 
     return page
 

lib/request/connect.py

@@ -7,6 +7,7 @@ See the file 'LICENSE' for copying permission
 
 import binascii
 import logging
+import random
 import re
 import socket
 import string
@@ -646,7 +647,7 @@ class Connect(object):
             if not multipart:
                 logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
 
-            if ex.code != conf.ignoreCode:
+            if ex.code not in (conf.ignoreCode or []):
                 if ex.code == _http_client.UNAUTHORIZED:
                     errMsg = "not authorized, try to provide right HTTP "
                     errMsg += "authentication type and valid credentials (%d)" % code
@@ -1102,7 +1103,8 @@ class Connect(object):
                 match = re.search(r"(\A|\b)%s=(?P<value>[^&;]*)" % re.escape(randomParameter), paramString)
                 if match:
                     origValue = match.group("value")
-                    retVal = re.sub(r"(\A|\b)%s=[^&;]*" % re.escape(randomParameter), "%s=%s" % (randomParameter, randomizeParameterValue(origValue)), paramString)
+                    newValue = randomizeParameterValue(origValue) if randomParameter not in kb.randomPool else random.sample(kb.randomPool[randomParameter], 1)[0]
+                    retVal = re.sub(r"(\A|\b)%s=[^&;]*" % re.escape(randomParameter), "%s=%s" % (randomParameter, newValue), paramString)
                 return retVal
 
             for randomParameter in conf.rParam:

lib/request/dns.py

@@ -84,7 +84,7 @@ class DNSServer(object):
         except:
             pass
         finally:
-            if response and "google" in response:
+            if response and b"google" in response:
                 raise socket.error("another DNS service already running on *:53")
 
     def pop(self, prefix=None, suffix=None):

lib/request/inject.py

@@ -39,6 +39,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.data import queries
+from lib.core.decorators import lockedmethod
 from lib.core.decorators import stackedmethod
 from lib.core.dicts import FROM_DUMMY_TABLE
 from lib.core.enums import CHARSET_TYPE
@@ -164,7 +165,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
 
     initTechnique(getTechnique())
 
-    query = agent.prefixQuery(kb.injection.data[getTechnique()].vector)
+    query = agent.prefixQuery(getTechniqueData().vector)
     query = agent.suffixQuery(query)
     payload = agent.payload(newValue=query)
     count = None
@@ -312,7 +313,7 @@ def _goBooleanProxy(expression):
     initTechnique(getTechnique())
 
     if conf.dnsDomain:
-        query = agent.prefixQuery(kb.injection.data[getTechnique()].vector)
+        query = agent.prefixQuery(getTechniqueData().vector)
         query = agent.suffixQuery(query)
         payload = agent.payload(newValue=query)
         output = _goDns(payload, expression)
@@ -320,7 +321,7 @@ def _goBooleanProxy(expression):
         if output is not None:
             return output
 
-    vector = kb.injection.data[getTechnique()].vector
+    vector = getTechniqueData().vector
     vector = vector.replace(INFERENCE_MARKER, expression)
     query = agent.prefixQuery(vector)
     query = agent.suffixQuery(query)
@@ -351,6 +352,7 @@ def _goUnion(expression, unpack=True, dump=False):
 
     return output
 
+@lockedmethod
 @stackedmethod
 def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True):
     """

lib/request/rangehandler.py

@@ -24,5 +24,6 @@ class HTTPRangeHandler(_urllib.request.BaseHandler):
 
     def http_error_416(self, req, fp, code, msg, hdrs):
         # HTTP's Range Not Satisfiable error
-        errMsg = "Invalid range"
+        errMsg = "there was a problem while connecting "
+        errMsg += "target ('406 - Range Not Satisfiable')"
         raise SqlmapConnectionException(errMsg)

lib/takeover/metasploit.py

@@ -68,29 +68,11 @@ class Metasploit(object):
         self.payloadConnStr = None
         self.localIP = getLocalIP()
         self.remoteIP = getRemoteIP() or conf.hostname
-        self._msfCli = normalizePath(os.path.join(conf.msfPath, "msfcli"))
-        self._msfConsole = normalizePath(os.path.join(conf.msfPath, "msfconsole"))
-        self._msfEncode = normalizePath(os.path.join(conf.msfPath, "msfencode"))
-        self._msfPayload = normalizePath(os.path.join(conf.msfPath, "msfpayload"))
-        self._msfVenom = normalizePath(os.path.join(conf.msfPath, "msfvenom"))
-
-        if IS_WIN:
-            _ = conf.msfPath
-            while _:
-                if os.path.exists(os.path.join(_, "scripts")):
-                    _ = os.path.join(_, "scripts", "setenv.bat")
-                    break
-                else:
-                    old = _
-                    _ = normalizePath(os.path.join(_, ".."))
-                    if _ == old:
-                        break
-
-            self._msfCli = "%s & ruby %s" % (_, self._msfCli)
-            self._msfConsole = "%s & ruby %s" % (_, self._msfConsole)
-            self._msfEncode = "ruby %s" % self._msfEncode
-            self._msfPayload = "%s & ruby %s" % (_, self._msfPayload)
-            self._msfVenom = "%s & ruby %s" % (_, self._msfVenom)
+        self._msfCli = normalizePath(os.path.join(conf.msfPath, "msfcli%s" % (".bat" if IS_WIN else "")))
+        self._msfConsole = normalizePath(os.path.join(conf.msfPath, "msfconsole%s" % (".bat" if IS_WIN else "")))
+        self._msfEncode = normalizePath(os.path.join(conf.msfPath, "msfencode%s" % (".bat" if IS_WIN else "")))
+        self._msfPayload = normalizePath(os.path.join(conf.msfPath, "msfpayload%s" % (".bat" if IS_WIN else "")))
+        self._msfVenom = normalizePath(os.path.join(conf.msfPath, "msfvenom%s" % (".bat" if IS_WIN else "")))
 
         self._msfPayloadsList = {
             "windows": {

lib/takeover/web.py

@@ -21,6 +21,7 @@ from lib.core.common import getManualDirectories
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSQLSnippet
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import isTechniqueAvailable
 from lib.core.common import isWindowsDriveLetterPath
 from lib.core.common import normalizePath
@@ -149,7 +150,7 @@ class Web(object):
         query = ""
 
         if isTechniqueAvailable(getTechnique()):
-            where = kb.injection.data[getTechnique()].where
+            where = getTechniqueData().where
 
             if where == PAYLOAD.WHERE.NEGATIVE:
                 randInt = randomInt()

lib/techniques/blind/inference.py

@@ -23,6 +23,7 @@ from lib.core.common import getCharset
 from lib.core.common import getCounter
 from lib.core.common import getPartRun
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import goGoodSamaritan
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
@@ -161,7 +162,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             length = None
 
         showEta = conf.eta and isinstance(length, int)
-        numThreads = min(conf.threads or 0, length or 0) or 1
+
+        if kb.bruteMode:
+            numThreads = 1
+        else:
+            numThreads = min(conf.threads or 0, length or 0) or 1
 
         if showEta:
             progress = ProgressBar(maxValue=length)
@@ -173,13 +178,13 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             else:
                 numThreads = 1
 
-        if conf.threads == 1 and not timeBasedCompare and not conf.predictOutput:
+        if conf.threads == 1 and not any((timeBasedCompare, conf.predictOutput)):
             warnMsg = "running in a single-thread mode. Please consider "
             warnMsg += "usage of option '--threads' for faster data retrieval"
             singleTimeWarnMessage(warnMsg)
 
-        if conf.verbose in (1, 2) and not showEta and not conf.api:
-            if isinstance(length, int) and conf.threads > 1:
+        if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
+            if isinstance(length, int) and numThreads > 1:
                 dataToStdout("[%s] [INFO] retrieved: %s" % (time.strftime("%X"), "_" * min(length, conf.progressWidth)))
                 dataToStdout("\r[%s] [INFO] retrieved: " % time.strftime("%X"))
             else:
@@ -229,10 +234,10 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
             result = not Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
 
-            if result and timeBasedCompare and kb.injection.data[getTechnique()].trueCode:
-                result = threadData.lastCode == kb.injection.data[getTechnique()].trueCode
+            if result and timeBasedCompare and getTechniqueData().trueCode:
+                result = threadData.lastCode == getTechniqueData().trueCode
                 if not result:
-                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, kb.injection.data[getTechnique()].trueCode)
+                    warnMsg = "detected HTTP code '%s' in validation phase is differing from expected '%s'" % (threadData.lastCode, getTechniqueData().trueCode)
                     singleTimeWarnMessage(warnMsg)
 
             incrementCounter(getTechnique())
@@ -341,8 +346,8 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                     result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
                     incrementCounter(getTechnique())
 
-                    if not timeBasedCompare:
-                        unexpectedCode |= threadData.lastCode not in (kb.injection.data[getTechnique()].falseCode, kb.injection.data[getTechnique()].trueCode)
+                    if not timeBasedCompare and getTechniqueData() is not None:
+                        unexpectedCode |= threadData.lastCode not in (getTechniqueData().falseCode, getTechniqueData().trueCode)
                         if unexpectedCode:
                             warnMsg = "unexpected HTTP code '%s' detected. Will use (extra) validation step in similar cases" % threadData.lastCode
                             singleTimeWarnMessage(warnMsg)
@@ -458,7 +463,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                         return decodeIntToUnicode(candidates[0])
 
         # Go multi-threading (--threads > 1)
-        if conf.threads > 1 and isinstance(length, int) and length > 1:
+        if numThreads > 1 and isinstance(length, int) and length > 1:
             threadData.shared.value = [None] * length
             threadData.shared.index = [firstChar]    # As list for python nested function scoping
             threadData.shared.start = firstChar
@@ -516,7 +521,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                                 if (endCharIndex - startCharIndex == conf.progressWidth) and (endCharIndex < length - 1):
                                     output = output[:-2] + ".."
 
-                                if conf.verbose in (1, 2) and not showEta and not conf.api:
+                                if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
                                     _ = count - firstChar
                                     output += '_' * (min(length, conf.progressWidth) - len(output))
                                     status = ' %d/%d (%d%%)' % (_, length, int(100.0 * _ / length))
@@ -546,7 +551,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                 finalValue = "".join(value)
                 infoMsg = "\r[%s] [INFO] retrieved: %s" % (time.strftime("%X"), filterControlChars(finalValue))
 
-            if conf.verbose in (1, 2) and not showEta and infoMsg and not conf.api:
+            if conf.verbose in (1, 2) and infoMsg and not any((showEta, conf.api, kb.bruteMode)):
                 dataToStdout(infoMsg)
 
         # No multi-threading (--threads = 1)
@@ -570,7 +575,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                         # One-shot query containing equals commonValue
                         testValue = unescaper.escape("'%s'" % commonValue) if "'" not in commonValue else unescaper.escape("%s" % commonValue, quote=False)
 
-                        query = kb.injection.data[getTechnique()].vector
+                        query = getTechniqueData().vector
                         query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)%s%s" % (expressionUnescaped, INFERENCE_EQUALS_CHAR, testValue)))
                         query = agent.suffixQuery(query)
 
@@ -594,7 +599,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                         subquery = queries[Backend.getIdentifiedDbms()].substring.query % (expressionUnescaped, 1, len(commonPattern))
                         testValue = unescaper.escape("'%s'" % commonPattern) if "'" not in commonPattern else unescaper.escape("%s" % commonPattern, quote=False)
 
-                        query = kb.injection.data[getTechnique()].vector
+                        query = getTechniqueData().vector
                         query = agent.prefixQuery(query.replace(INFERENCE_MARKER, "(%s)=%s" % (subquery, testValue)))
                         query = agent.suffixQuery(query)
 
@@ -631,7 +636,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
 
                 if showEta:
                     progress.progress(index)
-                elif conf.verbose in (1, 2) or conf.api:
+                elif (conf.verbose in (1, 2) and not kb.bruteMode) or conf.api:
                     dataToStdout(filterControlChars(val))
 
                 # some DBMSes (e.g. Firebird, DB2, etc.) have issues with trailing spaces
@@ -660,11 +665,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         elif partialValue:
             hashDBWrite(expression, "%s%s" % (PARTIAL_VALUE_MARKER if not conf.hexConvert else PARTIAL_HEX_VALUE_MARKER, partialValue))
 
-    if conf.hexConvert and not abortedFlag and not conf.api:
+    if conf.hexConvert and not any((abortedFlag, conf.api, kb.bruteMode)):
         infoMsg = "\r[%s] [INFO] retrieved: %s  %s\n" % (time.strftime("%X"), filterControlChars(finalValue), " " * retrievedLength)
         dataToStdout(infoMsg)
     else:
-        if conf.verbose in (1, 2) and not showEta and not conf.api:
+        if conf.verbose in (1, 2) and not any((showEta, conf.api, kb.bruteMode)):
             dataToStdout("\n")
 
         if (conf.verbose in (1, 2) and showEta) or conf.verbose >= 3:

lib/techniques/error/use.py

@@ -22,6 +22,7 @@ from lib.core.common import firstNotNone
 from lib.core.common import getConsoleWidth
 from lib.core.common import getPartRun
 from lib.core.common import getTechnique
+from lib.core.common import getTechniqueData
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import incrementCounter
@@ -124,7 +125,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                         nulledCastedField = queries[Backend.getIdentifiedDbms()].substring.query % (nulledCastedField, offset, kb.errorChunkLength)
 
                 # Forge the error-based SQL injection request
-                vector = kb.injection.data[getTechnique()].vector
+                vector = getTechniqueData().vector
                 query = agent.prefixQuery(vector)
                 query = agent.suffixQuery(query)
                 injExpression = expression.replace(field, nulledCastedField, 1) if field else expression
@@ -187,7 +188,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                     else:
                         break
 
-                    if output and conf.verbose in (1, 2) and not conf.api:
+                    if output and conf.verbose in (1, 2) and not any((conf.api, kb.bruteMode)):
                         if kb.fileReadMode:
                             dataToStdout(_formatPartialContent(output).replace(r"\n", "\n").replace(r"\t", "\t"))
                         elif offset > 1:

lib/techniques/union/test.py

@@ -201,7 +201,7 @@ def _unionPosition(comment, place, parameter, prefix, suffix, count, where=PAYLO
             if content and phrase in content:
                 validPayload = payload
                 kb.unionDuplicates = len(re.findall(phrase, content, re.I)) > 1
-                vector = (position, count, comment, prefix, suffix, kb.uChar, where, kb.unionDuplicates, False)
+                vector = (position, count, comment, prefix, suffix, kb.uChar, where, kb.unionDuplicates, conf.forcePartial)
 
                 if where == PAYLOAD.WHERE.ORIGINAL:
                     # Prepare expression with delimiters

lib/techniques/union/use.py

@@ -241,7 +241,7 @@ def unionUse(expression, unpack=True, dump=False):
     # SQL limiting the query output one entry at a time
     # NOTE: we assume that only queries that get data from a table can
     # return multiple entries
-    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or kb.forcePartialUnion or (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+    if value is None and (kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.NEGATIVE or kb.forcePartialUnion or conf.forcePartial or (dump and (conf.limitStart or conf.limitStop)) or "LIMIT " in expression.upper()) and " FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
         expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)
 
         if limitCond:

lib/utils/brute.py

@@ -102,7 +102,7 @@ def tableExists(tableFile, regex=None):
     threadData = getCurrentThreadData()
     threadData.shared.count = 0
     threadData.shared.limit = len(tables)
-    threadData.shared.value = []
+    threadData.shared.files = []
     threadData.shared.unique = set()
 
     def tableExistsThread():
@@ -128,7 +128,7 @@ def tableExists(tableFile, regex=None):
             kb.locks.io.acquire()
 
             if result and table.lower() not in threadData.shared.unique:
-                threadData.shared.value.append(table)
+                threadData.shared.files.append(table)
                 threadData.shared.unique.add(table.lower())
 
                 if conf.verbose in (1, 2) and not conf.api:
@@ -152,17 +152,17 @@ def tableExists(tableFile, regex=None):
     clearConsoleLine(True)
     dataToStdout("\n")
 
-    if not threadData.shared.value:
+    if not threadData.shared.files:
         warnMsg = "no table(s) found"
         logger.warn(warnMsg)
     else:
-        for item in threadData.shared.value:
+        for item in threadData.shared.files:
             if conf.db not in kb.data.cachedTables:
                 kb.data.cachedTables[conf.db] = [item]
             else:
                 kb.data.cachedTables[conf.db].append(item)
 
-    for _ in ((conf.db, item) for item in threadData.shared.value):
+    for _ in ((conf.db, item) for item in threadData.shared.files):
         if _ not in kb.brute.tables:
             kb.brute.tables.append(_)
 
@@ -224,7 +224,7 @@ def columnExists(columnFile, regex=None):
     threadData = getCurrentThreadData()
     threadData.shared.count = 0
     threadData.shared.limit = len(columns)
-    threadData.shared.value = []
+    threadData.shared.files = []
 
     def columnExistsThread():
         threadData = getCurrentThreadData()
@@ -244,7 +244,7 @@ def columnExists(columnFile, regex=None):
             kb.locks.io.acquire()
 
             if result:
-                threadData.shared.value.append(column)
+                threadData.shared.files.append(column)
 
                 if conf.verbose in (1, 2) and not conf.api:
                     clearConsoleLine(True)
@@ -269,13 +269,13 @@ def columnExists(columnFile, regex=None):
     clearConsoleLine(True)
     dataToStdout("\n")
 
-    if not threadData.shared.value:
+    if not threadData.shared.files:
         warnMsg = "no column(s) found"
         logger.warn(warnMsg)
     else:
         columns = {}
 
-        for column in threadData.shared.value:
+        for column in threadData.shared.files:
             if Backend.getIdentifiedDbms() in (DBMS.MYSQL,):
                 result = not inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s WHERE %s REGEXP '[^0-9]')", (column, table, column)))
             else:
@@ -327,7 +327,7 @@ def fileExists(pathFile):
     threadData = getCurrentThreadData()
     threadData.shared.count = 0
     threadData.shared.limit = len(paths)
-    threadData.shared.value = []
+    threadData.shared.files = []
 
     def fileExistsThread():
         threadData = getCurrentThreadData()
@@ -350,9 +350,9 @@ def fileExists(pathFile):
             kb.locks.io.acquire()
 
             if not isNoneValue(result):
-                threadData.shared.value.append(result)
+                threadData.shared.files.append(result)
 
-                if conf.verbose in (1, 2) and not conf.api:
+                if not conf.api:
                     clearConsoleLine(True)
                     infoMsg = "[%s] [INFO] retrieved: '%s'\n" % (time.strftime("%X"), path)
                     dataToStdout(infoMsg, True)
@@ -379,10 +379,10 @@ def fileExists(pathFile):
     clearConsoleLine(True)
     dataToStdout("\n")
 
-    if not threadData.shared.value:
+    if not threadData.shared.files:
         warnMsg = "no file(s) found"
         logger.warn(warnMsg)
     else:
-        retVal = threadData.shared.value
+        retVal = threadData.shared.files
 
     return retVal

lib/utils/hashdb.py

@@ -141,6 +141,8 @@ class HashDB(object):
                             self.cursor.execute("INSERT INTO storage VALUES (?, ?)", (hash_, value,))
                         except sqlite3.IntegrityError:
                             self.cursor.execute("UPDATE storage SET value=? WHERE id=?", (value, hash_,))
+                    except UnicodeError:  # e.g. surrogates not allowed (Issue #3851)
+                        break
                     except sqlite3.DatabaseError as ex:
                         if not os.path.exists(self.filepath):
                             debugMsg = "session file '%s' does not exist" % self.filepath

lib/utils/htmlentities.py

@@ -1,263 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
-See the file 'LICENSE' for copying permission
-"""
-
-# Reference: http://www.w3.org/TR/1999/REC-html401-19991224/sgml/entities.html
-
-htmlEntities = {
-    "quot": 34,
-    "amp": 38,
-    "lt": 60,
-    "gt": 62,
-    "nbsp": 160,
-    "iexcl": 161,
-    "cent": 162,
-    "pound": 163,
-    "curren": 164,
-    "yen": 165,
-    "brvbar": 166,
-    "sect": 167,
-    "uml": 168,
-    "copy": 169,
-    "ordf": 170,
-    "laquo": 171,
-    "not": 172,
-    "shy": 173,
-    "reg": 174,
-    "macr": 175,
-    "deg": 176,
-    "plusmn": 177,
-    "sup2": 178,
-    "sup3": 179,
-    "acute": 180,
-    "micro": 181,
-    "para": 182,
-    "middot": 183,
-    "cedil": 184,
-    "sup1": 185,
-    "ordm": 186,
-    "raquo": 187,
-    "frac14": 188,
-    "frac12": 189,
-    "frac34": 190,
-    "iquest": 191,
-    "Agrave": 192,
-    "Aacute": 193,
-    "Acirc": 194,
-    "Atilde": 195,
-    "Auml": 196,
-    "Aring": 197,
-    "AElig": 198,
-    "Ccedil": 199,
-    "Egrave": 200,
-    "Eacute": 201,
-    "Ecirc": 202,
-    "Euml": 203,
-    "Igrave": 204,
-    "Iacute": 205,
-    "Icirc": 206,
-    "Iuml": 207,
-    "ETH": 208,
-    "Ntilde": 209,
-    "Ograve": 210,
-    "Oacute": 211,
-    "Ocirc": 212,
-    "Otilde": 213,
-    "Ouml": 214,
-    "times": 215,
-    "Oslash": 216,
-    "Ugrave": 217,
-    "Uacute": 218,
-    "Ucirc": 219,
-    "Uuml": 220,
-    "Yacute": 221,
-    "THORN": 222,
-    "szlig": 223,
-    "agrave": 224,
-    "aacute": 225,
-    "acirc": 226,
-    "atilde": 227,
-    "auml": 228,
-    "aring": 229,
-    "aelig": 230,
-    "ccedil": 231,
-    "egrave": 232,
-    "eacute": 233,
-    "ecirc": 234,
-    "euml": 235,
-    "igrave": 236,
-    "iacute": 237,
-    "icirc": 238,
-    "iuml": 239,
-    "eth": 240,
-    "ntilde": 241,
-    "ograve": 242,
-    "oacute": 243,
-    "ocirc": 244,
-    "otilde": 245,
-    "ouml": 246,
-    "divide": 247,
-    "oslash": 248,
-    "ugrave": 249,
-    "uacute": 250,
-    "ucirc": 251,
-    "uuml": 252,
-    "yacute": 253,
-    "thorn": 254,
-    "yuml": 255,
-    "OElig": 338,
-    "oelig": 339,
-    "Scaron": 352,
-    "fnof": 402,
-    "scaron": 353,
-    "Yuml": 376,
-    "circ": 710,
-    "tilde": 732,
-    "Alpha": 913,
-    "Beta": 914,
-    "Gamma": 915,
-    "Delta": 916,
-    "Epsilon": 917,
-    "Zeta": 918,
-    "Eta": 919,
-    "Theta": 920,
-    "Iota": 921,
-    "Kappa": 922,
-    "Lambda": 923,
-    "Mu": 924,
-    "Nu": 925,
-    "Xi": 926,
-    "Omicron": 927,
-    "Pi": 928,
-    "Rho": 929,
-    "Sigma": 931,
-    "Tau": 932,
-    "Upsilon": 933,
-    "Phi": 934,
-    "Chi": 935,
-    "Psi": 936,
-    "Omega": 937,
-    "alpha": 945,
-    "beta": 946,
-    "gamma": 947,
-    "delta": 948,
-    "epsilon": 949,
-    "zeta": 950,
-    "eta": 951,
-    "theta": 952,
-    "iota": 953,
-    "kappa": 954,
-    "lambda": 955,
-    "mu": 956,
-    "nu": 957,
-    "xi": 958,
-    "omicron": 959,
-    "pi": 960,
-    "rho": 961,
-    "sigmaf": 962,
-    "sigma": 963,
-    "tau": 964,
-    "upsilon": 965,
-    "phi": 966,
-    "chi": 967,
-    "psi": 968,
-    "omega": 969,
-    "thetasym": 977,
-    "upsih": 978,
-    "piv": 982,
-    "bull": 8226,
-    "hellip": 8230,
-    "prime": 8242,
-    "Prime": 8243,
-    "oline": 8254,
-    "frasl": 8260,
-    "ensp": 8194,
-    "emsp": 8195,
-    "thinsp": 8201,
-    "zwnj": 8204,
-    "zwj": 8205,
-    "lrm": 8206,
-    "rlm": 8207,
-    "ndash": 8211,
-    "mdash": 8212,
-    "lsquo": 8216,
-    "rsquo": 8217,
-    "sbquo": 8218,
-    "ldquo": 8220,
-    "rdquo": 8221,
-    "bdquo": 8222,
-    "dagger": 8224,
-    "Dagger": 8225,
-    "permil": 8240,
-    "lsaquo": 8249,
-    "rsaquo": 8250,
-    "euro": 8364,
-    "weierp": 8472,
-    "image": 8465,
-    "real": 8476,
-    "trade": 8482,
-    "alefsym": 8501,
-    "larr": 8592,
-    "uarr": 8593,
-    "rarr": 8594,
-    "darr": 8595,
-    "harr": 8596,
-    "crarr": 8629,
-    "lArr": 8656,
-    "uArr": 8657,
-    "rArr": 8658,
-    "dArr": 8659,
-    "hArr": 8660,
-    "forall": 8704,
-    "part": 8706,
-    "exist": 8707,
-    "empty": 8709,
-    "nabla": 8711,
-    "isin": 8712,
-    "notin": 8713,
-    "ni": 8715,
-    "prod": 8719,
-    "sum": 8721,
-    "minus": 8722,
-    "lowast": 8727,
-    "radic": 8730,
-    "prop": 8733,
-    "infin": 8734,
-    "ang": 8736,
-    "and": 8743,
-    "or": 8744,
-    "cap": 8745,
-    "cup": 8746,
-    "int": 8747,
-    "there4": 8756,
-    "sim": 8764,
-    "cong": 8773,
-    "asymp": 8776,
-    "ne": 8800,
-    "equiv": 8801,
-    "le": 8804,
-    "ge": 8805,
-    "sub": 8834,
-    "sup": 8835,
-    "nsub": 8836,
-    "sube": 8838,
-    "supe": 8839,
-    "oplus": 8853,
-    "otimes": 8855,
-    "perp": 8869,
-    "sdot": 8901,
-    "lceil": 8968,
-    "rceil": 8969,
-    "lfloor": 8970,
-    "rfloor": 8971,
-    "lang": 9001,
-    "rang": 9002,
-    "loz": 9674,
-    "spades": 9824,
-    "clubs": 9827,
-    "hearts": 9829,
-    "diams": 9830,
-}

lib/utils/httpd.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
+See the file 'LICENSE' for copying permission
+"""
+
+from __future__ import print_function
+
+import mimetypes
+import gzip
+import os
+import re
+import sys
+import threading
+import time
+import traceback
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
+
+from lib.core.enums import HTTP_HEADER
+from lib.core.settings import UNICODE_ENCODING
+from lib.core.settings import VERSION_STRING
+from thirdparty import six
+from thirdparty.six.moves import BaseHTTPServer as _BaseHTTPServer
+from thirdparty.six.moves import http_client as _http_client
+from thirdparty.six.moves import socketserver as _socketserver
+from thirdparty.six.moves import urllib as _urllib
+
+HTTP_ADDRESS = "0.0.0.0"
+HTTP_PORT = 8951
+DEBUG = True
+HTML_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "data", "html"))
+DISABLED_CONTENT_EXTENSIONS = (".py", ".pyc", ".md", ".txt", ".bak", ".conf", ".zip", "~")
+
+class ThreadingServer(_socketserver.ThreadingMixIn, _BaseHTTPServer.HTTPServer):
+    def finish_request(self, *args, **kwargs):
+        try:
+            _BaseHTTPServer.HTTPServer.finish_request(self, *args, **kwargs)
+        except Exception:
+            if DEBUG:
+                traceback.print_exc()
+
+class ReqHandler(_BaseHTTPServer.BaseHTTPRequestHandler):
+    def do_GET(self):
+        path, query = self.path.split('?', 1) if '?' in self.path else (self.path, "")
+        params = {}
+        content = None
+
+        if query:
+            params.update(_urllib.parse.parse_qs(query))
+
+        for key in params:
+            if params[key]:
+                params[key] = params[key][-1]
+
+        self.url, self.params = path, params
+
+        if path == '/':
+            path = "index.html"
+
+        path = path.strip('/')
+
+        path = path.replace('/', os.path.sep)
+        path = os.path.abspath(os.path.join(HTML_DIR, path)).strip()
+
+        if not os.path.isfile(path) and os.path.isfile("%s.html" % path):
+            path = "%s.html" % path
+
+        if ".." not in os.path.relpath(path, HTML_DIR) and os.path.isfile(path) and not path.endswith(DISABLED_CONTENT_EXTENSIONS):
+            content = open(path, "rb").read()
+            self.send_response(_http_client.OK)
+            self.send_header(HTTP_HEADER.CONNECTION, "close")
+            self.send_header(HTTP_HEADER.CONTENT_TYPE, mimetypes.guess_type(path)[0] or "application/octet-stream")
+        else:
+            content = ("<!DOCTYPE html><html lang=\"en\"><head><title>404 Not Found</title></head><body><h1>Not Found</h1><p>The requested URL %s was not found on this server.</p></body></html>" % self.path.split('?')[0]).encode(UNICODE_ENCODING)
+            self.send_response(_http_client.NOT_FOUND)
+            self.send_header(HTTP_HEADER.CONNECTION, "close")
+
+        if content is not None:
+            for match in re.finditer(b"<\!(\w+)\!>", content):
+                name = match.group(1)
+                _ = getattr(self, "_%s" % name.lower(), None)
+                if _:
+                    content = self._format(content, **{name: _()})
+
+            if "gzip" in self.headers.get(HTTP_HEADER.ACCEPT_ENCODING):
+                self.send_header(HTTP_HEADER.CONTENT_ENCODING, "gzip")
+                _ = six.BytesIO()
+                compress = gzip.GzipFile("", "w+b", 9, _)
+                compress._stream = _
+                compress.write(content)
+                compress.flush()
+                compress.close()
+                content = compress._stream.getvalue()
+
+            self.send_header(HTTP_HEADER.CONTENT_LENGTH, str(len(content)))
+
+        self.end_headers()
+
+        if content:
+            self.wfile.write(content)
+
+        self.wfile.flush()
+
+    def _format(self, content, **params):
+        if content:
+            for key, value in params.items():
+                content = content.replace("<!%s!>" % key, value)
+
+        return content
+
+    def version_string(self):
+        return VERSION_STRING
+
+    def log_message(self, format, *args):
+        return
+
+    def finish(self):
+        try:
+            _BaseHTTPServer.BaseHTTPRequestHandler.finish(self)
+        except Exception:
+            if DEBUG:
+                traceback.print_exc()
+
+def start_httpd():
+    server = ThreadingServer((HTTP_ADDRESS, HTTP_PORT), ReqHandler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.daemon = True
+    thread.start()
+
+    print("[i] running HTTP server at '%s:%d'" % (HTTP_ADDRESS, HTTP_PORT))
+
+if __name__ == "__main__":
+    try:
+        start_httpd()
+
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        pass

plugins/dbms/mssqlserver/filesystem.py

@@ -52,7 +52,7 @@ class Filesystem(GenericFilesystem):
                 else:
                     scrString += " %s" % strLineChar
 
-                lineAddr += len(lineChar)
+                lineAddr += len(strLineChar) // 2
 
             fileLines.append(scrString)
 

plugins/generic/filesystem.py

@@ -21,6 +21,8 @@ from lib.core.common import isStackingAvailable
 from lib.core.common import isTechniqueAvailable
 from lib.core.common import readInput
 from lib.core.compat import xrange
+from lib.core.convert import encodeBase64
+from lib.core.convert import encodeHex
 from lib.core.convert import getText
 from lib.core.convert import getUnicode
 from lib.core.data import conf
@@ -134,8 +136,14 @@ class Filesystem(object):
     def fileContentEncode(self, content, encoding, single, chunkSize=256):
         retVal = []
 
-        if encoding:
-            content = getText(codecs.encode(content, encoding)).replace("\n", "")
+        if encoding == "hex":
+            content = encodeHex(content)
+        elif encoding == "base64":
+            content = encodeBase64(content)
+        else:
+            content = codecs.encode(content, encoding)
+
+        content = getText(content).replace("\n", "")
 
         if not single:
             if len(content) > chunkSize:

plugins/generic/takeover.py

@@ -10,6 +10,7 @@ import os
 from lib.core.common import Backend
 from lib.core.common import getSafeExString
 from lib.core.common import isStackingAvailable
+from lib.core.common import openFile
 from lib.core.common import readInput
 from lib.core.common import runningAsAdmin
 from lib.core.data import conf
@@ -137,7 +138,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry):
 
             if os.path.exists(filename):
                 try:
-                    with open(filename, "wb") as f:
+                    with openFile(filename, "wb") as f:
                         f.write("1")
                 except IOError as ex:
                     errMsg = "there has been a file opening/writing error "

sqlmap.py

@@ -238,7 +238,7 @@ def main():
             dataToStdout(excMsg)
             raise SystemExit
 
-        elif any(_ in excMsg for _ in ("ImportError", "Can't find file for module")):
+        elif any(_ in excMsg for _ in ("ImportError", "ModuleNotFoundError", "Can't find file for module")):
             errMsg = "invalid runtime environment ('%s')" % excMsg.split("Error: ")[-1].strip()
             logger.critical(errMsg)
             raise SystemExit
@@ -330,6 +330,12 @@ def main():
             logger.critical(errMsg)
             raise SystemExit
 
+        elif "can't allocate read lock" in excMsg:
+            errMsg = "there has been a problem in regular socket operation "
+            errMsg += "('%s')" % excMsg.strip().split('\n')[-1]
+            logger.critical(errMsg)
+            raise SystemExit
+
         elif "'DictObject' object has no attribute '" in excMsg and all(_ in errMsg for _ in ("(fingerprinted)", "(identified)")):
             errMsg = "there has been a problem in enumeration. "
             errMsg += "Because of a considerable chance of false-positive case "

tamper/plus2concat.py

@@ -10,7 +10,6 @@ import re
 
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import zeroDepthSearch
-from lib.core.compat import xrange
 from lib.core.enums import DBMS
 from lib.core.enums import PRIORITY
 
@@ -35,51 +34,22 @@ def tamper(payload, **kwargs):
     >>> tamper('SELECT CHAR(113)+CHAR(114)+CHAR(115) FROM DUAL')
     'SELECT CONCAT(CHAR(113),CHAR(114),CHAR(115)) FROM DUAL'
 
-    >>> tamper('SELECT (CHAR(113)+CHAR(114)+CHAR(115)) FROM DUAL')
-    'SELECT CONCAT(CHAR(113),CHAR(114),CHAR(115)) FROM DUAL'
+    >>> tamper('1 UNION ALL SELECT NULL,NULL,CHAR(113)+CHAR(118)+CHAR(112)+CHAR(112)+CHAR(113)+ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32))+CHAR(113)+CHAR(112)+CHAR(107)+CHAR(112)+CHAR(113)-- qtfe')
+    '1 UNION ALL SELECT NULL,NULL,CONCAT(CHAR(113),CHAR(118),CHAR(112),CHAR(112),CHAR(113),ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32)),CHAR(113),CHAR(112),CHAR(107),CHAR(112),CHAR(113))-- qtfe'
     """
 
     retVal = payload
 
     if payload:
-        prefix, suffix = '+' * len(re.search(r"\A(\+*)", payload).group(0)), '+' * len(re.search(r"(\+*)\Z", payload).group(0))
-        retVal = retVal.strip('+')
+        match = re.search(r"('[^']+'|CHAR\(\d+\))\+.*(?<=\+)('[^']+'|CHAR\(\d+\))", retVal)
+        if match:
+            part = match.group(0)
 
-        while True:
-            indexes = zeroDepthSearch(retVal, '+')
+            chars = [char for char in part]
+            for index in zeroDepthSearch(part, '+'):
+                chars[index] = ','
 
-            if indexes:
-                first, last = 0, 0
-                for i in xrange(1, len(indexes)):
-                    if ' ' in retVal[indexes[0]:indexes[i]]:
-                        break
-                    else:
-                        last = i
-
-                start = retVal[:indexes[first]].rfind(' ') + 1
-                end = (retVal[indexes[last] + 1:].find(' ') + indexes[last] + 1) if ' ' in retVal[indexes[last] + 1:] else len(retVal) - 1
-
-                chars = [char for char in retVal]
-                for index in indexes[first:last + 1]:
-                    chars[index] = ','
-
-                retVal = "%sCONCAT(%s)%s" % (retVal[:start], ''.join(chars)[start:end], retVal[end:])
-            else:
-                match = re.search(r"\((CHAR\(\d+.+\bCHAR\(\d+\))\)", retVal)
-                if match:
-                    part = match.group(0)
-                    indexes = set(zeroDepthSearch(match.group(1), '+'))
-                    if not indexes:
-                        break
-                    chars = [char for char in part]
-                    for i in xrange(1, len(chars)):
-                        if i - 1 in indexes:
-                            chars[i] = ','
-                    replacement = "CONCAT%s" % "".join(chars)
-                    retVal = retVal.replace(part, replacement)
-                else:
-                    break
-
-        retVal = "%s%s%s" % (prefix, retVal, suffix)
+            replacement = "CONCAT(%s)" % "".join(chars)
+            retVal = retVal.replace(part, replacement)
 
     return retVal

tamper/plus2fnconcat.py

@@ -36,63 +36,29 @@ def tamper(payload, **kwargs):
     >>> tamper('SELECT CHAR(113)+CHAR(114)+CHAR(115) FROM DUAL')
     'SELECT {fn CONCAT({fn CONCAT(CHAR(113),CHAR(114))},CHAR(115))} FROM DUAL'
 
-    >>> tamper('SELECT (CHAR(113)+CHAR(114)+CHAR(115)) FROM DUAL')
-    'SELECT {fn CONCAT({fn CONCAT(CHAR(113),CHAR(114))},CHAR(115))} FROM DUAL'
+    >>> tamper('1 UNION ALL SELECT NULL,NULL,CHAR(113)+CHAR(118)+CHAR(112)+CHAR(112)+CHAR(113)+ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32))+CHAR(113)+CHAR(112)+CHAR(107)+CHAR(112)+CHAR(113)-- qtfe')
+    '1 UNION ALL SELECT NULL,NULL,{fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT({fn CONCAT(CHAR(113),CHAR(118))},CHAR(112))},CHAR(112))},CHAR(113))},ISNULL(CAST(@@VERSION AS NVARCHAR(4000)),CHAR(32)))},CHAR(113))},CHAR(112))},CHAR(107))},CHAR(112))},CHAR(113))}-- qtfe'
     """
 
     retVal = payload
 
     if payload:
-        prefix, suffix = '+' * len(re.search(r"\A(\+*)", payload).group(0)), '+' * len(re.search(r"(\+*)\Z", payload).group(0))
-        retVal = retVal.strip('+')
+        match = re.search(r"('[^']+'|CHAR\(\d+\))\+.*(?<=\+)('[^']+'|CHAR\(\d+\))", retVal)
+        if match:
+            old = match.group(0)
+            parts = []
+            last = 0
 
-        while True:
-            indexes = zeroDepthSearch(retVal, '+')
+            for index in zeroDepthSearch(old, '+'):
+                parts.append(old[last:index].strip('+'))
+                last = index
 
-            if indexes:
-                first, last = 0, 0
-                for i in xrange(1, len(indexes)):
-                    if ' ' in retVal[indexes[0]:indexes[i]]:
-                        break
-                    else:
-                        last = i
+            parts.append(old[last:].strip('+'))
+            replacement = parts[0]
 
-                start = retVal[:indexes[first]].rfind(' ') + 1
-                end = (retVal[indexes[last] + 1:].find(' ') + indexes[last] + 1) if ' ' in retVal[indexes[last] + 1:] else len(retVal) - 1
+            for i in xrange(1, len(parts)):
+                replacement = "{fn CONCAT(%s,%s)}" % (replacement, parts[i])
 
-                count = 0
-                chars = [char for char in retVal]
-                for index in indexes[first:last + 1]:
-                    if count == 0:
-                        chars[index] = ','
-                    else:
-                        chars[index] = '\x01'
-                    count += 1
-
-                retVal = "%s%s%s)}%s" % (retVal[:start], "{fn CONCAT(" * count, ''.join(chars)[start:end].replace('\x01', ")},"), retVal[end:])
-            else:
-                match = re.search(r"\((CHAR\(\d+.+\bCHAR\(\d+\))\)", retVal)
-                if match:
-                    part = match.group(0)
-                    indexes = set(zeroDepthSearch(match.group(1), '+'))
-                    if not indexes:
-                        break
-
-                    count = 0
-                    chars = [char for char in part]
-                    for i in xrange(1, len(chars)):
-                        if i - 1 in indexes:
-                            if count == 0:
-                                chars[i] = ','
-                            else:
-                                chars[i] = '\x01'
-                            count += 1
-
-                    replacement = "%s%s}" % (("{fn CONCAT(" * count)[:-1], "".join(chars).replace('\x01', ")},"))
-                    retVal = retVal.replace(part, replacement)
-                else:
-                    break
-
-        retVal = "%s%s%s" % (prefix, retVal, suffix)
+            retVal = retVal.replace(old, replacement)
 
     return retVal

tamper/varnish.py

@@ -17,7 +17,7 @@ def tamper(payload, **kwargs):
     Appends a HTTP header 'X-originating-IP' to bypass Varnish Firewall
 
     Reference:
-        * http://h30499.www3.hp.com/t5/Fortify-Application-Security/Bypassing-web-application-firewalls-using-HTTP-headers/ba-p/6418366
+        * https://web.archive.org/web/20160815052159/http://community.hpe.com/t5/Protect-Your-Assets/Bypassing-web-application-firewalls-using-HTTP-headers/ba-p/6418366
 
     Notes:
         Examples:

thirdparty/identywaf/data.json

@@ -55,7 +55,8 @@
             "name": "360",
             "regex": "<title>493</title>|/wzws-waf-cgi/",
             "signatures": [
-                "9778:RVZXum61OEhCWapBYKcPk4JzWOpohM4JiUcMr2RXg1uQJbX3uhdOnthtOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A5chW1XaTC"
+                "9778:RVZXum61OEhCWapBYKcPk4JzWOpohM4JiUcMr2RXg1uQJbX3uhdOnthtOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A5chW1XaTC",
+                "9ccc:RVZXum61OEhCWapBYKcPk4JzWOpohM4JiUcMr2RXg1uQJbX3uhdOnthtOj+hX7AB16FcPxJPdLsXo2tKaK99n+i7c4VmkwI3FZjxtDtAeq+c36A4chW1XaTC"
             ]
         },
         "aesecure": {
@@ -677,6 +678,12 @@
             "signatures": [],
             "note": "Uses Incapsula (Reference: https://www.whitefirdesign.com/blog/2016/11/08/more-evidence-that-sitelocks-trueshield-web-application-firewall-is-really-incapsulas-waf/)"
         },
+        "sniper": {
+            "company": "Wins",
+            "name": "Sniper",
+            "regex": "document\\.title = [^;]+Sniper WAF",
+            "signatures": []
+        },
         "sonicwall": {
             "company": "Dell",
             "name": "SonicWALL",
@@ -777,6 +784,14 @@
                 "c02b:RVZXu261OElCWapBYKcPk4JzWOpohM4JiUcMr2RWg1uQJbX3uhdOnthsOj+hXrAB16FcPxJOdLsXo2tKaK99n+i7c4RmkgI2FZjxtDtAeq+c36A4chW1XaTC"
             ]
         },
+        "wapples": {
+            "company": "Penta Security",
+            "name": "Wapples",
+            "regex": "",
+            "signatures": [
+                "60b7:RVZXum60OEhCWKpAYKYPkoJyWOpohM4IiUYMrmRWg1uQJLX2uhZOnthtOj+hXrAA16FcPhJOdLoXomtKaK59nui7c4RmkgI2FZjxtDtAeq+c3qA4chW1XKTC"
+            ]
+        },
         "watchguard": {
             "company": "WatchGuard Technologies",
             "name": "WatchGuard",
@@ -809,6 +824,14 @@
                 "17a8:RVZXum60OEhCWKpAYKYPkoJyWOpohM4JiUcMrmRXg1qQJbX3uhdOnthtOj+hXrAB16FcPhJPdLoXomtKaK59nui6c4RmkgI2FZjxtDtAeq+c3qA4chS1XKTC"
             ]
         },
+        "webland": {
+            "company": "WebLand",
+            "name": "WebLand",
+            "regex": "Server: Apache Protected by Webland WAF",
+            "signatures": [
+                "4ba0:RVZXum60OEhCWKpAYKYPkoJzWOpohc4IiUYMr2RWg1uQJLX3uhZOnthsOj6hXrAA16BcPhJOdLoXomtKaK59nui6c4RmkgI2FZjxtDtAeq6c3qA4chS1XKTC"
+            ]
+        },
         "webseal": {
             "company": "IBM",
             "name": "WebSEAL",
@@ -824,7 +847,7 @@
             "signatures": []
         },
         "wordfence": {
-            "company": "Feedjit",
+            "company": "Defiant",
             "name": "Wordfence",
             "regex": "Generated by Wordfence|This response was generated by Wordfence|broke one of the Wordfence (advanced )?blocking rules|: wfWAF|/plugins/wordfence",
             "signatures": [