Fixes #3502

COMMITMENT

@@ -0,0 +1,46 @@
+GPL Cooperation Commitment
+Version 1.0
+
+Before filing or continuing to prosecute any legal proceeding or claim
+(other than a Defensive Action) arising from termination of a Covered
+License, we commit to extend to the person or entity ('you') accused
+of violating the Covered License the following provisions regarding
+cure and reinstatement, taken from GPL version 3. As used here, the
+term 'this License' refers to the specific Covered License being
+enforced.
+
+    However, if you cease all violation of this License, then your
+    license from a particular copyright holder is reinstated (a)
+    provisionally, unless and until the copyright holder explicitly
+    and finally terminates your license, and (b) permanently, if the
+    copyright holder fails to notify you of the violation by some
+    reasonable means prior to 60 days after the cessation.
+
+    Moreover, your license from a particular copyright holder is
+    reinstated permanently if the copyright holder notifies you of the
+    violation by some reasonable means, this is the first time you
+    have received notice of violation of this License (for any work)
+    from that copyright holder, and you cure the violation prior to 30
+    days after your receipt of the notice.
+
+We intend this Commitment to be irrevocable, and binding and
+enforceable against us and assignees of or successors to our
+copyrights.
+
+Definitions
+
+'Covered License' means the GNU General Public License, version 2
+(GPLv2), the GNU Lesser General Public License, version 2.1
+(LGPLv2.1), or the GNU Library General Public License, version 2
+(LGPLv2), all as published by the Free Software Foundation.
+
+'Defensive Action' means a legal proceeding or claim that We bring
+against you in response to a prior proceeding or claim initiated by
+you or your affiliate.
+
+'We' means each contributor to this repository as of the date of
+inclusion of this file, including subsidiaries of a corporate
+contributor.
+
+This work is available under a Creative Commons Attribution-ShareAlike
+4.0 International license (https://creativecommons.org/licenses/by-sa/4.0/).

extra/cloak/cloak.py

@@ -7,6 +7,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import os
 import sys
 import zlib
@@ -38,7 +40,7 @@ def decloak(inputFile=None, data=None):
     try:
         data = zlib.decompress(hideAscii(data))
     except:
-        print 'ERROR: the provided input file \'%s\' does not contain valid cloaked content' % inputFile
+        print('ERROR: the provided input file \'%s\' does not contain valid cloaked content' % inputFile)
         sys.exit(1)
     finally:
         f.close()
@@ -59,11 +61,11 @@ def main():
         if not args.inputFile:
             parser.error('Missing the input file, -h for help')
 
-    except (OptionError, TypeError), e:
-        parser.error(e)
+    except (OptionError, TypeError) as ex:
+        parser.error(ex)
 
     if not os.path.isfile(args.inputFile):
-        print 'ERROR: the provided input file \'%s\' is non existent' % args.inputFile
+        print('ERROR: the provided input file \'%s\' is non existent' % args.inputFile)
         sys.exit(1)
 
     if not args.decrypt:

extra/dbgtool/dbgtool.py

@@ -7,6 +7,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import os
 import sys
 import struct
@@ -19,7 +21,7 @@ def convert(inputFile):
     fileSize = fileStat.st_size
 
     if fileSize > 65280:
-        print "ERROR: the provided input file '%s' is too big for debug.exe" % inputFile
+        print("ERROR: the provided input file '%s' is too big for debug.exe" % inputFile)
         sys.exit(1)
 
     script = "n %s\nr cx\n" % os.path.basename(inputFile.replace(".", "_"))
@@ -59,7 +61,7 @@ def convert(inputFile):
 
 def main(inputFile, outputFile):
     if not os.path.isfile(inputFile):
-        print "ERROR: the provided input file '%s' is not a regular file" % inputFile
+        print("ERROR: the provided input file '%s' is not a regular file" % inputFile)
         sys.exit(1)
 
     script = convert(inputFile)
@@ -70,7 +72,7 @@ def main(inputFile, outputFile):
         sys.stdout.write(script)
         sys.stdout.close()
     else:
-        print script
+        print(script)
 
 if __name__ == "__main__":
     usage = "%s -i <input file> [-o <output file>]" % sys.argv[0]
@@ -86,8 +88,8 @@ if __name__ == "__main__":
         if not args.inputFile:
             parser.error("Missing the input file, -h for help")
 
-    except (OptionError, TypeError), e:
-        parser.error(e)
+    except (OptionError, TypeError) as ex:
+        parser.error(ex)
 
     inputFile = args.inputFile
     outputFile = args.outputFile

extra/icmpsh/icmpsh_m.py

@@ -128,7 +128,7 @@ def main(src, dst):
                 try:
                     # Send it to the target host
                     sock.sendto(ip.get_packet(), (dst, 0))
-                except socket.error, ex:
+                except socket.error as ex:
                     sys.stderr.write("'%s'\n" % ex)
                     sys.stderr.flush()
 

extra/safe2bin/safe2bin.py

@@ -7,6 +7,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import binascii
 import re
 import string
@@ -108,11 +110,11 @@ def main():
         if not args.inputFile:
             parser.error('Missing the input file, -h for help')
 
-    except (OptionError, TypeError), e:
-        parser.error(e)
+    except (OptionError, TypeError) as ex:
+        parser.error(ex)
 
     if not os.path.isfile(args.inputFile):
-        print 'ERROR: the provided input file \'%s\' is not a regular file' % args.inputFile
+        print('ERROR: the provided input file \'%s\' is not a regular file' % args.inputFile)
         sys.exit(1)
 
     f = open(args.inputFile, 'r')

extra/shutils/duplicates.py

@@ -5,6 +5,8 @@
 
 # Removes duplicate entries in wordlist like files
 
+from __future__ import print_function
+
 import sys
 
 if len(sys.argv) > 0:
@@ -17,7 +19,7 @@ if len(sys.argv) > 0:
                 str.encode(item)
                 if item in items:
                     if item:
-                        print item
+                        print(item)
                 else:
                     items.append(item)
             except:

extra/shutils/newlines.py

@@ -3,6 +3,8 @@
 # Runs pylint on all python scripts found in a directory tree
 # Reference: http://rowinggolfer.blogspot.com/2009/08/pylint-recursively.html
 
+from __future__ import print_function
+
 import os
 import sys
 
@@ -12,16 +14,16 @@ def check(filepath):
 
         if "\n\n\n" in content:
             index = content.find("\n\n\n")
-            print filepath, repr(content[index - 30:index + 30])
+            print(filepath, repr(content[index - 30:index + 30]))
 
 if __name__ == "__main__":
     try:
         BASE_DIRECTORY = sys.argv[1]
     except IndexError:
-        print "no directory specified, defaulting to current working directory"
+        print("no directory specified, defaulting to current working directory")
         BASE_DIRECTORY = os.getcwd()
 
-    print "looking for *.py scripts in subdirectories of ", BASE_DIRECTORY
+    print("looking for *.py scripts in subdirectories of '%s'" % BASE_DIRECTORY)
     for root, dirs, files in os.walk(BASE_DIRECTORY):
         if any(_ in root for _ in ("extra", "thirdparty")):
             continue

extra/shutils/postcommit-hook.sh

@@ -11,6 +11,7 @@ chmod +x .git/hooks/post-commit
 '
 
 SETTINGS="../../lib/core/settings.py"
+PYPI="../../extra/shutils/pypi.sh"
 
 declare -x SCRIPTPATH="${0}"
 
@@ -28,6 +29,6 @@ then
         git tag $NEW_TAG
         git push origin $NEW_TAG
         echo "Going to push PyPI package"
-        /bin/bash ${SCRIPTPATH%/*}/pypi.sh
+        /bin/bash ${SCRIPTPATH%/*}/$PYPI
     fi
 fi

extra/shutils/precommit-hook.sh

@@ -26,7 +26,7 @@ if [ -f $SETTINGS_FULLPATH ]
 then
     LINE=$(grep -o ${SETTINGS_FULLPATH} -e 'VERSION = "[0-9.]*"')
     declare -a LINE
-    INCREMENTED=$(python -c "import re, sys, time; version = re.search('\"([0-9.]*)\"', sys.argv[1]).group(1); _ = version.split('.'); _.append(0) if len(_) < 3 else _; _[-1] = str(int(_[-1]) + 1); month = str(time.gmtime().tm_mon); _[-1] = '0' if _[-2] != month else _[-1]; _[-2] = month; print sys.argv[1].replace(version, '.'.join(_))" "$LINE")
+    INCREMENTED=$(python -c "import re, sys, time; version = re.search('\"([0-9.]*)\"', sys.argv[1]).group(1); _ = version.split('.'); _.extend([0] * (4 - len(_))); _[-1] = str(int(_[-1]) + 1); month = str(time.gmtime().tm_mon); _[-1] = '0' if _[-2] != month else _[-1]; _[-2] = month; print sys.argv[1].replace(version, '.'.join(_))" "$LINE")
     if [ -n "$INCREMENTED" ]
     then
         sed -i "s/${LINE}/${INCREMENTED}/" $SETTINGS_FULLPATH
@@ -39,4 +39,4 @@ then
 fi
 
 truncate -s 0 "$CHECKSUM_FULLPATH"
-cd $PROJECT_FULLPATH && for i in $(find . -name "*.py" -o -name "*.xml" -o -iname "*_" | sort); do git ls-files $i --error-unmatch &>/dev/null && md5sum $i | stdbuf -i0 -o0 -e0 sed 's/\.\///' >> "$CHECKSUM_FULLPATH"; git add "$CHECKSUM_FULLPATH"; done
+cd $PROJECT_FULLPATH && for i in $(find . -name "*.py" -o -name "*.xml" -o -name "*_" -o -type f -regex "./[^./]*" | sort); do git ls-files $i --error-unmatch &>/dev/null && md5sum $i | stdbuf -i0 -o0 -e0 sed 's/\.\///' >> "$CHECKSUM_FULLPATH"; git add "$CHECKSUM_FULLPATH"; done

extra/shutils/pylint.py

@@ -3,6 +3,8 @@
 # Runs pylint on all python scripts found in a directory tree
 # Reference: http://rowinggolfer.blogspot.com/2009/08/pylint-recursively.html
 
+from __future__ import print_function
+
 import os
 import re
 import sys
@@ -17,26 +19,26 @@ def check(module):
 
     if module[-3:] == ".py":
 
-        print "CHECKING ", module
+        print("CHECKING ", module)
         pout = os.popen("pylint --rcfile=/dev/null %s" % module, 'r')
         for line in pout:
             if re.match(r"\AE:", line):
-                print line.strip()
+                print(line.strip())
             if __RATING__ and "Your code has been rated at" in line:
-                print line
+                print(line)
                 score = re.findall(r"\d.\d\d", line)[0]
                 total += float(score)
                 count += 1
 
 if __name__ == "__main__":
     try:
-        print sys.argv
+        print(sys.argv)
         BASE_DIRECTORY = sys.argv[1]
     except IndexError:
-        print "no directory specified, defaulting to current working directory"
+        print("no directory specified, defaulting to current working directory")
         BASE_DIRECTORY = os.getcwd()
 
-    print "looking for *.py scripts in subdirectories of ", BASE_DIRECTORY
+    print("looking for *.py scripts in subdirectories of ", BASE_DIRECTORY)
     for root, dirs, files in os.walk(BASE_DIRECTORY):
         if any(_ in root for _ in ("extra", "thirdparty")):
             continue
@@ -45,6 +47,6 @@ if __name__ == "__main__":
             check(filepath)
 
     if __RATING__:
-        print "==" * 50
-        print "%d modules found" % count
-        print "AVERAGE SCORE = %.02f" % (total / count)
+        print("==" * 50)
+        print("%d modules found" % count)
+        print("AVERAGE SCORE = %.02f" % (total / count))

extra/shutils/pypi.sh

@@ -30,6 +30,11 @@ setup(
     author='Bernardo Damele Assumpcao Guimaraes, Miroslav Stampar',
     author_email='[email protected], [email protected]',
     url='http://sqlmap.org',
+    project_urls={
+        'Documentation': 'https://github.com/sqlmapproject/sqlmap/wiki',
+        'Source': 'https://github.com/sqlmapproject/sqlmap/',
+        'Tracker': 'https://github.com/sqlmapproject/sqlmap/issues',
+    },
     download_url='https://github.com/sqlmapproject/sqlmap/archive/$VERSION.zip',
     license='GNU General Public License v2 (GPLv2)',
     packages=find_packages(),
@@ -174,4 +179,4 @@ sed -i "s/^TYPE =.*/TYPE = \"$TYPE\"/g" sqlmap/lib/core/settings.py
 sed -i "s/.*lib\/core\/settings\.py/`md5sum sqlmap/lib/core/settings.py | cut -d ' ' -f 1`  lib\/core\/settings\.py/g" sqlmap/txt/checksum.md5
 for file in $(find sqlmap -type f | grep -v -E "\.(git|yml)"); do echo include $file >> MANIFEST.in; done
 python setup.py sdist upload
-rm -rf $TMP_DIR
+rm -rf $TMP_DIR

extra/shutils/regressiontest.py

@@ -3,6 +3,8 @@
 # Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 # See the file 'LICENSE' for copying permission
 
+from __future__ import print_function
+
 import codecs
 import inspect
 import os
@@ -56,8 +58,8 @@ def send_email(msg):
         s.sendmail(FROM, TO, msg.as_string())
         s.quit()
     # Catch all for SMTP exceptions
-    except smtplib.SMTPException, e:
-        print "Failure to send email: %s" % str(e)
+    except smtplib.SMTPException as ex:
+        print("Failure to send email: '%s" % ex)
 
 def failure_email(msg):
     msg = prepare_email(msg)
@@ -157,7 +159,7 @@ if __name__ == "__main__":
 
     try:
         main()
-    except Exception, e:
+    except Exception:
         log_fd.write("An exception has occurred:\n%s" % str(traceback.format_exc()))
 
     log_fd.write("Regression test finished at %s\n\n" % time.strftime("%H:%M:%S %d-%m-%Y", time.gmtime()))

extra/sqlharvest/sqlharvest.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import cookielib
 import re
 import socket
@@ -75,8 +77,8 @@ def main():
             except KeyboardInterrupt:
                 raise
 
-            except Exception, msg:
-                print msg
+            except Exception as ex:
+                print(ex)
 
             if abort:
                 break
@@ -86,7 +88,7 @@ def main():
             sys.stdout.write("---------------\n")
 
             for sqlfile in files:
-                print sqlfile
+                print(sqlfile)
 
                 try:
                     req = urllib2.Request(sqlfile)
@@ -118,8 +120,8 @@ def main():
                 except KeyboardInterrupt:
                     raise
 
-                except Exception, msg:
-                    print msg
+                except Exception as ex:
+                    print(ex)
 
             else:
                 i += 1

extra/wafdetectify/wafdetectify.py

@@ -5,21 +5,28 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import cookielib
 import glob
 import httplib
 import inspect
 import os
 import re
+import socket
+import ssl
 import subprocess
 import sys
 import urllib2
 
 sys.dont_write_bytecode = True
 
+if hasattr(ssl, "_create_unverified_context"):
+    ssl._create_default_https_context = ssl._create_unverified_context
+
 NAME, VERSION, AUTHOR = "WAF Detectify", "0.1", "sqlmap developers (@sqlmap)"
 TIMEOUT = 10
-HEADERS = {"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate", "Cache-Control": "max-age=0"}
+HEADERS = {"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "identity", "Cache-Control": "max-age=0"}
 SQLMAP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
 SCRIPTS_DIR = os.path.join(SQLMAP_DIR, "waf")
 LEVEL_COLORS = {"o": "\033[00;94m", "x": "\033[00;91m", "!": "\033[00;93m", "i": "\033[00;92m"}
@@ -43,7 +50,7 @@ def get_page(get=None, url=None, host=None, data=None):
         conn = urllib2.urlopen(req, timeout=TIMEOUT)
         page = conn.read()
         headers = conn.info()
-    except Exception, ex:
+    except Exception as ex:
         code = getattr(ex, "code", None)
         page = ex.read() if hasattr(ex, "read") else getattr(ex, "msg", "")
         headers = ex.info() if hasattr(ex, "info") else {}
@@ -63,7 +70,7 @@ def colorize(message):
 def main():
     global WAF_FUNCTIONS
 
-    print colorize("%s #v%s\n by: %s\n" % (NAME, VERSION, AUTHOR))
+    print(colorize("%s #v%s\n by: %s\n" % (NAME, VERSION, AUTHOR)))
 
     if len(sys.argv) < 2:
         exit(colorize("[x] usage: python %s <hostname>" % os.path.split(__file__)[-1]))
@@ -88,8 +95,8 @@ def main():
             if filename[:-3] in sys.modules:
                 del sys.modules[filename[:-3]]
             module = __import__(filename[:-3].encode(sys.getfilesystemencoding() or "utf8"))
-        except ImportError, msg:
-            exit(colorize("[x] cannot import WAF script '%s' (%s)" % (filename[:-3], msg)))
+        except ImportError as ex:
+            exit(colorize("[x] cannot import WAF script '%s' (%s)" % (filename[:-3], ex)))
 
         _ = dict(inspect.getmembers(module))
         if "detect" not in _:
@@ -99,7 +106,14 @@ def main():
 
     WAF_FUNCTIONS = sorted(WAF_FUNCTIONS, key=lambda _: "generic" in _[1].lower())
 
-    print colorize("[i] checking '%s'..." % sys.argv[1])
+    print(colorize("[i] checking '%s'..." % sys.argv[1]))
+
+    hostname = sys.argv[1].split("//")[-1].split('/')[0]
+    try:
+        socket.getaddrinfo(hostname, None)
+    except socket.gaierror:
+        print(colorize("[x] host '%s' does not exist" % hostname))
+        exit(1)
 
     found = False
     for function, product in WAF_FUNCTIONS:
@@ -107,13 +121,12 @@ def main():
             continue
 
         if function(get_page):
-            print colorize("[!] WAF/IPS identified as '%s'" % product)
-            found = True
+            exit(colorize("[!] WAF/IPS identified as '%s'" % product))
 
     if not found:
-        print colorize("[o] nothing found")
+        print(colorize("[o] nothing found"))
 
-    print
+    print()
 
     exit(int(not found))
 

lib/controller/action.py

@@ -75,7 +75,7 @@ def action():
     if conf.getPasswordHashes:
         try:
             conf.dumper.userSettings("database management system users password hashes", conf.dbmsHandler.getPasswordHashes(), "password hash", CONTENT_TYPE.PASSWORDS)
-        except SqlmapNoneDataException, ex:
+        except SqlmapNoneDataException as ex:
             logger.critical(ex)
         except:
             raise
@@ -83,7 +83,7 @@ def action():
     if conf.getPrivileges:
         try:
             conf.dumper.userSettings("database management system users privileges", conf.dbmsHandler.getPrivileges(), "privilege", CONTENT_TYPE.PRIVILEGES)
-        except SqlmapNoneDataException, ex:
+        except SqlmapNoneDataException as ex:
             logger.critical(ex)
         except:
             raise
@@ -91,40 +91,90 @@ def action():
     if conf.getRoles:
         try:
             conf.dumper.userSettings("database management system users roles", conf.dbmsHandler.getRoles(), "role", CONTENT_TYPE.ROLES)
-        except SqlmapNoneDataException, ex:
+        except SqlmapNoneDataException as ex:
             logger.critical(ex)
         except:
             raise
 
     if conf.getDbs:
-        conf.dumper.dbs(conf.dbmsHandler.getDbs())
+        try:
+            conf.dumper.dbs(conf.dbmsHandler.getDbs())
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.getTables:
-        conf.dumper.dbTables(conf.dbmsHandler.getTables())
+        try:
+            conf.dumper.dbTables(conf.dbmsHandler.getTables())
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.commonTables:
-        conf.dumper.dbTables(tableExists(paths.COMMON_TABLES))
+        try:
+            conf.dumper.dbTables(tableExists(paths.COMMON_TABLES))
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.getSchema:
-        conf.dumper.dbTableColumns(conf.dbmsHandler.getSchema(), CONTENT_TYPE.SCHEMA)
+        try:
+            conf.dumper.dbTableColumns(conf.dbmsHandler.getSchema(), CONTENT_TYPE.SCHEMA)
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.getColumns:
-        conf.dumper.dbTableColumns(conf.dbmsHandler.getColumns(), CONTENT_TYPE.COLUMNS)
+        try:
+            conf.dumper.dbTableColumns(conf.dbmsHandler.getColumns(), CONTENT_TYPE.COLUMNS)
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.getCount:
-        conf.dumper.dbTablesCount(conf.dbmsHandler.getCount())
+        try:
+            conf.dumper.dbTablesCount(conf.dbmsHandler.getCount())
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.commonColumns:
-        conf.dumper.dbTableColumns(columnExists(paths.COMMON_COLUMNS))
+        try:
+            conf.dumper.dbTableColumns(columnExists(paths.COMMON_COLUMNS))
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.dumpTable:
-        conf.dbmsHandler.dumpTable()
+        try:
+            conf.dbmsHandler.dumpTable()
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.dumpAll:
-        conf.dbmsHandler.dumpAll()
+        try:
+            conf.dbmsHandler.dumpAll()
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.search:
-        conf.dbmsHandler.search()
+        try:
+            conf.dbmsHandler.search()
+        except SqlmapNoneDataException as ex:
+            logger.critical(ex)
+        except:
+            raise
 
     if conf.query:
         conf.dumper.query(conf.query, conf.dbmsHandler.sqlQuery(conf.query))

lib/controller/checks.py

@@ -86,7 +86,7 @@ from lib.core.settings import DUMMY_NON_SQLI_CHECK_APPENDIX
 from lib.core.settings import FI_ERROR_REGEX
 from lib.core.settings import FORMAT_EXCEPTION_STRINGS
 from lib.core.settings import HEURISTIC_CHECK_ALPHABET
-from lib.core.settings import IDS_WAF_CHECK_PAYLOAD
+from lib.core.settings import IPS_WAF_CHECK_PAYLOAD
 from lib.core.settings import IDS_WAF_CHECK_RATIO
 from lib.core.settings import IDS_WAF_CHECK_TIMEOUT
 from lib.core.settings import MAX_DIFFLIB_SEQUENCE_LENGTH
@@ -426,11 +426,14 @@ def checkSqlInjection(place, parameter, value):
                     templatePayload = None
                     vector = None
 
+                    origValue = value
+                    if kb.customInjectionMark in origValue:
+                        origValue = origValue.split(kb.customInjectionMark)[0]
+                        origValue = re.search(r"(\w*)\Z", origValue).group(1)
+
                     # Threat the parameter original value according to the
                     # test's <where> tag
                     if where == PAYLOAD.WHERE.ORIGINAL or conf.prefix:
-                        origValue = value
-
                         if kb.tamperFunctions:
                             templatePayload = agent.payload(place, parameter, value="", newValue=origValue, where=where)
                     elif where == PAYLOAD.WHERE.NEGATIVE:
@@ -440,7 +443,7 @@ def checkSqlInjection(place, parameter, value):
 
                         if conf.invalidLogical:
                             _ = int(kb.data.randomInt[:2])
-                            origValue = "%s AND %s LIKE %s" % (value, _, _ + 1)
+                            origValue = "%s AND %s LIKE %s" % (origValue, _, _ + 1)
                         elif conf.invalidBignum:
                             origValue = kb.data.randomInt[:6]
                         elif conf.invalidString:
@@ -615,7 +618,7 @@ def checkSqlInjection(place, parameter, value):
                                 page, headers, _ = Request.queryPage(reqPayload, place, content=True, raise404=False)
                                 output = extractRegexResult(check, page, re.DOTALL | re.IGNORECASE)
                                 output = output or extractRegexResult(check, threadData.lastHTTPError[2] if wasLastResponseHTTPError() else None, re.DOTALL | re.IGNORECASE)
-                                output = output or extractRegexResult(check, listToStrValue((headers[key] for key in headers.keys() if key.lower() != URI_HTTP_HEADER.lower()) if headers else None), re.DOTALL | re.IGNORECASE)
+                                output = output or extractRegexResult(check, listToStrValue((headers[key] for key in headers if key.lower() != URI_HTTP_HEADER.lower()) if headers else None), re.DOTALL | re.IGNORECASE)
                                 output = output or extractRegexResult(check, threadData.lastRedirectMsg[1] if threadData.lastRedirectMsg and threadData.lastRedirectMsg[0] == threadData.lastRequestUID else None, re.DOTALL | re.IGNORECASE)
 
                                 if output:
@@ -627,10 +630,10 @@ def checkSqlInjection(place, parameter, value):
 
                                         injectable = True
 
-                            except SqlmapConnectionException, msg:
+                            except SqlmapConnectionException as ex:
                                 debugMsg = "problem occurred most likely because the "
                                 debugMsg += "server hasn't recovered as expected from the "
-                                debugMsg += "error-based payload used ('%s')" % msg
+                                debugMsg += "error-based payload used ('%s')" % getSafeExString(ex)
                                 logger.debug(debugMsg)
 
                         # In case of time-based blind or stacked queries
@@ -1350,7 +1353,7 @@ def checkWaf():
     logger.info(infoMsg)
 
     retVal = False
-    payload = "%d %s" % (randomInt(), IDS_WAF_CHECK_PAYLOAD)
+    payload = "%d %s" % (randomInt(), IPS_WAF_CHECK_PAYLOAD)
 
     if PLACE.URI in conf.parameters:
         place = PLACE.POST
@@ -1418,11 +1421,12 @@ def identifyWaf():
         page, headers, code = None, None, None
         try:
             pushValue(kb.redirectChoice)
-            kb.redirectChoice = REDIRECTION.NO
+            kb.redirectChoice = REDIRECTION.YES
             if kwargs.get("get"):
                 kwargs["get"] = urlencode(kwargs["get"])
             kwargs["raise404"] = False
             kwargs["silent"] = True
+            kwargs["finalCode"] = True
             page, headers, code = Request.getPage(*args, **kwargs)
         except Exception:
             pass
@@ -1439,7 +1443,7 @@ def identifyWaf():
         try:
             logger.debug("checking for WAF/IPS product '%s'" % product)
             found = function(_)
-        except Exception, ex:
+        except Exception as ex:
             errMsg = "exception occurred while running "
             errMsg += "WAF script for '%s' ('%s')" % (product, getSafeExString(ex))
             logger.critical(errMsg)
@@ -1542,11 +1546,11 @@ def checkConnection(suppressOutput=False):
             except socket.gaierror:
                 errMsg = "host '%s' does not exist" % conf.hostname
                 raise SqlmapConnectionException(errMsg)
-            except socket.error, ex:
+            except socket.error as ex:
                 errMsg = "problem occurred while "
                 errMsg += "resolving a host name '%s' ('%s')" % (conf.hostname, getSafeExString(ex))
                 raise SqlmapConnectionException(errMsg)
-            except UnicodeError, ex:
+            except UnicodeError as ex:
                 errMsg = "problem occurred while "
                 errMsg += "handling a host name '%s' ('%s')" % (conf.hostname, getSafeExString(ex))
                 raise SqlmapDataException(errMsg)
@@ -1588,9 +1592,9 @@ def checkConnection(suppressOutput=False):
                 conf.url = re.sub(r"https?://", "https://", conf.url)
                 match = re.search(r":(\d+)", threadData.lastRedirectURL[1])
                 port = match.group(1) if match else 443
-                conf.url = re.sub(r":\d+/", ":%s/" % port, conf.url)
+                conf.url = re.sub(r":\d+(/|\Z)", ":%s\g<1>" % port, conf.url)
 
-    except SqlmapConnectionException, ex:
+    except SqlmapConnectionException as ex:
         if conf.ipv6:
             warnMsg = "check connection to a provided "
             warnMsg += "IPv6 address with a tool like ping6 "

lib/controller/controller.py

@@ -90,7 +90,7 @@ def _selectInjection():
         if point not in points:
             points[point] = injection
         else:
-            for key in points[point].keys():
+            for key in points[point]:
                 if key != 'data':
                     points[point][key] = points[point][key] or injection[key]
             points[point]['data'].update(injection['data'])
@@ -244,7 +244,7 @@ def _saveToResultsFile():
         if key not in results:
             results[key] = []
 
-        results[key].extend(injection.data.keys())
+        results[key].extend(list(injection.data.keys()))
 
     try:
         for key, value in results.items():
@@ -257,7 +257,7 @@ def _saveToResultsFile():
             conf.resultsFP.write(line)
 
         conf.resultsFP.flush()
-    except IOError, ex:
+    except IOError as ex:
         errMsg = "unable to write to the results file '%s' ('%s'). " % (conf.resultsFilename, getSafeExString(ex))
         raise SqlmapSystemException(errMsg)
 
@@ -427,7 +427,7 @@ def start():
                     checkStability()
 
                 # Do a little prioritization reorder of a testable parameter list
-                parameters = conf.parameters.keys()
+                parameters = list(conf.parameters.keys())
 
                 # Order of testing list (first to last)
                 orderList = (PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER, PLACE.URI, PLACE.POST, PLACE.GET)
@@ -491,7 +491,7 @@ def start():
                         elif parameter in conf.testParameter:
                             pass
 
-                        elif parameter == conf.rParam:
+                        elif parameter in conf.rParam:
                             testSqlInj = False
 
                             infoMsg = "skipping randomizing %s parameter '%s'" % (paramType, parameter)
@@ -689,7 +689,7 @@ def start():
         except SqlmapSilentQuitException:
             raise
 
-        except SqlmapBaseException, ex:
+        except SqlmapBaseException as ex:
             errMsg = getSafeExString(ex)
 
             if conf.multipleTargets:

lib/controller/handler.py

@@ -105,13 +105,13 @@ def setHandler():
 
                     if sqlalchemy.connector:
                         conf.dbmsConnector = sqlalchemy
-                except Exception, ex:
+                except Exception as ex:
                     exception = ex
 
             if not dialect or exception:
                 try:
                     conf.dbmsConnector.connect()
-                except Exception, ex:
+                except Exception as ex:
                     if exception:
                         raise exception
                     else:

lib/core/agent.py

@@ -107,7 +107,7 @@ class Agent(object):
             else:
                 origValue = filter(None, (re.search(_, origValue.split(BOUNDED_INJECTION_MARKER)[0]) for _ in (r"\w+\Z", r"[^\"'><]+\Z", r"[^ ]+\Z")))[0].group(0)
             origValue = origValue[origValue.rfind('/') + 1:]
-            for char in ('?', '=', ':', ','):
+            for char in ('?', '=', ':', ',', '&'):
                 if char in origValue:
                     origValue = origValue[origValue.rfind(char) + 1:]
         elif place == PLACE.CUSTOM_POST:

lib/core/bigarray.py

@@ -87,7 +87,7 @@ class BigArray(list):
             try:
                 with open(self.chunks[-1], "rb") as f:
                     self.chunks[-1] = pickle.loads(bz2.decompress(f.read()))
-            except IOError, ex:
+            except IOError as ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex.message
                 raise SqlmapSystemException(errMsg)
@@ -109,7 +109,7 @@ class BigArray(list):
             with open(filename, "w+b") as f:
                 f.write(bz2.compress(pickle.dumps(chunk, pickle.HIGHEST_PROTOCOL), BIGARRAY_COMPRESS_LEVEL))
             return filename
-        except (OSError, IOError), ex:
+        except (OSError, IOError) as ex:
             errMsg = "exception occurred while storing data "
             errMsg += "to a temporary file ('%s'). Please " % ex.message
             errMsg += "make sure that there is enough disk space left. If problem persists, "
@@ -126,7 +126,7 @@ class BigArray(list):
             try:
                 with open(self.chunks[index], "rb") as f:
                     self.cache = Cache(index, pickle.loads(bz2.decompress(f.read())), False)
-            except Exception, ex:
+            except Exception as ex:
                 errMsg = "exception occurred while retrieving data "
                 errMsg += "from a temporary file ('%s')" % ex.message
                 raise SqlmapSystemException(errMsg)
@@ -148,7 +148,7 @@ class BigArray(list):
         if y < 0:
             y += len(self)
 
-        index = y / self.chunk_length
+        index = y // self.chunk_length
         offset = y % self.chunk_length
         chunk = self.chunks[index]
 
@@ -159,7 +159,7 @@ class BigArray(list):
             return self.cache.data[offset]
 
     def __setitem__(self, y, value):
-        index = y / self.chunk_length
+        index = y // self.chunk_length
         offset = y % self.chunk_length
         chunk = self.chunks[index]
 

lib/core/common.py

@@ -84,6 +84,7 @@ from lib.core.enums import PLACE
 from lib.core.enums import PAYLOAD
 from lib.core.enums import REFLECTIVE_COUNTER
 from lib.core.enums import SORT_ORDER
+from lib.core.exception import SqlmapBaseException
 from lib.core.exception import SqlmapDataException
 from lib.core.exception import SqlmapGenericException
 from lib.core.exception import SqlmapNoneDataException
@@ -146,6 +147,7 @@ from lib.core.settings import PRINTABLE_CHAR_REGEX
 from lib.core.settings import PROBLEMATIC_CUSTOM_INJECTION_PATTERNS
 from lib.core.settings import PUSH_VALUE_EXCEPTION_RETRY_COUNT
 from lib.core.settings import PYVERSION
+from lib.core.settings import RANDOMIZATION_TLDS
 from lib.core.settings import REFERER_ALIASES
 from lib.core.settings import REFLECTED_BORDER_REGEX
 from lib.core.settings import REFLECTED_MAX_REGEX_PARTS
@@ -688,7 +690,7 @@ def paramToDict(place, parameters=None):
                     debugMsg += "is not inside the %s" % place
                     logger.debug(debugMsg)
 
-        elif len(conf.testParameter) != len(testableParameters.keys()):
+        elif len(conf.testParameter) != len(testableParameters):
             for parameter in conf.testParameter:
                 if parameter not in testableParameters:
                     debugMsg = "provided parameter '%s' " % parameter
@@ -872,9 +874,9 @@ def boldifyMessage(message):
 
     return retVal
 
-def setColor(message, color=None, bold=False):
+def setColor(message, color=None, bold=False, level=None):
     retVal = message
-    level = extractRegexResult(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message) or kb.get("stickyLevel")
+    level = level or extractRegexResult(r"\[(?P<result>%s)\]" % '|'.join(_[0] for _ in getPublicTypeMembers(LOGGING_LEVELS)), message)
 
     if isinstance(level, unicode):
         level = unicodeencode(level)
@@ -885,7 +887,6 @@ def setColor(message, color=None, bold=False):
         elif level:
             level = getattr(logging, level, None) if isinstance(level, basestring) else level
             retVal = LOGGER_HANDLER.colorize(message, level)
-            kb.stickyLevel = level if message and message[-1] != "\n" else None
 
     return retVal
 
@@ -913,7 +914,8 @@ def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status=
 
     if not kb.get("threadException"):
         if forceOutput or not (getCurrentThreadData().disableStdOut or kb.get("wizardMode")):
-            if kb.get("multiThreadMode"):
+            multiThreadMode = isMultiThreadMode()
+            if multiThreadMode:
                 logging._acquireLock()
 
             if isinstance(data, unicode):
@@ -931,7 +933,7 @@ def dataToStdout(data, forceOutput=False, bold=False, content_type=None, status=
             except IOError:
                 pass
 
-            if kb.get("multiThreadMode"):
+            if multiThreadMode:
                 logging._releaseLock()
 
             kb.prependFlag = isinstance(data, basestring) and (len(data) == 1 and data not in ('\n', '\r') or len(data) > 2 and data[0] == '\r' and data[-1] != '\n')
@@ -943,7 +945,7 @@ def dataToTrafficFile(data):
     try:
         conf.trafficFP.write(data)
         conf.trafficFP.flush()
-    except IOError, ex:
+    except IOError as ex:
         errMsg = "something went wrong while trying "
         errMsg += "to write to the traffic file '%s' ('%s')" % (conf.trafficFile, getSafeExString(ex))
         raise SqlmapSystemException(errMsg)
@@ -952,7 +954,7 @@ def dataToDumpFile(dumpFile, data):
     try:
         dumpFile.write(data)
         dumpFile.flush()
-    except IOError, ex:
+    except IOError as ex:
         if "No space left" in getUnicode(ex):
             errMsg = "no space left on output device"
             logger.error(errMsg)
@@ -972,7 +974,7 @@ def dataToOutFile(filename, data):
             try:
                 with open(retVal, "w+b") as f:  # has to stay as non-codecs because data is raw ASCII encoded data
                     f.write(unicodeencode(data))
-            except UnicodeEncodeError, ex:
+            except UnicodeEncodeError as ex:
                 _ = normalizeUnicode(filename)
                 if filename != _:
                     filename = _
@@ -980,7 +982,7 @@ def dataToOutFile(filename, data):
                     errMsg = "couldn't write to the "
                     errMsg += "output file ('%s')" % getSafeExString(ex)
                     raise SqlmapGenericException(errMsg)
-            except IOError, ex:
+            except IOError as ex:
                 errMsg = "something went wrong while trying to write "
                 errMsg += "to the output file ('%s')" % getSafeExString(ex)
                 raise SqlmapGenericException(errMsg)
@@ -995,7 +997,6 @@ def readInput(message, default=None, checkBatch=True, boolean=False):
     """
 
     retVal = None
-    kb.stickyLevel = None
 
     message = getUnicode(message)
 
@@ -1269,14 +1270,22 @@ def setPaths(rootPath):
     paths.SQLMAP_XML_BANNER_PATH = os.path.join(paths.SQLMAP_XML_PATH, "banner")
     paths.SQLMAP_XML_PAYLOADS_PATH = os.path.join(paths.SQLMAP_XML_PATH, "payloads")
 
-    _ = os.path.join(os.path.expandvars(os.path.expanduser("~")), ".sqlmap")
-    paths.SQLMAP_HOME_PATH = _
-    paths.SQLMAP_OUTPUT_PATH = getUnicode(paths.get("SQLMAP_OUTPUT_PATH", os.path.join(_, "output")), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
+    if IS_WIN:
+        if os.getenv("LOCALAPPDATA"):
+            paths.SQLMAP_HOME_PATH = os.path.expandvars("%LOCALAPPDATA%\\sqlmap")
+        elif os.getenv("USERPROFILE"):
+            paths.SQLMAP_HOME_PATH = os.path.expandvars("%USERPROFILE%\\Local Settings\\sqlmap")
+        else:
+            paths.SQLMAP_HOME_PATH = os.path.join(os.path.expandvars(os.path.expanduser("~")), "sqlmap")
+    else:
+        paths.SQLMAP_HOME_PATH = os.path.join(os.path.expandvars(os.path.expanduser("~")), ".sqlmap")
+
+    paths.SQLMAP_OUTPUT_PATH = getUnicode(paths.get("SQLMAP_OUTPUT_PATH", os.path.join(paths.SQLMAP_HOME_PATH, "output")), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
     paths.SQLMAP_DUMP_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "dump")
     paths.SQLMAP_FILES_PATH = os.path.join(paths.SQLMAP_OUTPUT_PATH, "%s", "files")
 
     # history files
-    paths.SQLMAP_HISTORY_PATH = getUnicode(os.path.join(_, "history"), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
+    paths.SQLMAP_HISTORY_PATH = getUnicode(os.path.join(paths.SQLMAP_HOME_PATH, "history"), encoding=sys.getfilesystemencoding() or UNICODE_ENCODING)
     paths.API_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "api.hst")
     paths.OS_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "os.hst")
     paths.SQL_SHELL_HISTORY = os.path.join(paths.SQLMAP_HISTORY_PATH, "sql.hst")
@@ -1446,7 +1455,7 @@ def parseTargetUrl():
 
     try:
         urlSplit = urlparse.urlsplit(conf.url)
-    except ValueError, ex:
+    except ValueError as ex:
         errMsg = "invalid URL '%s' has been given ('%s'). " % (conf.url, getSafeExString(ex))
         errMsg += "Please be sure that you don't have any leftover characters (e.g. '[' or ']') "
         errMsg += "in the hostname part"
@@ -1560,7 +1569,7 @@ def expandAsteriskForColumns(expression):
         columnsDict = conf.dbmsHandler.getColumns(onlyColNames=True)
 
         if columnsDict and conf.db in columnsDict and conf.tbl in columnsDict[conf.db]:
-            columns = columnsDict[conf.db][conf.tbl].keys()
+            columns = list(columnsDict[conf.db][conf.tbl].keys())
             columns.sort()
             columnsStr = ", ".join(column for column in columns)
             expression = expression.replace('*', columnsStr, 1)
@@ -1982,6 +1991,13 @@ def isHexEncodedString(subject):
 
     return re.match(r"\A[0-9a-fA-Fx]+\Z", subject) is not None
 
+def isMultiThreadMode():
+    """
+    Checks if running in multi-thread(ing) mode
+    """
+
+    return threading.activeCount() > 1
+
 @cachedmethod
 def getConsoleWidth(default=80):
     """
@@ -2028,7 +2044,6 @@ def clearConsoleLine(forceOutput=False):
         dataToStdout("\r%s\r" % (" " * (getConsoleWidth() - 1)), forceOutput)
 
     kb.prependFlag = False
-    kb.stickyLevel = None
 
 def parseXmlFile(xmlFile, handler):
     """
@@ -2038,7 +2053,7 @@ def parseXmlFile(xmlFile, handler):
     try:
         with contextlib.closing(StringIO(readCachedFileContent(xmlFile))) as stream:
             parse(stream, handler)
-    except (SAXParseException, UnicodeError), ex:
+    except (SAXParseException, UnicodeError) as ex:
         errMsg = "something appears to be wrong with "
         errMsg += "the file '%s' ('%s'). Please make " % (xmlFile, getSafeExString(ex))
         errMsg += "sure that you haven't made any changes to it"
@@ -2064,7 +2079,7 @@ def getSQLSnippet(dbms, sfile, **variables):
     retVal = re.sub(r"#.+", "", retVal)
     retVal = re.sub(r";\s+", "; ", retVal).strip("\r\n")
 
-    for _ in variables.keys():
+    for _ in variables:
         retVal = re.sub(r"%%%s%%" % _, variables[_].replace('\\', r'\\'), retVal)
 
     for _ in re.findall(r"%RANDSTR\d+%", retVal, re.I):
@@ -2104,7 +2119,7 @@ def readCachedFileContent(filename, mode="rb"):
                 try:
                     with openFile(filename, mode) as f:
                         kb.cache.content[filename] = f.read()
-                except (IOError, OSError, MemoryError), ex:
+                except (IOError, OSError, MemoryError) as ex:
                     errMsg = "something went wrong while trying "
                     errMsg += "to read the content of file '%s' ('%s')" % (filename, getSafeExString(ex))
                     raise SqlmapSystemException(errMsg)
@@ -2129,7 +2144,7 @@ def average(values):
     0.9
     """
 
-    return (sum(values) / len(values)) if values else None
+    return (1.0 * sum(values) / len(values)) if values else None
 
 @cachedmethod
 def stdev(values):
@@ -2218,12 +2233,12 @@ def getFileItems(filename, commentPrefix='#', unicoded=True, lowercase=False, un
                         retVal[line] = True
                     else:
                         retVal.append(line)
-    except (IOError, OSError, MemoryError), ex:
+    except (IOError, OSError, MemoryError) as ex:
         errMsg = "something went wrong while trying "
         errMsg += "to read the content of file '%s' ('%s')" % (filename, getSafeExString(ex))
         raise SqlmapSystemException(errMsg)
 
-    return retVal if not unique else retVal.keys()
+    return retVal if not unique else list(retVal.keys())
 
 def goGoodSamaritan(prevValue, originalCharset):
     """
@@ -2351,7 +2366,7 @@ def getUnicode(value, encoding=None, noneToNull=False):
         while True:
             try:
                 return unicode(value, encoding or (kb.get("pageEncoding") if kb.get("originalPage") else None) or UNICODE_ENCODING)
-            except UnicodeDecodeError, ex:
+            except UnicodeDecodeError as ex:
                 try:
                     return unicode(value, UNICODE_ENCODING)
                 except:
@@ -2407,7 +2422,7 @@ def pushValue(value):
             getCurrentThreadData().valueStack.append(copy.deepcopy(value))
             success = True
             break
-        except Exception, ex:
+        except Exception as ex:
             _ = ex
 
     if not success:
@@ -2981,16 +2996,21 @@ def parseSqliteTableSchema(value):
     Parses table column names and types from specified SQLite table schema
     """
 
+    retVal = False
+
     if value:
         table = {}
         columns = {}
 
         for match in re.finditer(r"(\w+)[\"'`]?\s+(INT|INTEGER|TINYINT|SMALLINT|MEDIUMINT|BIGINT|UNSIGNED BIG INT|INT2|INT8|INTEGER|CHARACTER|VARCHAR|VARYING CHARACTER|NCHAR|NATIVE CHARACTER|NVARCHAR|TEXT|CLOB|LONGTEXT|BLOB|NONE|REAL|DOUBLE|DOUBLE PRECISION|FLOAT|REAL|NUMERIC|DECIMAL|BOOLEAN|DATE|DATETIME|NUMERIC)\b", value, re.I):
+            retVal = True
             columns[match.group(1)] = match.group(2)
 
-        table[conf.tbl] = columns
+        table[safeSQLIdentificatorNaming(conf.tbl, True)] = columns
         kb.data.cachedColumns[conf.db] = table
 
+    return retVal
+
 def getTechniqueData(technique=None):
     """
     Returns injection data for technique specified
@@ -3056,7 +3076,7 @@ def saveConfig(conf, filename):
     config = UnicodeRawConfigParser()
     userOpts = {}
 
-    for family in optDict.keys():
+    for family in optDict:
         userOpts[family] = []
 
     for option, value in conf.items():
@@ -3095,7 +3115,7 @@ def saveConfig(conf, filename):
     with openFile(filename, "wb") as f:
         try:
             config.write(f)
-        except IOError, ex:
+        except IOError as ex:
             errMsg = "something went wrong while trying "
             errMsg += "to write to the configuration file '%s' ('%s')" % (filename, getSafeExString(ex))
             raise SqlmapSystemException(errMsg)
@@ -3347,8 +3367,7 @@ def unhandledExceptionMessage():
     errMsg += "repository at '%s'. If the exception persists, please open a new issue " % GIT_PAGE
     errMsg += "at '%s' " % ISSUES_PAGE
     errMsg += "with the following text and any other information required to "
-    errMsg += "reproduce the bug. The "
-    errMsg += "developers will try to reproduce the bug, fix it accordingly "
+    errMsg += "reproduce the bug. Developers will try to reproduce the bug, fix it accordingly "
     errMsg += "and get back to you\n"
     errMsg += "Running version: %s\n" % VERSION_STRING[VERSION_STRING.find('/') + 1:]
     errMsg += "Python version: %s\n" % PYVERSION
@@ -3442,7 +3461,7 @@ def createGithubIssue(errMsg, excMsg):
 
         try:
             content = urllib2.urlopen(req).read()
-        except Exception, ex:
+        except Exception as ex:
             content = None
 
         issueUrl = re.search(r"https://github.com/sqlmapproject/sqlmap/issues/\d+", content or "")
@@ -3484,7 +3503,7 @@ def maskSensitiveData(msg):
         retVal = retVal.replace(match.group(3), '*' * len(match.group(3)))
 
     # Fail-safe substitution
-    retVal = re.sub(r"(?i)\bhttps?://[^ ]+", lambda match: '*' * len(match.group(0)), retVal)
+    retVal = re.sub(r"(?i)(Command line:.+)\b(https?://[^ ]+)", lambda match: "%s%s" % (match.group(1), '*' * len(match.group(2))), retVal)
 
     if getpass.getuser():
         retVal = re.sub(r"(?i)\b%s\b" % re.escape(getpass.getuser()), '*' * len(getpass.getuser()), retVal)
@@ -3531,6 +3550,32 @@ def intersect(containerA, containerB, lowerCase=False):
 
     return retVal
 
+def decodeStringEscape(value):
+    """
+    Decodes escaped string values (e.g. "\\t" -> "\t")
+    """
+
+    retVal = value
+
+    if value and '\\' in value:
+        if isinstance(value, unicode):
+            retVal = retVal.encode(UNICODE_ENCODING)
+
+        try:
+            retVal = codecs.escape_decode(retVal)[0]
+        except:
+            try:
+                retVal = retVal.decode("string_escape")
+            except:
+                charset = string.whitespace.replace(" ", "")
+                for _ in charset:
+                    retVal = retVal.replace(repr(_).strip("'"), _)
+
+        if isinstance(value, unicode):
+            retVal = getUnicode(retVal)
+
+    return retVal
+
 def removeReflectiveValues(content, payload, suppressWarning=False):
     """
     Neutralizes reflective values in a given content based on a payload
@@ -3555,7 +3600,7 @@ def removeReflectiveValues(content, payload, suppressWarning=False):
                     retVal = content.replace(payload, REFLECTED_VALUE_MARKER)  # dummy approach
 
                     if len(parts) > REFLECTED_MAX_REGEX_PARTS:  # preventing CPU hogs
-                        regex = _("%s%s%s" % (REFLECTED_REPLACEMENT_REGEX.join(parts[:REFLECTED_MAX_REGEX_PARTS / 2]), REFLECTED_REPLACEMENT_REGEX, REFLECTED_REPLACEMENT_REGEX.join(parts[-REFLECTED_MAX_REGEX_PARTS / 2:])))
+                        regex = _("%s%s%s" % (REFLECTED_REPLACEMENT_REGEX.join(parts[:REFLECTED_MAX_REGEX_PARTS // 2]), REFLECTED_REPLACEMENT_REGEX, REFLECTED_REPLACEMENT_REGEX.join(parts[-REFLECTED_MAX_REGEX_PARTS // 2:])))
 
                     parts = filter(None, regex.split(REFLECTED_REPLACEMENT_REGEX))
 
@@ -3795,7 +3840,7 @@ def expandMnemonics(mnemonics, parser, args):
                 logger.debug(debugMsg)
             else:
                 found = sorted(options.keys(), key=lambda x: len(x))[0]
-                warnMsg = "detected ambiguity (mnemonic '%s' can be resolved to any of: %s). " % (name, ", ".join("'%s'" % key for key in options.keys()))
+                warnMsg = "detected ambiguity (mnemonic '%s' can be resolved to any of: %s). " % (name, ", ".join("'%s'" % key for key in options))
                 warnMsg += "Resolved to shortest of those ('%s')" % found
                 logger.warn(warnMsg)
 
@@ -3897,6 +3942,14 @@ def randomizeParameterValue(value):
 
         retVal = retVal.replace(original, candidate)
 
+    if re.match(r"\A[^@]+@.+\.[a-z]+\Z", value):
+        parts = retVal.split('.')
+        parts[-1] = random.sample(RANDOMIZATION_TLDS, 1)[0]
+        retVal = '.'.join(parts)
+
+    if not retVal:
+        retVal = randomStr(lowercase=True)
+
     return retVal
 
 @cachedmethod
@@ -4063,7 +4116,7 @@ def findPageForms(content, url, raise_=False, addToTargets=False):
                     continue
 
                 request = form.click()
-            except (ValueError, TypeError), ex:
+            except (ValueError, TypeError) as ex:
                 errMsg = "there has been a problem while "
                 errMsg += "processing page forms ('%s')" % getSafeExString(ex)
                 if raise_:
@@ -4145,6 +4198,9 @@ def getHostHeader(url):
         elif any(retVal.endswith(':%d' % _) for _ in (80, 443)):
             retVal = retVal.split(':')[0]
 
+    if retVal and retVal.count(':') > 1 and not any(_ in retVal for _ in ('[', ']')):
+        retVal = "[%s]" % retVal
+
     return retVal
 
 def checkDeprecatedOptions(args):
@@ -4193,7 +4249,7 @@ def evaluateCode(code, variables=None):
         exec(code, variables)
     except KeyboardInterrupt:
         raise
-    except Exception, ex:
+    except Exception as ex:
         errMsg = "an error occurred while evaluating provided code ('%s') " % getSafeExString(ex)
         raise SqlmapGenericException(errMsg)
 
@@ -4409,9 +4465,9 @@ def resetCookieJar(cookieJar):
                 errMsg = "no valid cookies found"
                 raise SqlmapGenericException(errMsg)
 
-        except cookielib.LoadError, msg:
+        except cookielib.LoadError as ex:
             errMsg = "there was a problem loading "
-            errMsg += "cookies file ('%s')" % re.sub(r"(cookies) file '[^']+'", r"\g<1>", str(msg))
+            errMsg += "cookies file ('%s')" % re.sub(r"(cookies) file '[^']+'", r"\g<1>", getSafeExString(ex))
             raise SqlmapGenericException(errMsg)
 
 def decloakToTemp(filename):
@@ -4598,9 +4654,8 @@ def parseRequestFile(reqFile, checkParams=True):
             reqResList = re.finditer(BURP_REQUEST_REGEX, content, re.I | re.S)
 
         for match in reqResList:
-            request = match if isinstance(match, basestring) else match.group(0)
+            request = match if isinstance(match, basestring) else match.group(1)
             request = re.sub(r"\A[^\w]+", "", request)
-
             schemePort = re.search(r"(http[\w]*)\:\/\/.*?\:([\d]+).+?={10,}", request, re.I | re.S)
 
             if schemePort:
@@ -4716,7 +4771,7 @@ def parseRequestFile(reqFile, checkParams=True):
     try:
         with openFile(reqFile, "rb") as f:
             content = f.read()
-    except (IOError, OSError, MemoryError), ex:
+    except (IOError, OSError, MemoryError) as ex:
         errMsg = "something went wrong while trying "
         errMsg += "to read the content of file '%s' ('%s')" % (reqFile, getSafeExString(ex))
         raise SqlmapSystemException(errMsg)
@@ -4735,11 +4790,11 @@ def getSafeExString(ex, encoding=None):
     Safe way how to get the proper exception represtation as a string
     (Note: errors to be avoided: 1) "%s" % Exception(u'\u0161') and 2) "%s" % str(Exception(u'\u0161'))
 
-    >>> getSafeExString(Exception('foobar'))
+    >>> getSafeExString(SqlmapBaseException('foobar'))
     u'foobar'
     """
 
-    retVal = ex
+    retVal = None
 
     if getattr(ex, "message", None):
         retVal = ex.message
@@ -4747,6 +4802,13 @@ def getSafeExString(ex, encoding=None):
         retVal = ex.msg
     elif isinstance(ex, (list, tuple)) and len(ex) > 1 and isinstance(ex[1], basestring):
         retVal = ex[1]
+    elif isinstance(ex, (list, tuple)) and len(ex) > 0 and isinstance(ex[0], basestring):
+        retVal = ex[0]
+
+    if retVal is None:
+        retVal = str(ex)
+    elif not isinstance(ex, SqlmapBaseException):
+        retVal = "%s: %s" % (type(ex).__name__, retVal)
 
     return getUnicode(retVal or "", encoding=encoding).strip()
 

lib/core/datatype.py

@@ -8,6 +8,8 @@ See the file 'LICENSE' for copying permission
 import copy
 import types
 
+from thirdparty.odict.odict import OrderedDict
+
 class AttribDict(dict):
     """
     This class defines the sqlmap object, inheriting from Python data
@@ -104,3 +106,37 @@ class InjectionDict(AttribDict):
         self.dbms = None
         self.dbms_version = None
         self.os = None
+
+# Reference: https://www.kunxi.org/2014/05/lru-cache-in-python
+class LRUDict(object):
+    def __init__(self, capacity):
+        self.capacity = capacity
+        self.cache = OrderedDict()
+
+    def __len__(self):
+        return len(self.cache)
+
+    def __contains__(self, key):
+        return key in self.cache
+
+    def __getitem__(self, key):
+        value = self.cache.pop(key)
+        self.cache[key] = value
+        return value
+
+    def get(self, key):
+        return self.__getitem__(key)
+
+    def __setitem__(self, key, value):
+        try:
+            self.cache.pop(key)
+        except KeyError:
+            if len(self.cache) >= self.capacity:
+                self.cache.popitem(last=False)
+        self.cache[key] = value
+
+    def set(self, key, value):
+        self.__setitem__(key, value)
+
+    def keys(self):
+        return self.cache.keys()

lib/core/decorators.py

@@ -7,10 +7,15 @@ See the file 'LICENSE' for copying permission
 
 import functools
 import hashlib
+import threading
 
+from lib.core.settings import MAX_CACHE_ITEMS
+from lib.core.datatype import LRUDict
 from lib.core.threads import getCurrentThreadData
 
-def cachedmethod(f, cache={}):
+_lock = threading.Lock()
+
+def cachedmethod(f, cache=LRUDict(capacity=MAX_CACHE_ITEMS)):
     """
     Method with a cached content
 
@@ -20,10 +25,17 @@ def cachedmethod(f, cache={}):
     @functools.wraps(f)
     def _(*args, **kwargs):
         key = int(hashlib.md5("|".join(str(_) for _ in (f, args, kwargs))).hexdigest(), 16) & 0x7fffffffffffffff
-        if key not in cache:
-            cache[key] = f(*args, **kwargs)
 
-        return cache[key]
+        try:
+            with _lock:
+                result = cache[key]
+        except KeyError:
+            result = f(*args, **kwargs)
+
+            with _lock:
+                cache[key] = result
+
+        return result
 
     return _
 

lib/core/dump.py

@@ -20,6 +20,7 @@ from lib.core.common import dataToStdout
 from lib.core.common import getSafeExString
 from lib.core.common import getUnicode
 from lib.core.common import isListLike
+from lib.core.common import isMultiThreadMode
 from lib.core.common import normalizeUnicode
 from lib.core.common import openFile
 from lib.core.common import prioritySortColumns
@@ -74,16 +75,17 @@ class Dump(object):
         if console:
             dataToStdout(text)
 
-        if kb.get("multiThreadMode"):
+        multiThreadMode = isMultiThreadMode()
+        if multiThreadMode:
             self._lock.acquire()
 
         try:
             self._outputFP.write(text)
-        except IOError, ex:
+        except IOError as ex:
             errMsg = "error occurred while writing to log file ('%s')" % getSafeExString(ex)
             raise SqlmapGenericException(errMsg)
 
-        if kb.get("multiThreadMode"):
+        if multiThreadMode:
             self._lock.release()
 
         kb.dataOutputFlag = True
@@ -99,7 +101,7 @@ class Dump(object):
         self._outputFile = os.path.join(conf.outputPath, "log")
         try:
             self._outputFP = openFile(self._outputFile, "ab" if not conf.flushSession else "wb")
-        except IOError, ex:
+        except IOError as ex:
             errMsg = "error occurred while opening log file ('%s')" % getSafeExString(ex)
             raise SqlmapGenericException(errMsg)
 
@@ -110,8 +112,6 @@ class Dump(object):
         self._write(data, content_type=content_type)
 
     def string(self, header, data, content_type=None, sort=True):
-        kb.stickyLevel = None
-
         if conf.api:
             self._write(data, content_type=content_type)
             return
@@ -133,7 +133,7 @@ class Dump(object):
             if "\n" in _:
                 self._write("%s:\n---\n%s\n---" % (header, _))
             else:
-                self._write("%s:    %s" % (header, ("'%s'" % _) if isinstance(data, basestring) else _))
+                self._write("%s: %s" % (header, ("'%s'" % _) if isinstance(data, basestring) else _))
         else:
             self._write("%s:\tNone" % header)
 
@@ -426,10 +426,10 @@ class Dump(object):
                     if not os.path.isdir(dumpDbPath):
                         try:
                             os.makedirs(dumpDbPath)
-                        except Exception, ex:
+                        except Exception as ex:
                             try:
                                 tempDir = tempfile.mkdtemp(prefix="sqlmapdb")
-                            except IOError, _:
+                            except IOError as _:
                                 errMsg = "unable to write to the temporary directory ('%s'). " % _
                                 errMsg += "Please make sure that your disk is not full and "
                                 errMsg += "that you have sufficient write permissions to "
@@ -624,8 +624,8 @@ class Dump(object):
                                 with open(filepath, "wb") as f:
                                     _ = safechardecode(value, True)
                                     f.write(_)
-                        except magic.MagicException, err:
-                            logger.debug(str(err))
+                        except magic.MagicException as ex:
+                            logger.debug(getSafeExString(ex))
 
                     if conf.dumpFormat == DUMP_FORMAT.CSV:
                         if field == fields:

lib/core/enums.py

@@ -156,13 +156,17 @@ class HASH:
 
 # Reference: http://www.zytrax.com/tech/web/mobile_ids.html
 class MOBILES:
-    BLACKBERRY = ("BlackBerry 9900", "Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+")
-    GALAXY = ("Samsung Galaxy S", "Mozilla/5.0 (Linux; U; Android 2.2; en-US; SGH-T959D Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1")
+    BLACKBERRY = ("BlackBerry Z10", "Mozilla/5.0 (BB10; Kbd) AppleWebKit/537.35+ (KHTML, like Gecko) Version/10.3.3.2205 Mobile Safari/537.35+")
+    GALAXY = ("Samsung Galaxy S7", "Mozilla/5.0 (Linux; Android 7.0; SM-G930V Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36")
     HP = ("HP iPAQ 6365", "Mozilla/4.0 (compatible; MSIE 4.01; Windows CE; PPC; 240x320; HP iPAQ h6300)")
-    HTC = ("HTC Sensation", "Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30")
-    IPHONE = ("Apple iPhone 4s", "Mozilla/5.0 (iPhone; CPU iPhone OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B179 Safari/7534.48.3")
+    HTC = ("HTC 10", "Mozilla/5.0 (Linux; Android 8.0.0; HTC 10 Build/OPR1.170623.027) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36")
+    HUAWEI = ("Huawei P8", "Mozilla/5.0 (Linux; Android 4.4.4; HUAWEI H891L Build/HuaweiH891L) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36")
+    IPHONE = ("Apple iPhone 8", "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1")
+    LUMIA = ("Microsoft Lumia 950", "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.14977")
     NEXUS = ("Google Nexus 7", "Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19")
     NOKIA = ("Nokia N97", "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/10.0.012; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) WicKed/7.1.12344")
+    PIXEL = ("Google Pixel", "Mozilla/5.0 (Linux; Android 8.0.0; Pixel Build/OPR3.170623.013) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.111 Mobile Safari/537.36")
+    XIAOMI = ("Xiaomi Mi 3", "Mozilla/5.0 (Linux; U; Android 4.4.4; en-gb; MI 3W Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36 XiaoMi/MiuiBrowser/2.1.1")
 
 class PROXY_TYPE:
     HTTP = "HTTP"
@@ -381,4 +385,4 @@ class TIMEOUT_STATE:
 
 class HINT:
     PREPEND = 0
-    APPEND = 1
+    APPEND = 1

lib/core/option.py

@@ -13,7 +13,6 @@ import os
 import random
 import re
 import socket
-import string
 import sys
 import tempfile
 import threading
@@ -33,13 +32,16 @@ from lib.core.common import Backend
 from lib.core.common import boldifyMessage
 from lib.core.common import checkFile
 from lib.core.common import dataToStdout
+from lib.core.common import decodeStringEscape
 from lib.core.common import getPublicTypeMembers
 from lib.core.common import getSafeExString
+from lib.core.common import getUnicode
 from lib.core.common import findLocalPort
 from lib.core.common import findPageForms
 from lib.core.common import getConsoleWidth
 from lib.core.common import getFileItems
 from lib.core.common import getFileType
+from lib.core.common import intersect
 from lib.core.common import normalizePath
 from lib.core.common import ntToPosixSlashes
 from lib.core.common import openFile
@@ -100,6 +102,7 @@ from lib.core.optiondict import optDict
 from lib.core.settings import CODECS_LIST_PAGE
 from lib.core.settings import CUSTOM_INJECTION_MARK_CHAR
 from lib.core.settings import DBMS_ALIASES
+from lib.core.settings import DEFAULT_GET_POST_DELIMITER
 from lib.core.settings import DEFAULT_PAGE_ENCODING
 from lib.core.settings import DEFAULT_TOR_HTTP_PORTS
 from lib.core.settings import DEFAULT_TOR_SOCKS_PORTS
@@ -193,7 +196,7 @@ def _loadQueries():
     tree = ElementTree()
     try:
         tree.parse(paths.QUERIES_XML)
-    except Exception, ex:
+    except Exception as ex:
         errMsg = "something appears to be wrong with "
         errMsg += "the file '%s' ('%s'). Please make " % (paths.QUERIES_XML, getSafeExString(ex))
         errMsg += "sure that you haven't made any changes to it"
@@ -223,10 +226,11 @@ def _setMultipleTargets():
 
     if os.path.isfile(conf.logFile):
         for target in parseRequestFile(conf.logFile):
-            url = target[0]
-            if url not in seen:
+            url, _, data, _, _ = target
+            key = re.sub(r"(\w+=)[^%s ]*" % (conf.paramDel or DEFAULT_GET_POST_DELIMITER), r"\g<1>", "%s %s" % (url, data))
+            if key not in seen:
                 kb.targets.add(target)
-                seen.add(url)
+                seen.add(key)
 
     elif os.path.isdir(conf.logFile):
         files = os.listdir(conf.logFile)
@@ -237,10 +241,11 @@ def _setMultipleTargets():
                 continue
 
             for target in parseRequestFile(os.path.join(conf.logFile, reqFile)):
-                url = target[0]
-                if url not in seen:
+                url, _, data, _, _ = target
+                key = re.sub(r"(\w+=)[^%s ]*" % (conf.paramDel or DEFAULT_GET_POST_DELIMITER), r"\g<1>", "%s %s" % (url, data))
+                if key not in seen:
                     kb.targets.add(target)
-                    seen.add(url)
+                    seen.add(key)
 
     else:
         errMsg = "the specified list of targets is not a file "
@@ -310,7 +315,7 @@ def _setRequestFromFile():
         infoMsg = "parsing second-order HTTP request from '%s'" % conf.secondReq
         logger.info(infoMsg)
 
-        target = parseRequestFile(conf.secondReq, False).next()
+        target = next(parseRequestFile(conf.secondReq, False))
         kb.secondReq = target
 
 def _setCrawler():
@@ -332,7 +337,7 @@ def _setCrawler():
                 if conf.verbose in (1, 2):
                     status = "%d/%d links visited (%d%%)" % (i + 1, len(targets), round(100.0 * (i + 1) / len(targets)))
                     dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status), True)
-            except Exception, ex:
+            except Exception as ex:
                 errMsg = "problem occurred while crawling at '%s' ('%s')" % (target, getSafeExString(ex))
                 logger.error(errMsg)
 
@@ -468,7 +473,7 @@ def _findPageForms():
                     dataToStdout("\r[%s] [INFO] %s" % (time.strftime("%X"), status), True)
             except KeyboardInterrupt:
                 break
-            except Exception, ex:
+            except Exception as ex:
                 errMsg = "problem occurred while searching for forms at '%s' ('%s')" % (target, getSafeExString(ex))
                 logger.error(errMsg)
 
@@ -543,11 +548,11 @@ def _setMetasploit():
 
     if conf.msfPath:
         for path in (conf.msfPath, os.path.join(conf.msfPath, "bin")):
-            if any(os.path.exists(normalizePath(os.path.join(path, _))) for _ in ("msfcli", "msfconsole")):
+            if any(os.path.exists(normalizePath(os.path.join(path, "%s%s" % (_, ".bat" if IS_WIN else "")))) for _ in ("msfcli", "msfconsole")):
                 msfEnvPathExists = True
-                if all(os.path.exists(normalizePath(os.path.join(path, _))) for _ in ("msfvenom",)):
+                if all(os.path.exists(normalizePath(os.path.join(path, "%s%s" % (_, ".bat" if IS_WIN else "")))) for _ in ("msfvenom",)):
                     kb.oldMsf = False
-                elif all(os.path.exists(normalizePath(os.path.join(path, _))) for _ in ("msfencode", "msfpayload")):
+                elif all(os.path.exists(normalizePath(os.path.join(path, "%s%s" % (_, ".bat" if IS_WIN else "")))) for _ in ("msfencode", "msfpayload")):
                     kb.oldMsf = True
                 else:
                     msfEnvPathExists = False
@@ -582,11 +587,11 @@ def _setMetasploit():
         for envPath in envPaths:
             envPath = envPath.replace(";", "")
 
-            if any(os.path.exists(normalizePath(os.path.join(envPath, _))) for _ in ("msfcli", "msfconsole")):
+            if any(os.path.exists(normalizePath(os.path.join(envPath, "%s%s" % (_, ".bat" if IS_WIN else "")))) for _ in ("msfcli", "msfconsole")):
                 msfEnvPathExists = True
-                if all(os.path.exists(normalizePath(os.path.join(envPath, _))) for _ in ("msfvenom",)):
+                if all(os.path.exists(normalizePath(os.path.join(envPath, "%s%s" % (_, ".bat" if IS_WIN else "")))) for _ in ("msfvenom",)):
                     kb.oldMsf = False
-                elif all(os.path.exists(normalizePath(os.path.join(envPath, _))) for _ in ("msfencode", "msfpayload")):
+                elif all(os.path.exists(normalizePath(os.path.join(envPath, "%s%s" % (_, ".bat" if IS_WIN else "")))) for _ in ("msfencode", "msfpayload")):
                     kb.oldMsf = True
                 else:
                     msfEnvPathExists = False
@@ -765,7 +770,7 @@ def _setTamperingFunctions():
 
             try:
                 module = __import__(filename[:-3].encode(sys.getfilesystemencoding() or UNICODE_ENCODING))
-            except Exception, ex:
+            except Exception as ex:
                 raise SqlmapSyntaxException("cannot import tamper module '%s' (%s)" % (filename[:-3], getSafeExString(ex)))
 
             priority = PRIORITY.NORMAL if not hasattr(module, "__priority__") else module.__priority__
@@ -798,7 +803,7 @@ def _setTamperingFunctions():
                 elif name == "dependencies":
                     try:
                         function()
-                    except Exception, ex:
+                    except Exception as ex:
                         errMsg = "error occurred while checking dependencies "
                         errMsg += "for tamper module '%s' ('%s')" % (filename[:-3], getSafeExString(ex))
                         raise SqlmapGenericException(errMsg)
@@ -843,8 +848,8 @@ def _setWafFunctions():
                 if filename[:-3] in sys.modules:
                     del sys.modules[filename[:-3]]
                 module = __import__(filename[:-3].encode(sys.getfilesystemencoding() or UNICODE_ENCODING))
-            except ImportError, msg:
-                raise SqlmapSyntaxException("cannot import WAF script '%s' (%s)" % (filename[:-3], msg))
+            except ImportError as ex:
+                raise SqlmapSyntaxException("cannot import WAF script '%s' (%s)" % (filename[:-3], getSafeExString(ex)))
 
             _ = dict(inspect.getmembers(module))
             if "detect" not in _:
@@ -971,7 +976,7 @@ def _setHTTPHandlers():
 
         try:
             _ = urlparse.urlsplit(conf.proxy)
-        except Exception, ex:
+        except Exception as ex:
             errMsg = "invalid proxy address '%s' ('%s')" % (conf.proxy, getSafeExString(ex))
             raise SqlmapSyntaxException(errMsg)
 
@@ -1192,7 +1197,7 @@ def _setHTTPAuthentication():
         elif authType == AUTH_TYPE.NTLM:
             regExp = "^(.*\\\\.*):(.*?)$"
             errMsg = "HTTP NTLM authentication credentials value must "
-            errMsg += "be in format 'DOMAIN\username:password'"
+            errMsg += "be in format 'DOMAIN\\username:password'"
         elif authType == AUTH_TYPE.PKI:
             errMsg = "HTTP PKI authentication require "
             errMsg += "usage of option `--auth-pki`"
@@ -1273,28 +1278,32 @@ def _setHTTPUserAgent():
           file choosed as user option
     """
 
+    debugMsg = "setting the HTTP User-Agent header"
+    logger.debug(debugMsg)
+
     if conf.mobile:
-        message = "which smartphone do you want sqlmap to imitate "
-        message += "through HTTP User-Agent header?\n"
-        items = sorted(getPublicTypeMembers(MOBILES, True))
+        if conf.randomAgent:
+            _ = random.sample([_[1] for _ in getPublicTypeMembers(MOBILES, True)], 1)[0]
+            conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, _))
+        else:
+            message = "which smartphone do you want sqlmap to imitate "
+            message += "through HTTP User-Agent header?\n"
+            items = sorted(getPublicTypeMembers(MOBILES, True))
 
-        for count in xrange(len(items)):
-            item = items[count]
-            message += "[%d] %s%s\n" % (count + 1, item[0], " (default)" if item == MOBILES.IPHONE else "")
+            for count in xrange(len(items)):
+                item = items[count]
+                message += "[%d] %s%s\n" % (count + 1, item[0], " (default)" if item == MOBILES.IPHONE else "")
 
-        test = readInput(message.rstrip('\n'), default=items.index(MOBILES.IPHONE) + 1)
+            test = readInput(message.rstrip('\n'), default=items.index(MOBILES.IPHONE) + 1)
 
-        try:
-            item = items[int(test) - 1]
-        except:
-            item = MOBILES.IPHONE
+            try:
+                item = items[int(test) - 1]
+            except:
+                item = MOBILES.IPHONE
 
-        conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, item[1]))
+            conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, item[1]))
 
     elif conf.agent:
-        debugMsg = "setting the HTTP User-Agent header"
-        logger.debug(debugMsg)
-
         conf.httpHeaders.append((HTTP_HEADER.USER_AGENT, conf.agent))
 
     elif not conf.randomAgent:
@@ -1373,7 +1382,7 @@ def _setHostname():
     if conf.url:
         try:
             conf.hostname = urlparse.urlsplit(conf.url).netloc.split(':')[0]
-        except ValueError, ex:
+        except ValueError as ex:
             errMsg = "problem occurred while "
             errMsg += "parsing an URL '%s' ('%s')" % (conf.url, getSafeExString(ex))
             raise SqlmapDataException(errMsg)
@@ -1408,6 +1417,41 @@ def _checkDependencies():
     if conf.dependencies:
         checkDependencies()
 
+def _createHomeDirectories():
+    """
+    Creates directories inside sqlmap's home directory
+    """
+
+    for context in "output", "history":
+        directory = paths["SQLMAP_%s_PATH" % context.upper()]
+        try:
+            if not os.path.isdir(directory):
+                os.makedirs(directory)
+
+            _ = os.path.join(directory, randomStr())
+            open(_, "w+b").close()
+            os.remove(_)
+
+            if conf.outputDir and context == "output":
+                warnMsg = "using '%s' as the %s directory" % (directory, context)
+                logger.warn(warnMsg)
+        except (OSError, IOError) as ex:
+            try:
+                tempDir = tempfile.mkdtemp(prefix="sqlmap%s" % context)
+            except Exception as _:
+                errMsg = "unable to write to the temporary directory ('%s'). " % _
+                errMsg += "Please make sure that your disk is not full and "
+                errMsg += "that you have sufficient write permissions to "
+                errMsg += "create temporary files and/or directories"
+                raise SqlmapSystemException(errMsg)
+
+            warnMsg = "unable to %s %s directory " % ("create" if not os.path.isdir(directory) else "write to the", context)
+            warnMsg += "'%s' (%s). " % (directory, getUnicode(ex))
+            warnMsg += "Using temporary directory '%s' instead" % getUnicode(tempDir)
+            logger.warn(warnMsg)
+
+            paths["SQLMAP_%s_PATH" % context.upper()] = tempDir
+
 def _createTemporaryDirectory():
     """
     Creates temporary directory for this run.
@@ -1427,7 +1471,7 @@ def _createTemporaryDirectory():
 
             warnMsg = "using '%s' as the temporary directory" % conf.tmpDir
             logger.warn(warnMsg)
-        except (OSError, IOError), ex:
+        except (OSError, IOError) as ex:
             errMsg = "there has been a problem while accessing "
             errMsg += "temporary directory location(s) ('%s')" % getSafeExString(ex)
             raise SqlmapSystemException(errMsg)
@@ -1435,7 +1479,7 @@ def _createTemporaryDirectory():
         try:
             if not os.path.isdir(tempfile.gettempdir()):
                 os.makedirs(tempfile.gettempdir())
-        except Exception, ex:
+        except Exception as ex:
             warnMsg = "there has been a problem while accessing "
             warnMsg += "system's temporary directory location(s) ('%s'). Please " % getSafeExString(ex)
             warnMsg += "make sure that there is enough disk space left. If problem persists, "
@@ -1454,7 +1498,7 @@ def _createTemporaryDirectory():
     if not os.path.isdir(tempfile.tempdir):
         try:
             os.makedirs(tempfile.tempdir)
-        except Exception, ex:
+        except Exception as ex:
             errMsg = "there has been a problem while setting "
             errMsg += "temporary directory location ('%s')" % getSafeExString(ex)
             raise SqlmapSystemException(errMsg)
@@ -1497,11 +1541,8 @@ def _cleanupOptions():
     else:
         conf.rParam = []
 
-    if conf.paramDel and '\\' in conf.paramDel:
-        try:
-            conf.paramDel = conf.paramDel.decode("string_escape")
-        except ValueError:
-            pass
+    if conf.paramDel:
+        conf.paramDel = decodeStringEscape(conf.paramDel)
 
     if conf.skip:
         conf.skip = conf.skip.replace(" ", "")
@@ -1613,7 +1654,7 @@ def _cleanupOptions():
         conf.code = int(conf.code)
 
     if conf.csvDel:
-        conf.csvDel = conf.csvDel.decode("string_escape")  # e.g. '\\t' -> '\t'
+        conf.csvDel = decodeStringEscape(conf.csvDel)
 
     if conf.torPort and isinstance(conf.torPort, basestring) and conf.torPort.isdigit():
         conf.torPort = int(conf.torPort)
@@ -1626,18 +1667,13 @@ def _cleanupOptions():
         setPaths(paths.SQLMAP_ROOT_PATH)
 
     if conf.string:
-        try:
-            conf.string = conf.string.decode("unicode_escape")
-        except:
-            charset = string.whitespace.replace(" ", "")
-            for _ in charset:
-                conf.string = conf.string.replace(_.encode("string_escape"), _)
+        conf.string = decodeStringEscape(conf.string)
 
     if conf.getAll:
         map(lambda _: conf.__setitem__(_, True), WIZARD.ALL)
 
     if conf.noCast:
-        for _ in DUMP_REPLACEMENTS.keys():
+        for _ in list(DUMP_REPLACEMENTS.keys()):
             del DUMP_REPLACEMENTS[_]
 
     if conf.dumpFormat:
@@ -1826,7 +1862,6 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.matchRatio = None
     kb.maxConnectionsFlag = False
     kb.mergeCookies = None
-    kb.multiThreadMode = False
     kb.negativeLogic = False
     kb.nullConnection = None
     kb.oldMsf = None
@@ -1879,7 +1914,6 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.reduceTests = None
     kb.tlsSNI = {}
     kb.stickyDBMS = False
-    kb.stickyLevel = None
     kb.storeCrawlingChoice = None
     kb.storeHashesChoice = None
     kb.suppressResumeInfo = False
@@ -2133,9 +2167,9 @@ def _setDNSServer():
         try:
             conf.dnsServer = DNSServer()
             conf.dnsServer.run()
-        except socket.error, msg:
+        except socket.error as ex:
             errMsg = "there was an error while setting up "
-            errMsg += "DNS server instance ('%s')" % msg
+            errMsg += "DNS server instance ('%s')" % getSafeExString(ex)
             raise SqlmapGenericException(errMsg)
     else:
         errMsg = "you need to run sqlmap as an administrator "
@@ -2303,7 +2337,7 @@ def _basicOptionValidation():
         errMsg = "option '--not-string' is incompatible with switch '--null-connection'"
         raise SqlmapSyntaxException(errMsg)
 
-    if conf.notString and conf.nullConnection:
+    if conf.tor and conf.osPwn:
         errMsg = "option '--tor' is incompatible with switch '--os-pwn'"
         raise SqlmapSyntaxException(errMsg)
 
@@ -2326,14 +2360,14 @@ def _basicOptionValidation():
     if conf.regexp:
         try:
             re.compile(conf.regexp)
-        except Exception, ex:
+        except Exception as ex:
             errMsg = "invalid regular expression '%s' ('%s')" % (conf.regexp, getSafeExString(ex))
             raise SqlmapSyntaxException(errMsg)
 
     if conf.crawlExclude:
         try:
             re.compile(conf.crawlExclude)
-        except Exception, ex:
+        except Exception as ex:
             errMsg = "invalid regular expression '%s' ('%s')" % (conf.crawlExclude, getSafeExString(ex))
             raise SqlmapSyntaxException(errMsg)
 
@@ -2422,8 +2456,14 @@ def _basicOptionValidation():
         raise SqlmapSyntaxException(errMsg)
 
     if conf.skip and conf.testParameter:
-        errMsg = "option '--skip' is incompatible with option '-p'"
-        raise SqlmapSyntaxException(errMsg)
+        if intersect(conf.skip, conf.testParameter):
+            errMsg = "option '--skip' is incompatible with option '-p'"
+            raise SqlmapSyntaxException(errMsg)
+
+    if conf.rParam and conf.testParameter:
+        if intersect(conf.rParam, conf.testParameter):
+            errMsg = "option '--randomize' is incompatible with option '-p'"
+            raise SqlmapSyntaxException(errMsg)
 
     if conf.mobile and conf.agent:
         errMsg = "switch '--mobile' is incompatible with option '--user-agent'"
@@ -2499,6 +2539,7 @@ def init():
     _cleanupEnvironment()
     _purge()
     _checkDependencies()
+    _createHomeDirectories()
     _createTemporaryDirectory()
     _basicOptionValidation()
     _setProxyList()

lib/core/optiondict.py

@@ -211,6 +211,7 @@ optDict = {
         "hexConvert": "boolean",
         "outputDir": "string",
         "parseErrors": "boolean",
+        "repair": "boolean",
         "saveConfig": "string",
         "scope": "string",
         "testFilter": "string",

lib/core/profiling.py

@@ -9,7 +9,7 @@ import codecs
 import os
 import cProfile
 
-from lib.core.common import getUnicode
+from lib.core.common import getSafeExString
 from lib.core.data import logger
 from lib.core.data import paths
 from lib.core.settings import UNICODE_ENCODING
@@ -25,8 +25,8 @@ def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None):
         from thirdparty.xdot import xdot
         import gtk
         import pydot
-    except ImportError, e:
-        errMsg = "profiling requires third-party libraries ('%s') " % getUnicode(e, UNICODE_ENCODING)
+    except ImportError as ex:
+        errMsg = "profiling requires third-party libraries ('%s') " % getSafeExString(ex)
         errMsg += "(Hint: 'sudo apt-get install python-pydot python-pyparsing python-profiler graphviz')"
         logger.error(errMsg)
 

lib/core/readlineng.py

@@ -56,9 +56,7 @@ if PLATFORM == 'mac' and _readline:
 # http://mail.python.org/pipermail/python-dev/2003-August/037845.html
 # has the original discussion.
 if _readline:
-    try:
-        _readline.clear_history()
-    except AttributeError:
+    if not hasattr(_readline, "clear_history"):
         def clear_history():
             pass
 

lib/core/replication.py

@@ -27,7 +27,7 @@ class Replication(object):
             self.connection = sqlite3.connect(dbpath)
             self.connection.isolation_level = None
             self.cursor = self.connection.cursor()
-        except sqlite3.OperationalError, ex:
+        except sqlite3.OperationalError as ex:
             errMsg = "error occurred while opening a replication "
             errMsg += "file '%s' ('%s')" % (self.filepath, getSafeExString(ex))
             raise SqlmapConnectionException(errMsg)
@@ -63,7 +63,7 @@ class Replication(object):
                         self.execute('CREATE TABLE "%s" (%s)' % (self.name, ','.join('"%s" %s' % (unsafeSQLIdentificatorNaming(colname), coltype) for colname, coltype in self.columns)))
                     else:
                         self.execute('CREATE TABLE "%s" (%s)' % (self.name, ','.join('"%s"' % unsafeSQLIdentificatorNaming(colname) for colname in self.columns)))
-                except Exception, ex:
+                except Exception as ex:
                     errMsg = "problem occurred ('%s') while initializing the sqlite database " % getSafeExString(ex, UNICODE_ENCODING)
                     errMsg += "located at '%s'" % self.parent.dbpath
                     raise SqlmapGenericException(errMsg)
@@ -82,7 +82,7 @@ class Replication(object):
         def execute(self, sql, parameters=[]):
             try:
                 self.parent.cursor.execute(sql, parameters)
-            except sqlite3.OperationalError, ex:
+            except sqlite3.OperationalError as ex:
                 errMsg = "problem occurred ('%s') while accessing sqlite database " % getSafeExString(ex, UNICODE_ENCODING)
                 errMsg += "located at '%s'. Please make sure that " % self.parent.dbpath
                 errMsg += "it's not used by some other program"

lib/core/settings.py

@@ -19,7 +19,7 @@ from lib.core.enums import DBMS_DIRECTORY_NAME
 from lib.core.enums import OS
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.3"
+VERSION = "1.3.3.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -39,7 +39,7 @@ BANNER = """\033[01;33m\
  ___ ___[.]_____ ___ ___  \033[01;37m{\033[01;%dm%s\033[01;37m}\033[01;33m
 |_ -| . [.]     | .'| . |
 |___|_  [.]_|_|_|__,|  _|
-      |_|V          |_|   \033[0m\033[4;37m%s\033[0m\n
+      |_|V...       |_|   \033[0m\033[4;37m%s\033[0m\n
 """ % (TYPE_COLORS.get(TYPE, 31), VERSION_STRING.split('/')[-1], SITE)
 
 # Minimum distance of ratio from kb.matchRatio to result in True
@@ -109,7 +109,7 @@ MAX_MURPHY_SLEEP_TIME = 3
 GOOGLE_REGEX = r"webcache\.googleusercontent\.com/search\?q=cache:[^:]+:([^+]+)\+&amp;cd=|url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)"
 
 # Regular expression used for extracting results from DuckDuckGo search
-DUCKDUCKGO_REGEX = r'"u":"([^"]+)'
+DUCKDUCKGO_REGEX = r'<a class="result__url" href="(htt[^"]+)'
 
 # Regular expression used for extracting results from Bing search
 BING_REGEX = r'<h2><a href="([^"]+)" h='
@@ -166,6 +166,9 @@ MAX_TECHNIQUES_PER_VALUE = 2
 # In case of missing piece of partial union dump, buffered array must be flushed after certain size
 MAX_BUFFERED_PARTIAL_UNION_LENGTH = 1024
 
+# Maximum size of cache used in @cachedmethod decorator
+MAX_CACHE_ITEMS = 256
+
 # Suffix used for naming meta databases in DBMS(es) without explicit database name
 METADB_SUFFIX = "_masterdb"
 
@@ -182,7 +185,7 @@ MAX_TIME_RESPONSES = 200
 MIN_UNION_RESPONSES = 5
 
 # After these number of blanks at the end inference should stop (just in case)
-INFERENCE_BLANK_BREAK = 10
+INFERENCE_BLANK_BREAK = 5
 
 # Use this replacement character for cases when inference is not able to retrieve the proper character value
 INFERENCE_UNKNOWN_CHAR = '?'
@@ -231,7 +234,7 @@ PYVERSION = sys.version.split()[0]
 
 # DBMS system databases
 MSSQL_SYSTEM_DBS = ("Northwind", "master", "model", "msdb", "pubs", "tempdb")
-MYSQL_SYSTEM_DBS = ("information_schema", "mysql", "performance_schema")
+MYSQL_SYSTEM_DBS = ("information_schema", "mysql", "performance_schema", "sys")
 PGSQL_SYSTEM_DBS = ("information_schema", "pg_catalog", "pg_toast", "pgagent")
 ORACLE_SYSTEM_DBS = ('ANONYMOUS', 'APEX_030200', 'APEX_PUBLIC_USER', 'APPQOSSYS', 'BI', 'CTXSYS', 'DBSNMP', 'DIP', 'EXFSYS', 'FLOWS_%', 'FLOWS_FILES', 'HR', 'IX', 'LBACSYS', 'MDDATA', 'MDSYS', 'MGMT_VIEW', 'OC', 'OE', 'OLAPSYS', 'ORACLE_OCM', 'ORDDATA', 'ORDPLUGINS', 'ORDSYS', 'OUTLN', 'OWBSYS', 'PM', 'SCOTT', 'SH', 'SI_INFORMTN_SCHEMA', 'SPATIAL_CSW_ADMIN_USR', 'SPATIAL_WFS_ADMIN_USR', 'SYS', 'SYSMAN', 'SYSTEM', 'WKPROXY', 'WKSYS', 'WK_TEST', 'WMSYS', 'XDB', 'XS$NULL')
 SQLITE_SYSTEM_DBS = ("sqlite_master", "sqlite_temp_master")
@@ -327,7 +330,7 @@ CURRENT_DB = "CD"
 SESSION_SQLITE_FILE = "session.sqlite"
 
 # Regular expressions used for finding file paths in error messages
-FILE_PATH_REGEXES = (r"<b>(?P<result>[^<>]+?)</b> on line \d+", r"in (?P<result>[^<>'\"]+?)['\"]? on line \d+", r"(?:[>(\[\s])(?P<result>[A-Za-z]:[\\/][\w. \\/-]*)", r"(?:[>(\[\s])(?P<result>/\w[/\w.~-]+)", r"href=['\"]file://(?P<result>/[^'\"]+)")
+FILE_PATH_REGEXES = (r"<b>(?P<result>[^<>]+?)</b> on line \d+", r"\bin (?P<result>[^<>'\"]+?)['\"]? on line \d+", r"(?:[>(\[\s])(?P<result>[A-Za-z]:[\\/][\w. \\/-]*)", r"(?:[>(\[\s])(?P<result>/\w[/\w.~-]+)", r"\bhref=['\"]file://(?P<result>/[^'\"]+)", r"\bin <b>(?P<result>[^<]+): line \d+")
 
 # Regular expressions used for parsing error messages (--parse-errors)
 ERROR_PARSING_REGEXES = (
@@ -360,7 +363,7 @@ COMMON_PASSWORD_SUFFIXES += ("!", ".", "*", "!!", "?", ";", "..", "!!!", ", ", "
 WEBSCARAB_SPLITTER = "### Conversation"
 
 # Splitter used between requests in BURP log files
-BURP_REQUEST_REGEX = r"={10,}\s+[^=]+={10,}\s(.+?)\s={10,}"
+BURP_REQUEST_REGEX = r"={10,}\s+([A-Z]{3,} .+?)\s+={10,}"
 
 # Regex used for parsing XML Burp saved history items
 BURP_XML_HISTORY_REGEX = r'<port>(\d+)</port>.+?<request base64="true"><!\[CDATA\[([^]]+)'
@@ -526,9 +529,6 @@ BRUTE_TABLE_EXISTS_TEMPLATE = "EXISTS(SELECT %d FROM %s)"
 # Template used for common column existence check
 BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT %s FROM %s)"
 
-# Payload used for checking of existence of IDS/IPS/WAF (dummier the better)
-IDS_WAF_CHECK_PAYLOAD = "AND 1=1 UNION ALL SELECT 1,NULL,'<script>alert(\"XSS\")</script>',table_name FROM information_schema.tables WHERE 2>1--/**/; EXEC xp_cmdshell('cat ../../../etc/passwd')#"
-
 # Data inside shellcodeexec to be filled with random string
 SHELLCODEEXEC_RANDOM_STRING_MARKER = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
 
@@ -538,13 +538,16 @@ CHECK_INTERNET_ADDRESS = "https://ipinfo.io/"
 # Value to look for in response to CHECK_INTERNET_ADDRESS
 CHECK_INTERNET_VALUE = "IP Address Details"
 
+# Payload used for checking of existence of WAF/IPS (dummier the better)
+IPS_WAF_CHECK_PAYLOAD = "AND 1=1 UNION ALL SELECT 1,NULL,'<script>alert(\"XSS\")</script>',table_name FROM information_schema.tables WHERE 2>1--/**/; EXEC xp_cmdshell('cat ../../../etc/passwd')#"
+
 # Vectors used for provoking specific WAF/IPS behavior(s)
 WAF_ATTACK_VECTORS = (
     "",  # NIL
     "search=<script>alert(1)</script>",
     "file=../../../../etc/passwd",
     "q=<invalid>foobar",
-    "id=1 %s" % IDS_WAF_CHECK_PAYLOAD
+    "id=1 %s" % IPS_WAF_CHECK_PAYLOAD
 )
 
 # Used for status representation in dictionary attack phase
@@ -677,6 +680,9 @@ CHECK_ZERO_COLUMNS_THRESHOLD = 10
 # Boldify all logger messages containing these "patterns"
 BOLD_PATTERNS = ("' injectable", "provided empty", "leftover chars", "might be injectable", "' is vulnerable", "is not injectable", "does not seem to be", "test failed", "test passed", "live test final result", "test shows that", "the back-end DBMS is", "created Github", "blocked by the target server", "protection is involved", "CAPTCHA", "specific response", "NULL connection is supported", "PASSED", "FAILED")
 
+# TLDs used in randomization of email-alike parameter values
+RANDOMIZATION_TLDS = ("com", "net", "ru", "org", "de", "jp", "cn", "fr", "it", "pl", "tv", "edu", "in", "ir", "es", "me", "info", "gr", "gov", "ca", "co", "se", "cz", "to", "vn", "nl", "cc", "az", "hu", "ua", "be", "no", "biz", "io", "ch", "ro", "sk", "eu", "us", "tw", "pt", "fi", "at", "lt", "kz", "cl", "hr", "pk", "lv", "la", "pe")
+
 # Generic www root directory names
 GENERIC_DOC_ROOT_DIRECTORY_NAMES = ("htdocs", "httpdocs", "public", "wwwroot", "www")
 
@@ -687,7 +693,7 @@ MAX_HELP_OPTION_LENGTH = 18
 MAX_CONNECT_RETRIES = 100
 
 # Strings for detecting formatting errors
-FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException")
+FORMAT_EXCEPTION_STRINGS = ("Type mismatch", "Error converting", "Please enter a", "Conversion failed", "String or binary data would be truncated", "Failed to convert", "unable to interpret text value", "Input string was not in a correct format", "System.FormatException", "java.lang.NumberFormatException", "ValueError: invalid literal", "TypeMismatchException", "CF_SQL_INTEGER", "CF_SQL_NUMERIC", " for CFSQLTYPE ", "cfqueryparam cfsqltype", "InvalidParamTypeException", "Invalid parameter type", "Attribute validation error for tag", "is not of type numeric", "<cfif Not IsNumeric(", "invalid input syntax for integer", "invalid input syntax for type", "invalid number", "character to number conversion error", "unable to interpret text value", "String was not recognized as a valid", "Convert.ToInt", "cannot be converted to a ", "InvalidDataException")
 
 # Regular expression used for extracting ASP.NET view state values
 VIEWSTATE_REGEX = r'(?i)(?P<name>__VIEWSTATE[^"]*)[^>]+value="(?P<result>[^"]+)'
@@ -770,6 +776,9 @@ BRUTE_DOC_ROOT_PREFIXES = {
     OS.WINDOWS: ("/xampp", "/Program Files/xampp", "/wamp", "/Program Files/wampp", "/apache", "/Program Files/Apache Group/Apache", "/Program Files/Apache Group/Apache2", "/Program Files/Apache Group/Apache2.2", "/Program Files/Apache Group/Apache2.4", "/Inetpub/wwwroot", "/Inetpub/wwwroot/%TARGET%", "/Inetpub/vhosts/%TARGET%")
 }
 
+# Table prefix to use in "takeover" functionalities (i.e. auxiliary tables used by sqlmap at the vulnerable DBMS)
+TAKEOVER_TABLE_PREFIX = "sqlmap"
+
 # Suffixes used in brute force search for web server document root
 BRUTE_DOC_ROOT_SUFFIXES = ("", "html", "htdocs", "httpdocs", "php", "public", "src", "site", "build", "web", "www", "data", "sites/all", "www/build")
 
@@ -806,3 +815,11 @@ th{
     font-size:12px;
 }
 </style>"""
+
+# Leaving (dirty) possibility to change values from here (e.g. `export SQLMAP__MAX_NUMBER_OF_THREADS=20`)
+
+for key, value in os.environ.items():
+    if key.upper().startswith("%s_" % SQLMAP_ENVIRONMENT_PREFIX):
+        _ = key[len(SQLMAP_ENVIRONMENT_PREFIX) + 1:].upper()
+        if _ in globals():
+            globals()[_] = value

lib/core/shell.py

@@ -9,6 +9,7 @@ import atexit
 import os
 
 from lib.core import readlineng as readline
+from lib.core.common import getSafeExString
 from lib.core.data import logger
 from lib.core.data import paths
 from lib.core.enums import AUTOCOMPLETE_TYPE
@@ -75,8 +76,8 @@ def saveHistory(completion=None):
         readline.set_history_length(MAX_HISTORY_LENGTH)
         try:
             readline.write_history_file(historyPath)
-        except IOError, msg:
-            warnMsg = "there was a problem writing the history file '%s' (%s)" % (historyPath, msg)
+        except IOError as ex:
+            warnMsg = "there was a problem writing the history file '%s' (%s)" % (historyPath, getSafeExString(ex))
             logger.warn(warnMsg)
     except KeyboardInterrupt:
         pass
@@ -99,8 +100,8 @@ def loadHistory(completion=None):
     if os.path.exists(historyPath):
         try:
             readline.read_history_file(historyPath)
-        except IOError, msg:
-            warnMsg = "there was a problem loading the history file '%s' (%s)" % (historyPath, msg)
+        except IOError as ex:
+            warnMsg = "there was a problem loading the history file '%s' (%s)" % (historyPath, getSafeExString(ex))
             logger.warn(warnMsg)
 
 def autoCompletion(completion=None, os=None, commands=None):

lib/core/subprocessng.py

@@ -31,7 +31,7 @@ def blockingReadFromFD(fd):
     while True:
         try:
             output += os.read(fd, 8192)
-        except (OSError, IOError), ioe:
+        except (OSError, IOError) as ioe:
             if ioe.args[0] in (errno.EAGAIN, errno.EINTR):
                 # Uncomment the following line if the process seems to
                 # take a huge amount of cpu time
@@ -52,7 +52,7 @@ def blockingWriteToFD(fd, data):
         try:
             data_length = len(data)
             wrote_data = os.write(fd, data)
-        except (OSError, IOError), io:
+        except (OSError, IOError) as io:
             if io.errno in (errno.EAGAIN, errno.EINTR):
                 continue
             else:
@@ -95,8 +95,8 @@ class Popen(subprocess.Popen):
                 (errCode, written) = WriteFile(x, input)
             except ValueError:
                 return self._close('stdin')
-            except (subprocess.pywintypes.error, Exception), why:
-                if why[0] in (109, errno.ESHUTDOWN):
+            except (subprocess.pywintypes.error, Exception) as ex:
+                if ex[0] in (109, errno.ESHUTDOWN):
                     return self._close('stdin')
                 raise
 
@@ -116,8 +116,8 @@ class Popen(subprocess.Popen):
                     (errCode, read) = ReadFile(x, nAvail, None)
             except (ValueError, NameError):
                 return self._close(which)
-            except (subprocess.pywintypes.error, Exception), why:
-                if why[0] in (109, errno.ESHUTDOWN):
+            except (subprocess.pywintypes.error, Exception) as ex:
+                if ex[0] in (109, errno.ESHUTDOWN):
                     return self._close(which)
                 raise
 
@@ -134,8 +134,8 @@ class Popen(subprocess.Popen):
 
             try:
                 written = os.write(self.stdin.fileno(), input)
-            except OSError, why:
-                if why[0] == errno.EPIPE:  # broken pipe
+            except OSError as ex:
+                if ex[0] == errno.EPIPE:  # broken pipe
                     return self._close('stdin')
                 raise
 

lib/core/target.py

@@ -427,8 +427,8 @@ def _setHashDB():
             try:
                 os.remove(conf.hashDBFile)
                 logger.info("flushing session file")
-            except OSError, msg:
-                errMsg = "unable to flush the session file (%s)" % msg
+            except OSError as ex:
+                errMsg = "unable to flush the session file ('%s')" % getSafeExString(ex)
                 raise SqlmapFilePathException(errMsg)
 
     conf.hashDB = HashDB(conf.hashDBFile)
@@ -558,7 +558,7 @@ def _setResultsFile():
         conf.resultsFilename = os.path.join(paths.SQLMAP_OUTPUT_PATH, time.strftime(RESULTS_FILE_FORMAT).lower())
         try:
             conf.resultsFP = openFile(conf.resultsFilename, "a", UNICODE_ENCODING, buffering=0)
-        except (OSError, IOError), ex:
+        except (OSError, IOError) as ex:
             try:
                 warnMsg = "unable to create results file '%s' ('%s'). " % (conf.resultsFilename, getUnicode(ex))
                 handle, conf.resultsFilename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.RESULTS, suffix=".csv")
@@ -566,7 +566,7 @@ def _setResultsFile():
                 conf.resultsFP = openFile(conf.resultsFilename, "w+", UNICODE_ENCODING, buffering=0)
                 warnMsg += "Using temporary file '%s' instead" % conf.resultsFilename
                 logger.warn(warnMsg)
-            except IOError, _:
+            except IOError as _:
                 errMsg = "unable to write to the temporary directory ('%s'). " % _
                 errMsg += "Please make sure that your disk is not full and "
                 errMsg += "that you have sufficient write permissions to "
@@ -590,7 +590,7 @@ def _createFilesDir():
     if not os.path.isdir(conf.filePath):
         try:
             os.makedirs(conf.filePath)
-        except OSError, ex:
+        except OSError as ex:
             tempDir = tempfile.mkdtemp(prefix="sqlmapfiles")
             warnMsg = "unable to create files directory "
             warnMsg += "'%s' (%s). " % (conf.filePath, getUnicode(ex))
@@ -612,7 +612,7 @@ def _createDumpDir():
     if not os.path.isdir(conf.dumpPath):
         try:
             os.makedirs(conf.dumpPath)
-        except OSError, ex:
+        except OSError as ex:
             tempDir = tempfile.mkdtemp(prefix="sqlmapdump")
             warnMsg = "unable to create dump directory "
             warnMsg += "'%s' (%s). " % (conf.dumpPath, getUnicode(ex))
@@ -630,45 +630,15 @@ def _createTargetDirs():
     Create the output directory.
     """
 
-    for context in "output", "history":
-        directory = paths["SQLMAP_%s_PATH" % context.upper()]
-        try:
-            if not os.path.isdir(directory):
-                os.makedirs(directory)
-
-            _ = os.path.join(directory, randomStr())
-            open(_, "w+b").close()
-            os.remove(_)
-
-            if conf.outputDir and context == "output":
-                warnMsg = "using '%s' as the %s directory" % (directory, context)
-                logger.warn(warnMsg)
-        except (OSError, IOError), ex:
-            try:
-                tempDir = tempfile.mkdtemp(prefix="sqlmap%s" % context)
-            except Exception, _:
-                errMsg = "unable to write to the temporary directory ('%s'). " % _
-                errMsg += "Please make sure that your disk is not full and "
-                errMsg += "that you have sufficient write permissions to "
-                errMsg += "create temporary files and/or directories"
-                raise SqlmapSystemException(errMsg)
-
-            warnMsg = "unable to %s %s directory " % ("create" if not os.path.isdir(directory) else "write to the", context)
-            warnMsg += "'%s' (%s). " % (directory, getUnicode(ex))
-            warnMsg += "Using temporary directory '%s' instead" % getUnicode(tempDir)
-            logger.warn(warnMsg)
-
-            paths["SQLMAP_%s_PATH" % context.upper()] = tempDir
-
     conf.outputPath = os.path.join(getUnicode(paths.SQLMAP_OUTPUT_PATH), normalizeUnicode(getUnicode(conf.hostname)))
 
     try:
         if not os.path.isdir(conf.outputPath):
             os.makedirs(conf.outputPath)
-    except (OSError, IOError, TypeError), ex:
+    except (OSError, IOError, TypeError) as ex:
         try:
             tempDir = tempfile.mkdtemp(prefix="sqlmapoutput")
-        except Exception, _:
+        except Exception as _:
             errMsg = "unable to write to the temporary directory ('%s'). " % _
             errMsg += "Please make sure that your disk is not full and "
             errMsg += "that you have sufficient write permissions to "
@@ -691,7 +661,7 @@ def _createTargetDirs():
             f.write("  # %s" % getUnicode(subprocess.list2cmdline(sys.argv), encoding=sys.stdin.encoding))
             if conf.data:
                 f.write("\n\n%s" % getUnicode(conf.data))
-    except IOError, ex:
+    except IOError as ex:
         if "denied" in getUnicode(ex):
             errMsg = "you don't have enough permissions "
         else:
@@ -767,4 +737,4 @@ def setupTargetEnv():
     _resumeHashDBValues()
     _setResultsFile()
     _setAuthCred()
-    _setAuxOptions()
+    _setAuxOptions()

lib/core/testing.py

@@ -54,42 +54,42 @@ def smokeTest():
 
     if not checkIntegrity():
         retVal = False
-    else:
-        for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
-            if any(_ in root for _ in ("thirdparty", "extra")):
-                continue
 
-            for filename in files:
-                if os.path.splitext(filename)[1].lower() == ".py" and filename != "__init__.py":
-                    length += 1
+    for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
+        if any(_ in root for _ in ("thirdparty", "extra")):
+            continue
 
-        for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
-            if any(_ in root for _ in ("thirdparty", "extra")):
-                continue
+        for filename in files:
+            if os.path.splitext(filename)[1].lower() == ".py" and filename != "__init__.py":
+                length += 1
 
-            for filename in files:
-                if os.path.splitext(filename)[1].lower() == ".py" and filename != "__init__.py":
-                    path = os.path.join(root, os.path.splitext(filename)[0])
-                    path = path.replace(paths.SQLMAP_ROOT_PATH, '.')
-                    path = path.replace(os.sep, '.').lstrip('.')
-                    try:
-                        __import__(path)
-                        module = sys.modules[path]
-                    except Exception, msg:
+    for root, _, files in os.walk(paths.SQLMAP_ROOT_PATH):
+        if any(_ in root for _ in ("thirdparty", "extra")):
+            continue
+
+        for filename in files:
+            if os.path.splitext(filename)[1].lower() == ".py" and filename != "__init__.py":
+                path = os.path.join(root, os.path.splitext(filename)[0])
+                path = path.replace(paths.SQLMAP_ROOT_PATH, '.')
+                path = path.replace(os.sep, '.').lstrip('.')
+                try:
+                    __import__(path)
+                    module = sys.modules[path]
+                except Exception as ex:
+                    retVal = False
+                    dataToStdout("\r")
+                    errMsg = "smoke test failed at importing module '%s' (%s):\n%s" % (path, os.path.join(root, filename), ex)
+                    logger.error(errMsg)
+                else:
+                    # Run doc tests
+                    # Reference: http://docs.python.org/library/doctest.html
+                    (failure_count, test_count) = doctest.testmod(module)
+                    if failure_count > 0:
                         retVal = False
-                        dataToStdout("\r")
-                        errMsg = "smoke test failed at importing module '%s' (%s):\n%s" % (path, os.path.join(root, filename), msg)
-                        logger.error(errMsg)
-                    else:
-                        # Run doc tests
-                        # Reference: http://docs.python.org/library/doctest.html
-                        (failure_count, test_count) = doctest.testmod(module)
-                        if failure_count > 0:
-                            retVal = False
 
-                    count += 1
-                    status = '%d/%d (%d%%) ' % (count, length, round(100.0 * count / length))
-                    dataToStdout("\r[%s] [INFO] complete: %s" % (time.strftime("%X"), status))
+                count += 1
+                status = '%d/%d (%d%%) ' % (count, length, round(100.0 * count / length))
+                dataToStdout("\r[%s] [INFO] complete: %s" % (time.strftime("%X"), status))
 
     clearConsoleLine()
     if retVal:
@@ -275,10 +275,10 @@ def runCase(parse):
         result = start()
     except KeyboardInterrupt:
         pass
-    except SqlmapBaseException, e:
-        handled_exception = e
-    except Exception, e:
-        unhandled_exception = e
+    except SqlmapBaseException as ex:
+        handled_exception = ex
+    except Exception as ex:
+        unhandled_exception = ex
     finally:
         sys.stdout.seek(0)
         console = sys.stdout.read()

lib/core/threads.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import difflib
 import random
 import threading
@@ -16,6 +18,7 @@ from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.datatype import AttribDict
 from lib.core.enums import PAYLOAD
+from lib.core.exception import SqlmapBaseException
 from lib.core.exception import SqlmapConnectionException
 from lib.core.exception import SqlmapThreadException
 from lib.core.exception import SqlmapUserQuitException
@@ -91,9 +94,10 @@ def exceptionHandledFunction(threadFunction, silent=False):
         kb.threadContinue = False
         kb.threadException = True
         raise
-    except Exception, ex:
+    except Exception as ex:
         if not silent and kb.get("threadContinue"):
-            logger.error("thread %s: %s" % (threading.currentThread().getName(), ex.message))
+            errMsg = ex.message if isinstance(ex, SqlmapBaseException) else "%s: %s" % (type(ex).__name__, ex.message)
+            logger.error("thread %s: '%s'" % (threading.currentThread().getName(), errMsg))
 
             if conf.get("verbose") > 1:
                 traceback.print_exc()
@@ -108,7 +112,6 @@ def setDaemon(thread):
 def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardException=True, threadChoice=False, startThreadMsg=True):
     threads = []
 
-    kb.multiThreadMode = True
     kb.threadContinue = True
     kb.threadException = False
 
@@ -150,7 +153,7 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
 
             try:
                 thread.start()
-            except Exception, ex:
+            except Exception as ex:
                 errMsg = "error occurred while starting new thread ('%s')" % ex.message
                 logger.critical(errMsg)
                 break
@@ -166,8 +169,8 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
                     alive = True
                     time.sleep(0.1)
 
-    except (KeyboardInterrupt, SqlmapUserQuitException), ex:
-        print
+    except (KeyboardInterrupt, SqlmapUserQuitException) as ex:
+        print()
         kb.prependFlag = False
         kb.threadContinue = False
         kb.threadException = True
@@ -184,8 +187,8 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
         if forwardException:
             raise
 
-    except (SqlmapConnectionException, SqlmapValueException), ex:
-        print
+    except (SqlmapConnectionException, SqlmapValueException) as ex:
+        print()
         kb.threadException = True
         logger.error("thread %s: %s" % (threading.currentThread().getName(), ex.message))
 
@@ -195,14 +198,13 @@ def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardExceptio
     except:
         from lib.core.common import unhandledExceptionMessage
 
-        print
+        print()
         kb.threadException = True
         errMsg = unhandledExceptionMessage()
         logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
         traceback.print_exc()
 
     finally:
-        kb.multiThreadMode = False
         kb.bruteMode = False
         kb.threadContinue = True
         kb.threadException = False

lib/core/update.py

@@ -51,7 +51,7 @@ def update():
 
             try:
                 open(os.path.join(directory, "sqlmap.py"), "w+b")
-            except Exception, ex:
+            except Exception as ex:
                 errMsg = "unable to update content of directory '%s' ('%s')" % (directory, getSafeExString(ex))
                 logger.error(errMsg)
             else:
@@ -85,7 +85,7 @@ def update():
                                 version = re.search(r"(?m)^VERSION\s*=\s*['\"]([^'\"]+)", f.read()).group(1)
                                 logger.info("updated to the latest version '%s#dev'" % version)
                                 success = True
-                    except Exception, ex:
+                    except Exception as ex:
                         logger.error("update could not be completed ('%s')" % getSafeExString(ex))
                     else:
                         if not success:
@@ -103,14 +103,14 @@ def update():
         debugMsg = "sqlmap will try to update itself using 'git' command"
         logger.debug(debugMsg)
 
-        dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
+        dataToStdout("\r[%s] [INFO] update in progress" % time.strftime("%X"))
 
         try:
             process = subprocess.Popen("git checkout . && git pull %s HEAD" % GIT_REPOSITORY, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=paths.SQLMAP_ROOT_PATH.encode(sys.getfilesystemencoding() or UNICODE_ENCODING))
             pollProcess(process, True)
             stdout, stderr = process.communicate()
             success = not process.returncode
-        except (IOError, OSError), ex:
+        except (IOError, OSError) as ex:
             success = False
             stderr = getSafeExString(ex)
 

lib/core/wordlist.py

@@ -43,7 +43,7 @@ class Wordlist(object):
             if os.path.splitext(self.current)[1].lower() == ".zip":
                 try:
                     _ = zipfile.ZipFile(self.current, 'r')
-                except zipfile.error, ex:
+                except zipfile.error as ex:
                     errMsg = "something appears to be wrong with "
                     errMsg += "the file '%s' ('%s'). Please make " % (self.current, getSafeExString(ex))
                     errMsg += "sure that you haven't made any changes to it"
@@ -68,15 +68,15 @@ class Wordlist(object):
         while True:
             self.counter += 1
             try:
-                retVal = self.iter.next().rstrip()
-            except zipfile.error, ex:
+                retVal = next(self.iter).rstrip()
+            except zipfile.error as ex:
                 errMsg = "something appears to be wrong with "
                 errMsg += "the file '%s' ('%s'). Please make " % (self.current, getSafeExString(ex))
                 errMsg += "sure that you haven't made any changes to it"
                 raise SqlmapInstallationException(errMsg)
             except StopIteration:
                 self.adjust()
-                retVal = self.iter.next().rstrip()
+                retVal = next(self.iter).rstrip()
             if not self.proc_count or self.counter % self.proc_count == self.proc_id:
                 break
         return retVal

lib/parse/cmdline.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import os
 import re
 import shlex
@@ -29,6 +31,7 @@ from lib.core.exception import SqlmapShellQuitException
 from lib.core.exception import SqlmapSyntaxException
 from lib.core.settings import BASIC_HELP_ITEMS
 from lib.core.settings import DUMMY_URL
+from lib.core.settings import INFERENCE_UNKNOWN_CHAR
 from lib.core.settings import IS_WIN
 from lib.core.settings import MAX_HELP_OPTION_LENGTH
 from lib.core.settings import VERSION_STRING
@@ -592,6 +595,9 @@ def cmdLineParser(argv=None):
         general.add_option("--parse-errors", dest="parseErrors", action="store_true",
                            help="Parse and display DBMS error messages from responses")
 
+        general.add_option("--repair", dest="repair", action="store_true",
+                           help="Redump entries having unknown character marker (%s)" % INFERENCE_UNKNOWN_CHAR)
+
         general.add_option("--save", dest="saveConfig",
                            help="Save options to a configuration INI file")
 
@@ -792,7 +798,7 @@ def cmdLineParser(argv=None):
                     command = raw_input("sqlmap-shell> ").strip()
                     command = getUnicode(command, encoding=sys.stdin.encoding)
                 except (KeyboardInterrupt, EOFError):
-                    print
+                    print()
                     raise SqlmapShellQuitException
 
                 if not command:
@@ -814,7 +820,7 @@ def cmdLineParser(argv=None):
             try:
                 for arg in shlex.split(command):
                     argv.append(getUnicode(arg, encoding=sys.stdin.encoding))
-            except ValueError, ex:
+            except ValueError as ex:
                 raise SqlmapSyntaxException("something went wrong during command line parsing ('%s')" % ex.message)
 
         for i in xrange(len(argv)):
@@ -842,7 +848,7 @@ def cmdLineParser(argv=None):
                 argv[i] = argv[i][:-1]
                 conf.skipThreadCheck = True
             elif argv[i] == "--version":
-                print VERSION_STRING.split('/')[-1]
+                print(VERSION_STRING.split('/')[-1])
                 raise SystemExit
             elif argv[i] in ("-h", "--help"):
                 advancedHelp = False
@@ -866,7 +872,7 @@ def cmdLineParser(argv=None):
 
         try:
             (args, _) = parser.parse_args(argv)
-        except UnicodeEncodeError, ex:
+        except UnicodeEncodeError as ex:
             dataToStdout("\n[!] %s\n" % ex.object.encode("unicode-escape"))
             raise SystemExit
         except SystemExit:
@@ -895,8 +901,8 @@ def cmdLineParser(argv=None):
 
         return args
 
-    except (OptionError, TypeError), e:
-        parser.error(e)
+    except (OptionError, TypeError) as ex:
+        parser.error(ex)
 
     except SystemExit:
         # Protection against Windows dummy double clicking

lib/parse/configfile.py

@@ -39,7 +39,7 @@ def configFileProxy(section, option, datatype):
                 value = config.getfloat(section, option) if config.get(section, option) else 0.0
             else:
                 value = config.get(section, option)
-        except ValueError, ex:
+        except ValueError as ex:
             errMsg = "error occurred while processing the option "
             errMsg += "'%s' in provided configuration file ('%s')" % (option, getUnicode(ex))
             raise SqlmapSyntaxException(errMsg)
@@ -71,7 +71,7 @@ def configFileParser(configFile):
     try:
         config = UnicodeRawConfigParser()
         config.readfp(configFP)
-    except Exception, ex:
+    except Exception as ex:
         errMsg = "you have provided an invalid and/or unreadable configuration file ('%s')" % getSafeExString(ex)
         raise SqlmapSyntaxException(errMsg)
 

lib/parse/handler.py

@@ -35,7 +35,7 @@ class FingerprintHandler(ContentHandler):
         if key == "dbmsVersion":
             self._info[key] = value
         else:
-            if key not in self._info.keys():
+            if key not in self._info:
                 self._info[key] = set()
 
             for _ in value.split("|"):

lib/parse/payloads.py

@@ -78,7 +78,7 @@ def parseXmlNode(node):
 def loadBoundaries():
     try:
         doc = et.parse(paths.BOUNDARIES_XML)
-    except Exception, ex:
+    except Exception as ex:
         errMsg = "something appears to be wrong with "
         errMsg += "the file '%s' ('%s'). Please make " % (paths.BOUNDARIES_XML, getSafeExString(ex))
         errMsg += "sure that you haven't made any changes to it"
@@ -93,7 +93,7 @@ def loadPayloads():
 
         try:
             doc = et.parse(payloadFilePath)
-        except Exception, ex:
+        except Exception as ex:
             errMsg = "something appears to be wrong with "
             errMsg += "the file '%s' ('%s'). Please make " % (payloadFilePath, getSafeExString(ex))
             errMsg += "sure that you haven't made any changes to it"

lib/request/basic.py

@@ -17,6 +17,7 @@ from lib.core.common import Backend
 from lib.core.common import extractErrorMessage
 from lib.core.common import extractRegexResult
 from lib.core.common import getPublicTypeMembers
+from lib.core.common import getSafeExString
 from lib.core.common import getUnicode
 from lib.core.common import isListLike
 from lib.core.common import randomStr
@@ -56,7 +57,7 @@ def forgeHeaders(items=None, base=None):
 
     items = items or {}
 
-    for _ in items.keys():
+    for _ in list(items.keys()):
         if items[_] is None:
             del items[_]
 
@@ -280,10 +281,10 @@ def decodePage(page, contentEncoding, contentType):
                     raise Exception("size too large")
 
             page = data.read()
-        except Exception, msg:
+        except Exception as ex:
             if "<html" not in page:  # in some cases, invalid "Content-Encoding" appears for plain HTML (should be ignored)
                 errMsg = "detected invalid data for declared content "
-                errMsg += "encoding '%s' ('%s')" % (contentEncoding, msg)
+                errMsg += "encoding '%s' ('%s')" % (contentEncoding, getSafeExString(ex))
                 singleTimeLogMessage(errMsg, logging.ERROR)
 
                 warnMsg = "turning off page compression"
@@ -312,43 +313,40 @@ def decodePage(page, contentEncoding, contentType):
 
     # can't do for all responses because we need to support binary files too
     if not isinstance(page, unicode) and "text/" in contentType:
-        if kb.heuristicMode:
-            kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
-            page = getUnicode(page, kb.pageEncoding)
-        else:
-            # e.g. &#195;&#235;&#224;&#226;&#224;
-            if "&#" in page:
-                page = re.sub(r"&#(\d{1,3});", lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)
+        # e.g. &#x9;&#195;&#235;&#224;&#226;&#224;
+        if "&#" in page:
+            page = re.sub(r"&#x([0-9a-f]{1,2});", lambda _: (_.group(1) if len(_.group(1)) == 2 else "0%s" % _.group(1)).decode("hex"), page)
+            page = re.sub(r"&#(\d{1,3});", lambda _: chr(int(_.group(1))) if int(_.group(1)) < 256 else _.group(0), page)
 
-            # e.g. %20%28%29
-            if "%" in page:
-                page = re.sub(r"%([0-9a-fA-F]{2})", lambda _: _.group(1).decode("hex"), page)
+        # e.g. %20%28%29
+        if "%" in page:
+            page = re.sub(r"%([0-9a-fA-F]{2})", lambda _: _.group(1).decode("hex"), page)
 
-            # e.g. &amp;
-            page = re.sub(r"&([^;]+);", lambda _: chr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 256) < 256 else _.group(0), page)
+        # e.g. &amp;
+        page = re.sub(r"&([^;]+);", lambda _: chr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 256) < 256 else _.group(0), page)
 
-            kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
+        kb.pageEncoding = kb.pageEncoding or checkCharEncoding(getHeuristicCharEncoding(page))
 
-            if (kb.pageEncoding or "").lower() == "utf-8-sig":
-                kb.pageEncoding = "utf-8"
-                if page and page.startswith("\xef\xbb\xbf"):  # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
-                    page = page[3:]
+        if (kb.pageEncoding or "").lower() == "utf-8-sig":
+            kb.pageEncoding = "utf-8"
+            if page and page.startswith("\xef\xbb\xbf"):  # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
+                page = page[3:]
 
-            page = getUnicode(page, kb.pageEncoding)
+        page = getUnicode(page, kb.pageEncoding)
 
-            # e.g. &#8217;&#8230;&#8482;
-            if "&#" in page:
-                def _(match):
-                    retVal = match.group(0)
-                    try:
-                        retVal = unichr(int(match.group(1)))
-                    except (ValueError, OverflowError):
-                        pass
-                    return retVal
-                page = re.sub(r"&#(\d+);", _, page)
+        # e.g. &#8217;&#8230;&#8482;
+        if "&#" in page:
+            def _(match):
+                retVal = match.group(0)
+                try:
+                    retVal = unichr(int(match.group(1)))
+                except (ValueError, OverflowError):
+                    pass
+                return retVal
+            page = re.sub(r"&#(\d+);", _, page)
 
-            # e.g. &zeta;
-            page = re.sub(r"&([^;]+);", lambda _: unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
+        # e.g. &zeta;
+        page = re.sub(r"&([^;]+);", lambda _: unichr(htmlEntities[_.group(1)]) if htmlEntities.get(_.group(1), 0) > 255 else _.group(0), page)
 
     return page
 

lib/request/connect.py

@@ -44,6 +44,7 @@ from lib.core.common import getHostHeader
 from lib.core.common import getRequestHeader
 from lib.core.common import getSafeExString
 from lib.core.common import getUnicode
+from lib.core.common import isMultiThreadMode
 from lib.core.common import logHTTPTraffic
 from lib.core.common import pushValue
 from lib.core.common import popValue
@@ -102,7 +103,7 @@ from lib.core.settings import MAX_MURPHY_SLEEP_TIME
 from lib.core.settings import META_REFRESH_REGEX
 from lib.core.settings import MIN_TIME_RESPONSES
 from lib.core.settings import MAX_TIME_RESPONSES
-from lib.core.settings import IDS_WAF_CHECK_PAYLOAD
+from lib.core.settings import IPS_WAF_CHECK_PAYLOAD
 from lib.core.settings import IS_WIN
 from lib.core.settings import LARGE_CHUNK_TRIM_MARKER
 from lib.core.settings import PAYLOAD_DELIMITER
@@ -270,6 +271,7 @@ class Connect(object):
         crawling = kwargs.get("crawling", False)
         checking = kwargs.get("checking", False)
         skipRead = kwargs.get("skipRead", False)
+        finalCode = kwargs.get("finalCode", False)
 
         if multipart:
             post = multipart
@@ -496,7 +498,7 @@ class Connect(object):
                 if hasattr(conn, "redurl"):
                     page = (threadData.lastRedirectMsg[1] if kb.redirectChoice == REDIRECTION.NO else Connect._connReadProxy(conn)) if not skipRead else None
                     skipLogTraffic = kb.redirectChoice == REDIRECTION.NO
-                    code = conn.redcode
+                    code = conn.redcode if not finalCode else code
                 else:
                     page = Connect._connReadProxy(conn) if not skipRead else None
 
@@ -555,11 +557,11 @@ class Connect(object):
                     if hasattr(conn.fp, '_sock'):
                         conn.fp._sock.close()
                     conn.close()
-                except Exception, ex:
+                except Exception as ex:
                     warnMsg = "problem occurred during connection closing ('%s')" % getSafeExString(ex)
                     logger.warn(warnMsg)
 
-        except SqlmapConnectionException, ex:
+        except SqlmapConnectionException as ex:
             if conf.proxyList and not kb.threadException:
                 warnMsg = "unable to connect to the target URL ('%s')" % ex
                 logger.critical(warnMsg)
@@ -568,7 +570,7 @@ class Connect(object):
             else:
                 raise
 
-        except urllib2.HTTPError, ex:
+        except urllib2.HTTPError as ex:
             page = None
             responseHeaders = None
 
@@ -646,7 +648,7 @@ class Connect(object):
                     debugMsg = "got HTTP error code: %d (%s)" % (code, status)
                     logger.debug(debugMsg)
 
-        except (urllib2.URLError, socket.error, socket.timeout, httplib.HTTPException, struct.error, binascii.Error, ProxyError, SqlmapCompressionException, WebSocketException, TypeError, ValueError):
+        except (urllib2.URLError, socket.error, socket.timeout, httplib.HTTPException, struct.error, binascii.Error, ProxyError, SqlmapCompressionException, WebSocketException, TypeError, ValueError, OverflowError):
             tbMsg = traceback.format_exc()
 
             if checking:
@@ -730,7 +732,7 @@ class Connect(object):
                 else:
                     logger.debug(warnMsg)
                 return Connect._retryProxy(**kwargs)
-            elif kb.testMode or kb.multiThreadMode:
+            elif kb.testMode or isMultiThreadMode():
                 logger.critical(warnMsg)
                 return None, None, None
             else:
@@ -816,7 +818,7 @@ class Connect(object):
 
         if conf.httpHeaders:
             headers = OrderedDict(conf.httpHeaders)
-            contentType = max(headers[_] if _.upper() == HTTP_HEADER.CONTENT_TYPE.upper() else None for _ in headers.keys())
+            contentType = max(headers[_] if _.upper() == HTTP_HEADER.CONTENT_TYPE.upper() else None for _ in headers)
 
             if (kb.postHint or conf.skipUrlEncode) and postUrlEncode:
                 postUrlEncode = False
@@ -833,7 +835,7 @@ class Connect(object):
 
                     try:
                         payload = function(payload=payload, headers=auxHeaders, delimiter=delimiter, hints=hints)
-                    except Exception, ex:
+                    except Exception as ex:
                         errMsg = "error occurred while running tamper "
                         errMsg += "function '%s' ('%s')" % (function.func_name, getSafeExString(ex))
                         raise SqlmapGenericException(errMsg)
@@ -1038,10 +1040,10 @@ class Connect(object):
         if conf.rParam:
             def _randomizeParameter(paramString, randomParameter):
                 retVal = paramString
-                match = re.search(r"(\A|\b)%s=(?P<value>[^&;]+)" % re.escape(randomParameter), paramString)
+                match = re.search(r"(\A|\b)%s=(?P<value>[^&;]*)" % re.escape(randomParameter), paramString)
                 if match:
                     origValue = match.group("value")
-                    retVal = re.sub(r"(\A|\b)%s=[^&;]+" % re.escape(randomParameter), "%s=%s" % (randomParameter, randomizeParameterValue(origValue)), paramString)
+                    retVal = re.sub(r"(\A|\b)%s=[^&;]*" % re.escape(randomParameter), "%s=%s" % (randomParameter, randomizeParameterValue(origValue)), paramString)
                 return retVal
 
             for randomParameter in conf.rParam:
@@ -1096,7 +1098,7 @@ class Connect(object):
             while True:
                 try:
                     compiler.parse(unicodeencode(conf.evalCode.replace(';', '\n')))
-                except SyntaxError, ex:
+                except SyntaxError as ex:
                     if ex.text:
                         original = replacement = ex.text.strip()
                         if '=' in original:
@@ -1124,7 +1126,7 @@ class Connect(object):
             originals.update(variables)
             evaluateCode(conf.evalCode, variables)
 
-            for variable in variables.keys():
+            for variable in list(variables.keys()):
                 if variable.endswith(EVALCODE_KEYWORD_SUFFIX):
                     value = variables[variable]
                     del variables[variable]
@@ -1282,7 +1284,7 @@ class Connect(object):
 
         if conf.secondUrl:
             page, headers, code = Connect.getPage(url=conf.secondUrl, cookie=cookie, ua=ua, silent=silent, auxHeaders=auxHeaders, response=response, raise404=False, ignoreTimeout=timeBasedCompare, refreshing=True)
-        elif kb.secondReq and IDS_WAF_CHECK_PAYLOAD not in urllib.unquote(value or ""):
+        elif kb.secondReq and IPS_WAF_CHECK_PAYLOAD not in urllib.unquote(value or ""):
             def _(value):
                 if kb.customInjectionMark in (value or ""):
                     if payload is None:
@@ -1306,8 +1308,8 @@ class Connect(object):
         elif noteResponseTime:
             kb.responseTimes.setdefault(kb.responseTimeMode, [])
             kb.responseTimes[kb.responseTimeMode].append(threadData.lastQueryDuration)
-            if len(kb.responseTimes) > MAX_TIME_RESPONSES:
-                kb.responseTimes = kb.responseTimes[-MAX_TIME_RESPONSES:]
+            if len(kb.responseTimes[kb.responseTimeMode]) > MAX_TIME_RESPONSES:
+                kb.responseTimes[kb.responseTimeMode] = kb.responseTimes[kb.responseTimeMode][-MAX_TIME_RESPONSES // 2:]
 
         if not response and removeReflection:
             page = removeReflectiveValues(page, payload)

lib/request/direct.py

@@ -25,6 +25,7 @@ from lib.core.enums import CUSTOM_LOGGING
 from lib.core.enums import DBMS
 from lib.core.enums import EXPECTED
 from lib.core.enums import TIMEOUT_STATE
+from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.core.settings import UNICODE_ENCODING
 from lib.utils.timeout import timeout
 
@@ -53,7 +54,7 @@ def direct(query, content=True):
 
     if not select and "EXEC " not in query.upper():
         timeout(func=conf.dbmsConnector.execute, args=(query,), duration=conf.timeout, default=None)
-    elif not (output and "sqlmapoutput" not in query and "sqlmapfile" not in query):
+    elif not (output and ("%soutput" % TAKEOVER_TABLE_PREFIX) not in query and ("%sfile" % TAKEOVER_TABLE_PREFIX) not in query):
         output, state = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)
         if state == TIMEOUT_STATE.NORMAL:
             hashDBWrite(query, output, True)

lib/request/dns.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import os
 import re
 import socket
@@ -145,13 +147,13 @@ if __name__ == "__main__":
                 if _ is None:
                     break
                 else:
-                    print "[i] %s" % _
+                    print("[i] %s" % _)
 
             time.sleep(1)
 
-    except socket.error, ex:
+    except socket.error as ex:
         if 'Permission' in str(ex):
-            print "[x] Please run with sudo/Administrator privileges"
+            print("[x] Please run with sudo/Administrator privileges")
         else:
             raise
     except KeyboardInterrupt:

lib/request/httpshandler.py

@@ -63,7 +63,7 @@ class HTTPSConnection(httplib.HTTPSConnection):
                         break
                     else:
                         sock.close()
-                except (ssl.SSLError, socket.error, httplib.BadStatusLine), ex:
+                except (ssl.SSLError, socket.error, httplib.BadStatusLine) as ex:
                     self._tunnel_host = None
                     logger.debug("SSL connection error occurred ('%s')" % getSafeExString(ex))
 
@@ -83,7 +83,7 @@ class HTTPSConnection(httplib.HTTPSConnection):
                         break
                     else:
                         sock.close()
-                except (ssl.SSLError, socket.error, httplib.BadStatusLine), ex:
+                except (ssl.SSLError, socket.error, httplib.BadStatusLine) as ex:
                     self._tunnel_host = None
                     logger.debug("SSL connection error occurred ('%s')" % getSafeExString(ex))
 

lib/request/inject.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import re
 import time
 
@@ -277,7 +279,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
                         raise SqlmapDataException(errMsg)
 
                 except KeyboardInterrupt:
-                    print
+                    print()
                     warnMsg = "user aborted during dumping phase"
                     logger.warn(warnMsg)
 
@@ -344,8 +346,13 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
     affected parameter.
     """
 
-    if conf.hexConvert:
-        charsetType = CHARSET_TYPE.HEXADECIMAL
+    if conf.hexConvert and expected != EXPECTED.BOOL and Backend.getIdentifiedDbms():
+        if not hasattr(queries[Backend.getIdentifiedDbms()], "hex"):
+            warnMsg = "switch '--hex' is currently not supported on DBMS %s" % Backend.getIdentifiedDbms()
+            singleTimeWarnMessage(warnMsg)
+            conf.hexConvert = False
+        else:
+            charsetType = CHARSET_TYPE.HEXADECIMAL
 
     kb.safeCharEncode = safeCharEncode
     kb.resumeValues = resumeValue

lib/request/pkihandler.py

@@ -24,7 +24,7 @@ class HTTPSPKIAuthHandler(urllib2.HTTPSHandler):
         try:
             # Reference: https://docs.python.org/2/library/ssl.html#ssl.SSLContext.load_cert_chain
             return httplib.HTTPSConnection(host, cert_file=self.auth_file, key_file=self.auth_file, timeout=conf.timeout)
-        except IOError, ex:
+        except IOError as ex:
             errMsg = "error occurred while using key "
             errMsg += "file '%s' ('%s')" % (self.auth_file, getSafeExString(ex))
             raise SqlmapConnectionException(errMsg)

lib/request/redirecthandler.py

@@ -16,6 +16,7 @@ from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.common import getHostHeader
+from lib.core.common import getSafeExString
 from lib.core.common import getUnicode
 from lib.core.common import logHTTPTraffic
 from lib.core.common import readInput
@@ -75,9 +76,9 @@ class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
 
         try:
             content = fp.read(MAX_CONNECTION_TOTAL_SIZE)
-        except Exception, msg:
+        except Exception as ex:
             dbgMsg = "there was a problem while retrieving "
-            dbgMsg += "redirect response content (%s)" % msg
+            dbgMsg += "redirect response content ('%s')" % getSafeExString(ex)
             logger.debug(dbgMsg)
         finally:
             if content:
@@ -140,8 +141,8 @@ class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
 
             try:
                 result = urllib2.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers)
-            except urllib2.HTTPError, e:
-                result = e
+            except urllib2.HTTPError as ex:
+                result = ex
 
                 # Dirty hack for http://bugs.python.org/issue15701
                 try:
@@ -153,7 +154,7 @@ class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
 
                 if not hasattr(result, "read"):
                     def _(self, length=None):
-                        return e.msg
+                        return ex.msg
                     result.read = types.MethodType(_, result)
 
                 if not getattr(result, "url", None):

lib/takeover/abstraction.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import sys
 
 from extra.safe2bin.safe2bin import safechardecode
@@ -129,11 +131,11 @@ class Abstraction(Web, UDF, XP_cmdshell):
                 command = raw_input("os-shell> ")
                 command = getUnicode(command, encoding=sys.stdin.encoding)
             except KeyboardInterrupt:
-                print
+                print()
                 errMsg = "user aborted"
                 logger.error(errMsg)
             except EOFError:
-                print
+                print()
                 errMsg = "exit"
                 logger.error(errMsg)
                 break

lib/takeover/metasploit.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import os
 import re
 import select
@@ -483,7 +485,7 @@ class Metasploit:
         send_all(proc, "getuid\n")
 
         if conf.privEsc:
-            print
+            print()
 
             infoMsg = "trying to escalate privileges using Meterpreter "
             infoMsg += "'getsystem' command which tries different "
@@ -556,7 +558,7 @@ class Metasploit:
 
                 # For --os-pwn and --os-bof
                 pwnBofCond = self.connectionStr.startswith("reverse")
-                pwnBofCond &= "Starting the payload handler" in out
+                pwnBofCond &= any(_ in out for _ in ("Starting the payload handler", "Started reverse"))
 
                 # For --os-smbrelay
                 smbRelayCond = "Server started" in out
@@ -619,7 +621,7 @@ class Metasploit:
             payloadSize = int(match.group(2))
 
             if extra == "BufferRegister=EAX":
-                payloadSize = payloadSize / 2
+                payloadSize = payloadSize // 2
 
             debugMsg = "the shellcode size is %d bytes" % payloadSize
             logger.debug(debugMsg)

lib/takeover/web.py

@@ -77,7 +77,7 @@ class Web:
         if not cmd:
             cmd = conf.osCmd
 
-        cmdUrl = "%s?cmd=%s" % (self.webBackdoorUrl, cmd)
+        cmdUrl = "%s?cmd=%s" % (self.webBackdoorUrl, getUnicode(cmd))
         page, _, _ = Request.getPage(url=cmdUrl, direct=True, silent=True, timeout=BACKDOOR_RUN_CMD_TIMEOUT)
 
         if page is not None:

lib/techniques/blind/inference.py

@@ -82,7 +82,9 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
     retVal = hashDBRetrieve(expression, checkConf=True)
 
     if retVal:
-        if PARTIAL_HEX_VALUE_MARKER in retVal:
+        if conf.repair and INFERENCE_UNKNOWN_CHAR in retVal:
+            pass
+        elif PARTIAL_HEX_VALUE_MARKER in retVal:
             retVal = retVal.replace(PARTIAL_HEX_VALUE_MARKER, "")
 
             if retVal and conf.hexConvert:
@@ -631,9 +633,12 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
                     dataToStdout(filterControlChars(val))
 
                 # some DBMSes (e.g. Firebird, DB2, etc.) have issues with trailing spaces
-                if len(partialValue) > INFERENCE_BLANK_BREAK and partialValue[-INFERENCE_BLANK_BREAK:].isspace() and partialValue.strip(' ')[-1:] != '\n':
+                if len(partialValue) > INFERENCE_BLANK_BREAK and partialValue[-INFERENCE_BLANK_BREAK:].isspace():
                     finalValue = partialValue[:-INFERENCE_BLANK_BREAK]
                     break
+                elif charsetType and partialValue[-1:].isspace():
+                    finalValue = partialValue[:-1]
+                    break
 
                 if (lastChar > 0 and index >= lastChar):
                     finalValue = "" if length == 0 else partialValue
@@ -645,7 +650,6 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         abortedFlag = True
     finally:
         kb.prependFlag = False
-        kb.stickyLevel = None
         retrievedLength = len(finalValue or "")
 
         if finalValue is not None:

lib/techniques/dns/use.py

@@ -57,7 +57,7 @@ def dnsUse(payload, expression):
             while True:
                 count += 1
                 prefix, suffix = ("%s" % randomStr(length=3, alphabet=DNS_BOUNDARIES_ALPHABET) for _ in xrange(2))
-                chunk_length = MAX_DNS_LABEL / 2 if Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.MYSQL, DBMS.PGSQL) else MAX_DNS_LABEL / 4 - 2
+                chunk_length = MAX_DNS_LABEL // 2 if Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.MYSQL, DBMS.PGSQL) else MAX_DNS_LABEL // 4 - 2
                 _, _, _, _, _, _, fieldToCastStr, _ = agent.getFields(expression)
                 nulledCastedField = agent.nullAndCastField(fieldToCastStr)
                 extendedField = re.search(r"[^ ,]*%s[^ ,]*" % re.escape(fieldToCastStr), expression).group(0)

lib/techniques/error/use.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import re
 import time
 
@@ -92,7 +94,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                     candidate = len(result) - len(kb.chars.stop)
                     current = candidate if candidate != current else current - 1
             else:
-                current = current / 2
+                current = current // 2
 
         if kb.errorChunkLength:
             hashDBWrite(HASHDB_KEYS.KB_ERROR_CHUNK_LENGTH, kb.errorChunkLength)
@@ -242,7 +244,7 @@ def _errorFields(expression, expressionFields, expressionFieldsList, num=None, e
 
         if not suppressOutput:
             if kb.fileReadMode and output and output.strip():
-                print
+                print()
             elif output is not None and not (threadData.resumed and kb.suppressResumeInfo) and not (emptyFields and field in emptyFields):
                 status = "[%s] [INFO] %s: '%s'" % (time.strftime("%X"), "resumed" if threadData.resumed else "retrieved", output if kb.safeCharEncode else safecharencode(output))
 
@@ -403,7 +405,7 @@ def errorUse(expression, dump=False):
                             with kb.locks.limit:
                                 try:
                                     threadData.shared.counter += 1
-                                    num = threadData.shared.limits.next()
+                                    num = next(threadData.shared.limits)
                                 except StopIteration:
                                     break
 

lib/techniques/union/test.py

@@ -74,7 +74,7 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
                     highCols += ORDER_BY_STEP
                 else:
                     while not found:
-                        mid = highCols - (highCols - lowCols) / 2
+                        mid = highCols - (highCols - lowCols) // 2
                         if _orderByTest(mid):
                             lowCols = mid
                         else:

lib/techniques/union/use.py

@@ -313,7 +313,7 @@ def unionUse(expression, unpack=True, dump=False):
                             with kb.locks.limit:
                                 try:
                                     threadData.shared.counter += 1
-                                    num = threadData.shared.limits.next()
+                                    num = next(threadData.shared.limits)
                                 except StopIteration:
                                     break
 

lib/utils/api.py

@@ -6,6 +6,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import contextlib
 import httplib
 import logging
@@ -95,7 +97,7 @@ class Database(object):
                     self.cursor.execute(statement, arguments)
                 else:
                     self.cursor.execute(statement)
-            except sqlite3.OperationalError, ex:
+            except sqlite3.OperationalError as ex:
                 if "locked" not in getSafeExString(ex):
                     raise
             else:
@@ -158,11 +160,11 @@ class Task(object):
         saveConfig(self.options, configFile)
 
         if os.path.exists("sqlmap.py"):
-            self.process = Popen(["python", "sqlmap.py", "--api", "-c", configFile], shell=False, close_fds=not IS_WIN)
+            self.process = Popen([sys.executable or "python", "sqlmap.py", "--api", "-c", configFile], shell=False, close_fds=not IS_WIN)
         elif os.path.exists(os.path.join(os.getcwd(), "sqlmap.py")):
-            self.process = Popen(["python", "sqlmap.py", "--api", "-c", configFile], shell=False, cwd=os.getcwd(), close_fds=not IS_WIN)
+            self.process = Popen([sys.executable or "python", "sqlmap.py", "--api", "-c", configFile], shell=False, cwd=os.getcwd(), close_fds=not IS_WIN)
         elif os.path.exists(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), "sqlmap.py")):
-            self.process = Popen(["python", "sqlmap.py", "--api", "-c", configFile], shell=False, cwd=os.path.join(os.path.abspath(os.path.dirname(sys.argv[0]))), close_fds=not IS_WIN)
+            self.process = Popen([sys.executable or "python", "sqlmap.py", "--api", "-c", configFile], shell=False, cwd=os.path.join(os.path.abspath(os.path.dirname(sys.argv[0]))), close_fds=not IS_WIN)
         else:
             self.process = Popen(["sqlmap", "--api", "-c", configFile], shell=False, close_fds=not IS_WIN)
 
@@ -266,7 +268,7 @@ def setRestAPILog():
         try:
             conf.databaseCursor = Database(conf.database)
             conf.databaseCursor.connect("client")
-        except sqlite3.OperationalError, ex:
+        except sqlite3.OperationalError as ex:
             raise SqlmapConnectionException("%s ('%s')" % (ex, conf.database))
 
         # Set a logging handler that writes log messages to a IPC database
@@ -689,7 +691,7 @@ def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, adapter=REST
             eventlet.monkey_patch()
         logger.debug("Using adapter '%s' to run bottle" % adapter)
         run(host=host, port=port, quiet=True, debug=True, server=adapter)
-    except socket.error, ex:
+    except socket.error as ex:
         if "already in use" in getSafeExString(ex):
             logger.error("Address already in use ('%s:%s')" % (host, port))
         else:
@@ -697,7 +699,7 @@ def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, adapter=REST
     except ImportError:
         if adapter.lower() not in server_names:
             errMsg = "Adapter '%s' is unknown. " % adapter
-            errMsg += "List of supported adapters: %s" % ', '.join(sorted(server_names.keys()))
+            errMsg += "List of supported adapters: %s" % ', '.join(sorted(list(server_names.keys())))
         else:
             errMsg = "Server support for adapter '%s' is not installed on this system " % adapter
             errMsg += "(Note: you can try to install it with 'sudo apt-get install python-%s' or 'sudo pip install %s')" % (adapter, adapter)
@@ -743,7 +745,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
 
     try:
         _client(addr)
-    except Exception, ex:
+    except Exception as ex:
         if not isinstance(ex, urllib2.HTTPError) or ex.code == httplib.UNAUTHORIZED:
             errMsg = "There has been a problem while connecting to the "
             errMsg += "REST-JSON API server at '%s' " % addr
@@ -762,7 +764,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
             command = raw_input("api%s> " % (" (%s)" % taskid if taskid else "")).strip()
             command = re.sub(r"\A(\w+)", lambda match: match.group(1).lower(), command)
         except (EOFError, KeyboardInterrupt):
-            print
+            print()
             break
 
         if command in ("data", "log", "status", "stop", "kill"):
@@ -798,7 +800,7 @@ def client(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, username=Non
 
             try:
                 argv = ["sqlmap.py"] + shlex.split(command)[1:]
-            except Exception, ex:
+            except Exception as ex:
                 logger.error("Error occurred while parsing arguments ('%s')" % ex)
                 taskid = None
                 continue

lib/utils/crawler.py

@@ -63,14 +63,14 @@ def crawl(target):
                 try:
                     if current:
                         content = Request.getPage(url=current, crawling=True, raise404=False)[0]
-                except SqlmapConnectionException, ex:
+                except SqlmapConnectionException as ex:
                     errMsg = "connection exception detected ('%s'). skipping " % getSafeExString(ex)
                     errMsg += "URL '%s'" % current
                     logger.critical(errMsg)
                 except SqlmapSyntaxException:
                     errMsg = "invalid URL detected. skipping '%s'" % current
                     logger.critical(errMsg)
-                except httplib.InvalidURL, ex:
+                except httplib.InvalidURL as ex:
                     errMsg = "invalid URL detected ('%s'). skipping " % getSafeExString(ex)
                     errMsg += "URL '%s'" % current
                     logger.critical(errMsg)
@@ -138,7 +138,7 @@ def crawl(target):
                 url = urlparse.urljoin(target, "/sitemap.xml")
                 try:
                     items = parseSitemap(url)
-                except SqlmapConnectionException, ex:
+                except SqlmapConnectionException as ex:
                     if "page not found" in getSafeExString(ex):
                         found = False
                         logger.warn("'sitemap.xml' not found")

lib/utils/hash.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 try:
     from crypt import crypt
 except:  # removed ImportError because of https://github.com/sqlmapproject/sqlmap/issues/3171
@@ -19,7 +21,7 @@ try:
 
     # problems with ctypes (Reference: https://github.com/sqlmapproject/sqlmap/issues/2952)
     _ = multiprocessing.Value('i')
-except (ImportError, OSError):
+except (ImportError, OSError, AttributeError):
     pass
 else:
     try:
@@ -599,7 +601,7 @@ def attackCachedUsersPasswords():
         for (_, hash_, password) in results:
             lut[hash_.lower()] = password
 
-        for user in kb.data.cachedUsersPasswords.keys():
+        for user in kb.data.cachedUsersPasswords:
             for i in xrange(len(kb.data.cachedUsersPasswords[user])):
                 if (kb.data.cachedUsersPasswords[user][i] or "").strip():
                     value = kb.data.cachedUsersPasswords[user][i].lower().split()[0]
@@ -609,7 +611,7 @@ def attackCachedUsersPasswords():
 def attackDumpedTable():
     if kb.data.dumpedTable:
         table = kb.data.dumpedTable
-        columns = table.keys()
+        columns = list(table.keys())
         count = table["__infos__"]["count"]
 
         if not count:
@@ -770,8 +772,8 @@ def _bruteProcessVariantA(attack_info, hash_regex, suffix, retVal, proc_id, proc
             except (UnicodeEncodeError, UnicodeDecodeError):
                 pass  # ignore possible encoding problems caused by some words in custom dictionaries
 
-            except Exception, e:
-                warnMsg = "there was a problem while hashing entry: %s (%s). " % (repr(word), e)
+            except Exception as ex:
+                warnMsg = "there was a problem while hashing entry: %s ('%s'). " % (repr(word), getSafeExString(ex))
                 warnMsg += "Please report by e-mail to '%s'" % DEV_EMAIL_ADDRESS
                 logger.critical(warnMsg)
 
@@ -847,8 +849,8 @@ def _bruteProcessVariantB(user, hash_, kwargs, hash_regex, suffix, retVal, found
             except (UnicodeEncodeError, UnicodeDecodeError):
                 pass  # ignore possible encoding problems caused by some words in custom dictionaries
 
-            except Exception, e:
-                warnMsg = "there was a problem while hashing entry: %s (%s). " % (repr(word), e)
+            except Exception as ex:
+                warnMsg = "there was a problem while hashing entry: %s ('%s'). " % (repr(word), getSafeExString(ex))
                 warnMsg += "Please report by e-mail to '%s'" % DEV_EMAIL_ADDRESS
                 logger.critical(warnMsg)
 
@@ -983,7 +985,7 @@ def dictionaryAttack(attack_dict):
                     else:
                         logger.info("using default dictionary")
 
-                    dictPaths = filter(None, dictPaths)
+                    dictPaths = [_ for _ in dictPaths if _]
 
                     for dictPath in dictPaths:
                         checkFile(dictPath)
@@ -998,7 +1000,7 @@ def dictionaryAttack(attack_dict):
 
                     kb.wordlists = dictPaths
 
-                except Exception, ex:
+                except Exception as ex:
                     warnMsg = "there was a problem while loading dictionaries"
                     warnMsg += " ('%s')" % getSafeExString(ex)
                     logger.critical(warnMsg)
@@ -1061,7 +1063,7 @@ def dictionaryAttack(attack_dict):
                         _bruteProcessVariantA(attack_info, hash_regex, suffix, retVal, 0, 1, kb.wordlists, custom_wordlist, conf.api)
 
                 except KeyboardInterrupt:
-                    print
+                    print()
                     processException = True
                     warnMsg = "user aborted during dictionary-based attack phase (Ctrl+C was pressed)"
                     logger.warn(warnMsg)
@@ -1157,7 +1159,7 @@ def dictionaryAttack(attack_dict):
                             found = found_.value
 
                     except KeyboardInterrupt:
-                        print
+                        print()
                         processException = True
                         warnMsg = "user aborted during dictionary-based attack phase (Ctrl+C was pressed)"
                         logger.warn(warnMsg)

lib/utils/hashdb.py

@@ -41,7 +41,7 @@ class HashDB(object):
                 threadData.hashDBCursor = connection.cursor()
                 threadData.hashDBCursor.execute("CREATE TABLE IF NOT EXISTS storage (id INTEGER PRIMARY KEY, value TEXT)")
                 connection.commit()
-            except Exception, ex:
+            except Exception as ex:
                 errMsg = "error occurred while opening a session "
                 errMsg += "file '%s' ('%s')" % (self.filepath, getSafeExString(ex))
                 raise SqlmapConnectionException(errMsg)
@@ -81,18 +81,16 @@ class HashDB(object):
                     try:
                         for row in self.cursor.execute("SELECT value FROM storage WHERE id=?", (hash_,)):
                             retVal = row[0]
-                    except sqlite3.OperationalError, ex:
+                    except (sqlite3.OperationalError, sqlite3.DatabaseError) as ex:
                         if any(_ in getSafeExString(ex) for _ in ("locked", "no such table")):
                             warnMsg = "problem occurred while accessing session file '%s' ('%s')" % (self.filepath, getSafeExString(ex))
                             singleTimeWarnMessage(warnMsg)
                         elif "Could not decode" in getSafeExString(ex):
                             break
                         else:
-                            raise
-                    except sqlite3.DatabaseError, ex:
-                        errMsg = "error occurred while accessing session file '%s' ('%s'). " % (self.filepath, getSafeExString(ex))
-                        errMsg += "If the problem persists please rerun with '--flush-session'"
-                        raise SqlmapConnectionException(errMsg)
+                            errMsg = "error occurred while accessing session file '%s' ('%s'). " % (self.filepath, getSafeExString(ex))
+                            errMsg += "If the problem persists please rerun with '--flush-session'"
+                            raise SqlmapConnectionException(errMsg)
                     else:
                         break
 
@@ -141,7 +139,7 @@ class HashDB(object):
                             self.cursor.execute("INSERT INTO storage VALUES (?, ?)", (hash_, value,))
                         except sqlite3.IntegrityError:
                             self.cursor.execute("UPDATE storage SET value=? WHERE id=?", (value, hash_,))
-                    except sqlite3.DatabaseError, ex:
+                    except sqlite3.DatabaseError as ex:
                         if not os.path.exists(self.filepath):
                             debugMsg = "session file '%s' does not exist" % self.filepath
                             logger.debug(debugMsg)

lib/utils/pivotdumptable.py

@@ -175,7 +175,7 @@ def pivotDumpTable(table, colList, count=None, blind=True, alias=None):
         warnMsg += "will display partial output"
         logger.warn(warnMsg)
 
-    except SqlmapConnectionException, ex:
+    except SqlmapConnectionException as ex:
         errMsg = "connection exception detected ('%s'). sqlmap " % getSafeExString(ex)
         errMsg += "will display partial output"
 

lib/utils/progress.py

@@ -29,7 +29,7 @@ class ProgressBar(object):
 
     def _convertSeconds(self, value):
         seconds = value
-        minutes = seconds / 60
+        minutes = seconds // 60
         seconds = seconds - (minutes * 60)
 
         return "%.2d:%.2d" % (minutes, seconds)

lib/utils/purge.py

@@ -79,5 +79,5 @@ def purge(directory):
 
     try:
         shutil.rmtree(directory)
-    except OSError, ex:
+    except OSError as ex:
         logger.error("problem occurred while removing directory '%s' ('%s')" % (directory, getSafeExString(ex)))

lib/utils/search.py

@@ -45,6 +45,7 @@ def _search(dork):
     if not dork:
         return None
 
+    data = None
     headers = {}
 
     headers[HTTP_HEADER.USER_AGENT] = dict(conf.httpHeaders).get(HTTP_HEADER.USER_AGENT, DUMMY_SEARCH_USER_AGENT)
@@ -53,7 +54,7 @@ def _search(dork):
     try:
         req = urllib2.Request("https://www.google.com/ncr", headers=headers)
         conn = urllib2.urlopen(req)
-    except Exception, ex:
+    except Exception as ex:
         errMsg = "unable to connect to Google ('%s')" % getSafeExString(ex)
         raise SqlmapConnectionException(errMsg)
 
@@ -87,12 +88,12 @@ def _search(dork):
             responseMsg += "%s\n%s\n" % (responseHeaders, page)
 
         logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
-    except urllib2.HTTPError, e:
+    except urllib2.HTTPError as ex:
         try:
-            page = e.read()
-        except Exception, ex:
+            page = ex.read()
+        except Exception as _:
             warnMsg = "problem occurred while trying to get "
-            warnMsg += "an error page information (%s)" % getSafeExString(ex)
+            warnMsg += "an error page information (%s)" % getSafeExString(_)
             logger.critical(warnMsg)
             return None
     except (urllib2.URLError, httplib.error, socket.error, socket.timeout, socks.ProxyError):
@@ -123,12 +124,12 @@ def _search(dork):
             url = "https://www.bing.com/search?q=%s&first=%d" % (urlencode(dork, convall=True), (gpage - 1) * 10 + 1)
             regex = BING_REGEX
         else:
-            url = "https://duckduckgo.com/d.js?"
-            url += "q=%s&p=%d&s=100" % (urlencode(dork, convall=True), gpage)
+            url = "https://duckduckgo.com/html/"
+            data = "q=%s&s=%d" % (urlencode(dork, convall=True), (gpage - 1) * 30)
             regex = DUCKDUCKGO_REGEX
 
         try:
-            req = urllib2.Request(url, headers=headers)
+            req = urllib2.Request(url, data=data, headers=headers)
             conn = urllib2.urlopen(req)
 
             requestMsg = "HTTP request:\nGET %s" % url
@@ -149,19 +150,29 @@ def _search(dork):
                 responseMsg += "%s\n%s\n" % (responseHeaders, page)
 
             logger.log(CUSTOM_LOGGING.TRAFFIC_IN, responseMsg)
-        except urllib2.HTTPError, e:
+        except urllib2.HTTPError as ex:
             try:
-                page = e.read()
+                page = ex.read()
+                page = decodePage(page, ex.headers.get("Content-Encoding"), ex.headers.get("Content-Type"))
             except socket.timeout:
                 warnMsg = "connection timed out while trying "
-                warnMsg += "to get error page information (%d)" % e.code
+                warnMsg += "to get error page information (%d)" % ex.code
                 logger.critical(warnMsg)
                 return None
         except:
             errMsg = "unable to connect"
             raise SqlmapConnectionException(errMsg)
 
-        retVal = [urllib.unquote(match.group(1)) for match in re.finditer(regex, page, re.I | re.S)]
+        retVal = [urllib.unquote(match.group(1).replace("&", "&")) for match in re.finditer(regex, page, re.I | re.S)]
+
+        if not retVal and "issue with the Tor Exit Node you are currently using" in page:
+            warnMsg = "DuckDuckGo has detected 'unusual' traffic from "
+            warnMsg += "used (Tor) IP address"
+
+            if conf.proxyList:
+                raise SqlmapBaseException(warnMsg)
+            else:
+                logger.critical(warnMsg)
 
     return retVal
 
@@ -172,7 +183,7 @@ def search(dork):
 
     try:
         return _search(dork)
-    except SqlmapBaseException, ex:
+    except SqlmapBaseException as ex:
         if conf.proxyList:
             logger.critical(getSafeExString(ex))
 

lib/utils/sqlalchemy.py

@@ -76,8 +76,8 @@ class SQLAlchemy(GenericConnector):
                 raise
             except SqlmapFilePathException:
                 raise
-            except Exception, msg:
-                raise SqlmapConnectionException("SQLAlchemy connection issue ('%s')" % msg[0])
+            except Exception as ex:
+                raise SqlmapConnectionException("SQLAlchemy connection issue ('%s')" % ex[0])
 
             self.printConnected()
         else:
@@ -89,17 +89,17 @@ class SQLAlchemy(GenericConnector):
             for row in self.cursor.fetchall():
                 retVal.append(tuple(row))
             return retVal
-        except _sqlalchemy.exc.ProgrammingError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % msg.message if hasattr(msg, "message") else msg)
+        except _sqlalchemy.exc.ProgrammingError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % ex.message if hasattr(ex, "message") else ex)
             return None
 
     def execute(self, query):
         try:
             self.cursor = self.connector.execute(query)
-        except (_sqlalchemy.exc.OperationalError, _sqlalchemy.exc.ProgrammingError), msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % msg.message if hasattr(msg, "message") else msg)
-        except _sqlalchemy.exc.InternalError, msg:
-            raise SqlmapConnectionException(msg[1])
+        except (_sqlalchemy.exc.OperationalError, _sqlalchemy.exc.ProgrammingError) as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % ex.message if hasattr(ex, "message") else ex)
+        except _sqlalchemy.exc.InternalError as ex:
+            raise SqlmapConnectionException(ex[1])
 
     def select(self, query):
         self.execute(query)

lib/utils/timeout.py

@@ -22,8 +22,8 @@ def timeout(func, args=(), kwargs={}, duration=1, default=None):
             try:
                 self.result = func(*args, **kwargs)
                 self.timeout_state = TIMEOUT_STATE.NORMAL
-            except Exception, msg:
-                logger.log(CUSTOM_LOGGING.TRAFFIC_IN, msg)
+            except Exception as ex:
+                logger.log(CUSTOM_LOGGING.TRAFFIC_IN, ex)
                 self.result = default
                 self.timeout_state = TIMEOUT_STATE.EXCEPTION
 

lib/utils/xrange.py

@@ -58,7 +58,7 @@ class xrange(object):
         return self._len()
 
     def _len(self):
-        return max(0, int((self.stop - self.start) / self.step))
+        return max(0, int((self.stop - self.start) // self.step))
 
     def __contains__(self, value):
         return (self.start <= value < self.stop) and (value - self.start) % self.step == 0

plugins/dbms/access/connector.py

@@ -43,8 +43,8 @@ class Connector(GenericConnector):
 
         try:
             self.connector = pyodbc.connect('Driver={Microsoft Access Driver (*.mdb)};Dbq=%s;Uid=Admin;Pwd=;' % self.db)
-        except (pyodbc.Error, pyodbc.OperationalError), msg:
-            raise SqlmapConnectionException(getSafeExString(msg))
+        except (pyodbc.Error, pyodbc.OperationalError) as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.initCursor()
         self.printConnected()
@@ -52,17 +52,17 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except pyodbc.ProgrammingError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
+        except pyodbc.ProgrammingError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
             return None
 
     def execute(self, query):
         try:
             self.cursor.execute(query)
-        except (pyodbc.OperationalError, pyodbc.ProgrammingError), msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
-        except pyodbc.Error, msg:
-            raise SqlmapConnectionException(getSafeExString(msg))
+        except (pyodbc.OperationalError, pyodbc.ProgrammingError) as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
+        except pyodbc.Error as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.connector.commit()
 

plugins/dbms/db2/connector.py

@@ -35,8 +35,8 @@ class Connector(GenericConnector):
         try:
             database = "DRIVER={IBM DB2 ODBC DRIVER};DATABASE=%s;HOSTNAME=%s;PORT=%s;PROTOCOL=TCPIP;" % (self.db, self.hostname, self.port)
             self.connector = ibm_db_dbi.connect(database, self.user, self.password)
-        except ibm_db_dbi.OperationalError, msg:
-            raise SqlmapConnectionException(msg)
+        except ibm_db_dbi.OperationalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.initCursor()
         self.printConnected()
@@ -44,17 +44,17 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except ibm_db_dbi.ProgrammingError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
+        except ibm_db_dbi.ProgrammingError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex))
             return None
 
     def execute(self, query):
         try:
             self.cursor.execute(query)
-        except (ibm_db_dbi.OperationalError, ibm_db_dbi.ProgrammingError), msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
-        except ibm_db_dbi.InternalError, msg:
-            raise SqlmapConnectionException(getSafeExString(msg))
+        except (ibm_db_dbi.OperationalError, ibm_db_dbi.ProgrammingError) as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex))
+        except ibm_db_dbi.InternalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.connector.commit()
 

plugins/dbms/firebird/connector.py

@@ -42,8 +42,8 @@ class Connector(GenericConnector):
         try:
             # Reference: http://www.daniweb.com/forums/thread248499.html
             self.connector = kinterbasdb.connect(host=self.hostname.encode(UNICODE_ENCODING), database=self.db.encode(UNICODE_ENCODING), user=self.user.encode(UNICODE_ENCODING), password=self.password.encode(UNICODE_ENCODING), charset="UTF8")
-        except kinterbasdb.OperationalError, msg:
-            raise SqlmapConnectionException(getSafeExString(msg))
+        except kinterbasdb.OperationalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.initCursor()
         self.printConnected()
@@ -51,17 +51,17 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except kinterbasdb.OperationalError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
+        except kinterbasdb.OperationalError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
             return None
 
     def execute(self, query):
         try:
             self.cursor.execute(query)
-        except kinterbasdb.OperationalError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
-        except kinterbasdb.Error, msg:
-            raise SqlmapConnectionException(getSafeExString(msg))
+        except kinterbasdb.OperationalError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
+        except kinterbasdb.Error as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.connector.commit()
 

plugins/dbms/hsqldb/connector.py

@@ -14,6 +14,7 @@ except:
 import logging
 
 from lib.core.common import checkFile
+from lib.core.common import getSafeExString
 from lib.core.common import readInput
 from lib.core.data import conf
 from lib.core.data import logger
@@ -41,15 +42,15 @@ class Connector(GenericConnector):
             args = "-Djava.class.path=%s" % jar
             jvm_path = jpype.getDefaultJVMPath()
             jpype.startJVM(jvm_path, args)
-        except Exception, msg:
-            raise SqlmapConnectionException(msg[0])
+        except Exception as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         try:
             driver = 'org.hsqldb.jdbc.JDBCDriver'
             connection_string = 'jdbc:hsqldb:mem:.'  # 'jdbc:hsqldb:hsql://%s/%s' % (self.hostname, self.db)
             self.connector = jaydebeapi.connect(driver, connection_string, str(self.user), str(self.password))
-        except Exception, msg:
-            raise SqlmapConnectionException(msg[0])
+        except Exception as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.initCursor()
         self.printConnected()
@@ -57,8 +58,8 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except Exception, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % msg[1])
+        except Exception as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex))
             return None
 
     def execute(self, query):
@@ -67,8 +68,8 @@ class Connector(GenericConnector):
         try:
             self.cursor.execute(query)
             retVal = True
-        except Exception, msg:  # TODO: fix with specific error
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % msg[1])
+        except Exception as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex))
 
         self.connector.commit()
 

plugins/dbms/informix/connector.py

@@ -35,8 +35,8 @@ class Connector(GenericConnector):
         try:
             database = "DATABASE=%s;HOSTNAME=%s;PORT=%s;PROTOCOL=TCPIP;" % (self.db, self.hostname, self.port)
             self.connector = ibm_db_dbi.connect(database, self.user, self.password)
-        except ibm_db_dbi.OperationalError, msg:
-            raise SqlmapConnectionException(getSafeExString(msg))
+        except ibm_db_dbi.OperationalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.initCursor()
         self.printConnected()
@@ -44,17 +44,17 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except ibm_db_dbi.ProgrammingError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
+        except ibm_db_dbi.ProgrammingError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
             return None
 
     def execute(self, query):
         try:
             self.cursor.execute(query)
-        except (ibm_db_dbi.OperationalError, ibm_db_dbi.ProgrammingError), msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
-        except ibm_db_dbi.InternalError, msg:
-            raise SqlmapConnectionException(getSafeExString(msg))
+        except (ibm_db_dbi.OperationalError, ibm_db_dbi.ProgrammingError) as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
+        except ibm_db_dbi.InternalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.connector.commit()
 

plugins/dbms/maxdb/enumeration.py

@@ -67,7 +67,7 @@ class Enumeration(GenericEnumeration):
         else:
             dbs = self.getDbs()
 
-        for db in filter(None, dbs):
+        for db in (_ for _ in dbs if _):
             dbs[dbs.index(db)] = safeSQLIdentificatorNaming(db)
 
         infoMsg = "fetching tables for database"

plugins/dbms/mssqlserver/connector.py

@@ -13,6 +13,7 @@ except:
 
 import logging
 
+from lib.core.common import getSafeExString
 from lib.core.convert import utf8encode
 from lib.core.data import conf
 from lib.core.data import logger
@@ -41,8 +42,8 @@ class Connector(GenericConnector):
 
         try:
             self.connector = pymssql.connect(host="%s:%d" % (self.hostname, self.port), user=self.user, password=self.password, database=self.db, login_timeout=conf.timeout, timeout=conf.timeout)
-        except (pymssql2.Error, _mssql.MssqlDatabaseException), msg:
-            raise SqlmapConnectionException(msg)
+        except (pymssql.Error, _mssql.MssqlDatabaseException) as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
         except ValueError:
             raise SqlmapConnectionException
 
@@ -52,8 +53,8 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except (pymssql.Error, _mssql.MssqlDatabaseException), msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % str(msg).replace("\n", " "))
+        except (pymssql.Error, _mssql.MssqlDatabaseException) as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex).replace("\n", " "))
             return None
 
     def execute(self, query):
@@ -62,10 +63,10 @@ class Connector(GenericConnector):
         try:
             self.cursor.execute(utf8encode(query))
             retVal = True
-        except (pymssql.OperationalError, pymssql.ProgrammingError), msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % str(msg).replace("\n", " "))
-        except pymssql.InternalError, msg:
-            raise SqlmapConnectionException(msg)
+        except (pymssql.OperationalError, pymssql.ProgrammingError) as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex).replace("\n", " "))
+        except pymssql.InternalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         return retVal
 

plugins/dbms/mssqlserver/enumeration.py

@@ -83,7 +83,7 @@ class Enumeration(GenericEnumeration):
         for db in dbs:
             dbs[dbs.index(db)] = safeSQLIdentificatorNaming(db)
 
-        dbs = filter(None, dbs)
+        dbs = [_ for _ in dbs if _]
 
         infoMsg = "fetching tables for database"
         infoMsg += "%s: %s" % ("s" if len(dbs) > 1 else "", ", ".join(db if isinstance(db, basestring) else db[0] for db in sorted(dbs)))
@@ -110,7 +110,7 @@ class Enumeration(GenericEnumeration):
                         break
 
                 if not isNoneValue(value):
-                    value = filter(None, arrayizeValue(value))
+                    value = [_ for _ in arrayizeValue(value) if _]
                     value = [safeSQLIdentificatorNaming(unArrayizeValue(_), True) for _ in value]
                     kb.data.cachedTables[db] = value
 
@@ -340,7 +340,7 @@ class Enumeration(GenericEnumeration):
             colQuery = "%s%s" % (colCond, colCondParam)
             colQuery = colQuery % unsafeSQLIdentificatorNaming(column)
 
-            for db in filter(None, dbs.keys()):
+            for db in (_ for _ in dbs if _):
                 db = safeSQLIdentificatorNaming(db)
 
                 if conf.excludeSysDbs and db in self.excludeDbsList:

plugins/dbms/mysql/connector.py

@@ -38,8 +38,8 @@ class Connector(GenericConnector):
 
         try:
             self.connector = pymysql.connect(host=self.hostname, user=self.user, passwd=self.password, db=self.db, port=self.port, connect_timeout=conf.timeout, use_unicode=True)
-        except (pymysql.OperationalError, pymysql.InternalError, pymysql.ProgrammingError, struct.error), msg:
-            raise SqlmapConnectionException(getSafeExString(msg))
+        except (pymysql.OperationalError, pymysql.InternalError, pymysql.ProgrammingError, struct.error) as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.initCursor()
         self.printConnected()
@@ -47,8 +47,8 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except pymysql.ProgrammingError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
+        except pymysql.ProgrammingError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
             return None
 
     def execute(self, query):
@@ -57,10 +57,10 @@ class Connector(GenericConnector):
         try:
             self.cursor.execute(query)
             retVal = True
-        except (pymysql.OperationalError, pymysql.ProgrammingError), msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(msg))
-        except pymysql.InternalError, msg:
-            raise SqlmapConnectionException(getSafeExString(msg))
+        except (pymysql.OperationalError, pymysql.ProgrammingError) as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % getSafeExString(ex))
+        except pymysql.InternalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.connector.commit()
 

plugins/dbms/oracle/connector.py

@@ -14,6 +14,7 @@ import logging
 import os
 import re
 
+from lib.core.common import getSafeExString
 from lib.core.convert import utf8encode
 from lib.core.data import conf
 from lib.core.data import logger
@@ -43,17 +44,17 @@ class Connector(GenericConnector):
         try:
             self.connector = cx_Oracle.connect(dsn=self.__dsn, user=self.user, password=self.password, mode=cx_Oracle.SYSDBA)
             logger.info("successfully connected as SYSDBA")
-        except (cx_Oracle.OperationalError, cx_Oracle.DatabaseError, cx_Oracle.InterfaceError), ex:
-            if "Oracle Client library" in str(ex):
-                msg = re.sub(r"DPI-\d+:\s+", "", str(ex))
+        except (cx_Oracle.OperationalError, cx_Oracle.DatabaseError, cx_Oracle.InterfaceError) as ex:
+            if "Oracle Client library" in getSafeExString(ex):
+                msg = re.sub(r"DPI-\d+:\s+", "", getSafeExString(ex))
                 msg = re.sub(r': ("[^"]+")', r" (\g<1>)", msg)
                 msg = re.sub(r". See (http[^ ]+)", r'. See "\g<1>"', msg)
                 raise SqlmapConnectionException(msg)
 
             try:
                 self.connector = cx_Oracle.connect(dsn=self.__dsn, user=self.user, password=self.password)
-            except (cx_Oracle.OperationalError, cx_Oracle.DatabaseError, cx_Oracle.InterfaceError), msg:
-                raise SqlmapConnectionException(msg)
+            except (cx_Oracle.OperationalError, cx_Oracle.DatabaseError, cx_Oracle.InterfaceError) as ex:
+                raise SqlmapConnectionException(ex)
 
         self.initCursor()
         self.printConnected()
@@ -61,8 +62,8 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except cx_Oracle.InterfaceError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % msg)
+        except cx_Oracle.InterfaceError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex))
             return None
 
     def execute(self, query):
@@ -71,8 +72,8 @@ class Connector(GenericConnector):
         try:
             self.cursor.execute(utf8encode(query))
             retVal = True
-        except cx_Oracle.DatabaseError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % msg)
+        except cx_Oracle.DatabaseError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex))
 
         self.connector.commit()
 

plugins/dbms/postgresql/connector.py

@@ -13,6 +13,7 @@ try:
 except:
     pass
 
+from lib.core.common import getSafeExString
 from lib.core.data import logger
 from lib.core.exception import SqlmapConnectionException
 from plugins.generic.connector import Connector as GenericConnector
@@ -36,8 +37,8 @@ class Connector(GenericConnector):
 
         try:
             self.connector = psycopg2.connect(host=self.hostname, user=self.user, password=self.password, database=self.db, port=self.port)
-        except psycopg2.OperationalError, msg:
-            raise SqlmapConnectionException(msg)
+        except psycopg2.OperationalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.connector.set_client_encoding('UNICODE')
 
@@ -47,8 +48,8 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except psycopg2.ProgrammingError, msg:
-            logger.warn(msg)
+        except psycopg2.ProgrammingError as ex:
+            logger.warn(getSafeExString(ex))
             return None
 
     def execute(self, query):
@@ -57,10 +58,10 @@ class Connector(GenericConnector):
         try:
             self.cursor.execute(query)
             retVal = True
-        except (psycopg2.OperationalError, psycopg2.ProgrammingError), msg:
-            logger.warn(("(remote) %s" % msg).strip())
-        except psycopg2.InternalError, msg:
-            raise SqlmapConnectionException(msg)
+        except (psycopg2.OperationalError, psycopg2.ProgrammingError) as ex:
+            logger.warn(("(remote) '%s'" % getSafeExString(ex)).strip())
+        except psycopg2.InternalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.connector.commit()
 

plugins/dbms/sqlite/connector.py

@@ -12,6 +12,7 @@ except:
 
 import logging
 
+from lib.core.common import getSafeExString
 from lib.core.convert import utf8encode
 from lib.core.data import conf
 from lib.core.data import logger
@@ -45,7 +46,7 @@ class Connector(GenericConnector):
             cursor.execute("SELECT * FROM sqlite_master")
             cursor.close()
 
-        except (self.__sqlite.DatabaseError, self.__sqlite.OperationalError), msg:
+        except (self.__sqlite.DatabaseError, self.__sqlite.OperationalError):
             warnMsg = "unable to connect using SQLite 3 library, trying with SQLite 2"
             logger.warn(warnMsg)
 
@@ -59,8 +60,8 @@ class Connector(GenericConnector):
 
                 self.__sqlite = sqlite
                 self.connector = self.__sqlite.connect(database=self.db, check_same_thread=False, timeout=conf.timeout)
-            except (self.__sqlite.DatabaseError, self.__sqlite.OperationalError), msg:
-                raise SqlmapConnectionException(msg[0])
+            except (self.__sqlite.DatabaseError, self.__sqlite.OperationalError) as ex:
+                raise SqlmapConnectionException(getSafeExString(ex))
 
         self.initCursor()
         self.printConnected()
@@ -68,17 +69,17 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except self.__sqlite.OperationalError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % msg[0])
+        except self.__sqlite.OperationalError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex))
             return None
 
     def execute(self, query):
         try:
             self.cursor.execute(utf8encode(query))
-        except self.__sqlite.OperationalError, msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % msg[0])
-        except self.__sqlite.DatabaseError, msg:
-            raise SqlmapConnectionException(msg[0])
+        except self.__sqlite.OperationalError as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex))
+        except self.__sqlite.DatabaseError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         self.connector.commit()
 

plugins/dbms/sybase/connector.py

@@ -13,6 +13,7 @@ except:
 
 import logging
 
+from lib.core.common import getSafeExString
 from lib.core.convert import utf8encode
 from lib.core.data import conf
 from lib.core.data import logger
@@ -41,8 +42,8 @@ class Connector(GenericConnector):
 
         try:
             self.connector = pymssql.connect(host="%s:%d" % (self.hostname, self.port), user=self.user, password=self.password, database=self.db, login_timeout=conf.timeout, timeout=conf.timeout)
-        except (pymssql.Error, _mssql.MssqlDatabaseException), msg:
-            raise SqlmapConnectionException(msg)
+        except (pymssql.Error, _mssql.MssqlDatabaseException) as ex:
+            raise SqlmapConnectionException(ex)
         except ValueError:
             raise SqlmapConnectionException
 
@@ -52,8 +53,8 @@ class Connector(GenericConnector):
     def fetchall(self):
         try:
             return self.cursor.fetchall()
-        except (pymssql.Error, _mssql.MssqlDatabaseException), msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % str(msg).replace("\n", " "))
+        except (pymssql.Error, _mssql.MssqlDatabaseException) as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex).replace("\n", " "))
             return None
 
     def execute(self, query):
@@ -62,10 +63,10 @@ class Connector(GenericConnector):
         try:
             self.cursor.execute(utf8encode(query))
             retVal = True
-        except (pymssql.OperationalError, pymssql.ProgrammingError), msg:
-            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) %s" % str(msg).replace("\n", " "))
-        except pymssql.InternalError, msg:
-            raise SqlmapConnectionException(msg)
+        except (pymssql.OperationalError, pymssql.ProgrammingError) as ex:
+            logger.log(logging.WARN if conf.dbmsHandler else logging.DEBUG, "(remote) '%s'" % getSafeExString(ex).replace("\n", " "))
+        except pymssql.InternalError as ex:
+            raise SqlmapConnectionException(getSafeExString(ex))
 
         return retVal
 

plugins/dbms/sybase/enumeration.py

@@ -128,7 +128,7 @@ class Enumeration(GenericEnumeration):
         for db in dbs:
             dbs[dbs.index(db)] = safeSQLIdentificatorNaming(db)
 
-        dbs = filter(None, dbs)
+        dbs = [_ for _ in dbs if _]
 
         infoMsg = "fetching tables for database"
         infoMsg += "%s: %s" % ("s" if len(dbs) > 1 else "", ", ".join(db if isinstance(db, basestring) else db[0] for db in sorted(dbs)))

plugins/generic/connector.py

@@ -52,8 +52,8 @@ class Connector:
                 self.cursor.close()
             if self.connector:
                 self.connector.close()
-        except Exception, msg:
-            logger.debug(msg)
+        except Exception as ex:
+            logger.debug(ex)
         finally:
             self.closed()
 

plugins/generic/custom.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 import re
 import sys
 
@@ -70,7 +72,7 @@ class Custom:
 
                 output = NULL
 
-        except SqlmapNoneDataException, ex:
+        except SqlmapNoneDataException as ex:
             logger.warn(ex)
 
         return output
@@ -90,11 +92,11 @@ class Custom:
                 query = getUnicode(query, encoding=sys.stdin.encoding)
                 query = query.strip("; ")
             except KeyboardInterrupt:
-                print
+                print()
                 errMsg = "user aborted"
                 logger.error(errMsg)
             except EOFError:
-                print
+                print()
                 errMsg = "exit"
                 logger.error(errMsg)
                 break
@@ -128,8 +130,8 @@ class Custom:
 
             snippet = getSQLSnippet(Backend.getDbms(), filename)
 
-            if snippet and all(query.strip().upper().startswith("SELECT") for query in filter(None, snippet.split(';' if ';' in snippet else '\n'))):
-                for query in filter(None, snippet.split(';' if ';' in snippet else '\n')):
+            if snippet and all(query.strip().upper().startswith("SELECT") for query in (_ for _ in snippet.split(';' if ';' in snippet else '\n') if _)):
+                for query in (_ for _ in snippet.split(';' if ';' in snippet else '\n') if _):
                     query = query.strip()
                     if query:
                         conf.dumper.query(query, self.sqlQuery(query))

plugins/generic/databases.py

@@ -179,7 +179,7 @@ class Databases:
             kb.data.cachedDbs.sort()
 
         if kb.data.cachedDbs:
-            kb.data.cachedDbs = filter(None, list(set(flattenValue(kb.data.cachedDbs))))
+            kb.data.cachedDbs = [_ for _ in set(flattenValue(kb.data.cachedDbs)) if _]
 
         return kb.data.cachedDbs
 
@@ -285,7 +285,7 @@ class Databases:
                     values = inject.getValue(query, blind=False, time=False)
 
             if not isNoneValue(values):
-                values = filter(None, arrayizeValue(values))
+                values = [_ for _ in arrayizeValue(values) if _]
 
                 if len(values) > 0 and not isListLike(values[0]):
                     values = [(dbs[0], _) for _ in values]
@@ -415,7 +415,7 @@ class Databases:
                 kb.data.cachedTables[db] = sorted(tables) if tables else tables
 
         if kb.data.cachedTables:
-            for db in kb.data.cachedTables.keys():
+            for db in kb.data.cachedTables:
                 kb.data.cachedTables[db] = list(set(kb.data.cachedTables[db]))
 
         return kb.data.cachedTables
@@ -462,7 +462,7 @@ class Databases:
         for col in colList:
             colList[colList.index(col)] = safeSQLIdentificatorNaming(col)
 
-        colList = filter(None, colList)
+        colList = [_ for _ in colList if _]
 
         if conf.tbl:
             if Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2, DBMS.HSQLDB, DBMS.H2):
@@ -622,7 +622,13 @@ class Databases:
                             index += 1
 
                 if Backend.isDbms(DBMS.SQLITE):
-                    parseSqliteTableSchema(unArrayizeValue(values))
+                    if dumpMode and colList:
+                        if conf.db not in kb.data.cachedColumns:
+                            kb.data.cachedColumns[conf.db] = {}
+                        kb.data.cachedColumns[conf.db][safeSQLIdentificatorNaming(conf.tbl, True)] = dict((_, None) for _ in colList)
+                    else:
+                        parseSqliteTableSchema(unArrayizeValue(values))
+
                 elif not isNoneValue(values):
                     table = {}
                     columns = {}
@@ -718,9 +724,15 @@ class Databases:
                     query += condQuery
 
                 elif Backend.isDbms(DBMS.SQLITE):
-                    query = rootQuery.blind.query % unsafeSQLIdentificatorNaming(tbl)
-                    value = unArrayizeValue(inject.getValue(query, union=False, error=False))
-                    parseSqliteTableSchema(value)
+                    if dumpMode and colList:
+                        if conf.db not in kb.data.cachedColumns:
+                            kb.data.cachedColumns[conf.db] = {}
+                        kb.data.cachedColumns[conf.db][safeSQLIdentificatorNaming(conf.tbl, True)] = dict((_, None) for _ in colList)
+                    else:
+                        query = rootQuery.blind.query % unsafeSQLIdentificatorNaming(tbl)
+                        value = unArrayizeValue(inject.getValue(query, union=False, error=False))
+                        parseSqliteTableSchema(unArrayizeValue(value))
+
                     return kb.data.cachedColumns
 
                 table = {}
@@ -898,6 +910,7 @@ class Databases:
         else:
             query = "SELECT %s FROM %s.%s" % (queries[Backend.getIdentifiedDbms()].count.query % '*', safeSQLIdentificatorNaming(db), safeSQLIdentificatorNaming(table, True))
 
+        query = agent.whereQuery(query)
         count = inject.getValue(query, expected=EXPECTED.INT, charsetType=CHARSET_TYPE.DIGITS)
 
         if isNumPosStrValue(count):

plugins/generic/entries.py

@@ -139,7 +139,7 @@ class Entries:
                     continue
 
                 columns = kb.data.cachedColumns[safeSQLIdentificatorNaming(conf.db)][safeSQLIdentificatorNaming(tbl, True)]
-                colList = sorted(filter(None, columns.keys()))
+                colList = sorted(column for column in columns if column)
 
                 if conf.exclude:
                     colList = [_ for _ in colList if _ not in conf.exclude.split(',')]
@@ -444,13 +444,13 @@ class Entries:
                                                         "db": safeSQLIdentificatorNaming(conf.db)}
                     try:
                         attackDumpedTable()
-                    except (IOError, OSError), ex:
+                    except (IOError, OSError) as ex:
                         errMsg = "an error occurred while attacking "
                         errMsg += "table dump ('%s')" % getSafeExString(ex)
                         logger.critical(errMsg)
                     conf.dumper.dbTableValues(kb.data.dumpedTable)
 
-            except SqlmapConnectionException, ex:
+            except SqlmapConnectionException as ex:
                 errMsg = "connection exception detected in dumping phase "
                 errMsg += "('%s')" % getSafeExString(ex)
                 logger.critical(errMsg)
@@ -517,7 +517,7 @@ class Entries:
         choice = readInput(message, default='a')
 
         if not choice or choice in ('a', 'A'):
-            dumpFromDbs = dbs.keys()
+            dumpFromDbs = list(dbs.keys())
         elif choice in ('q', 'Q'):
             return
         else:
@@ -553,7 +553,7 @@ class Entries:
                     continue
 
                 conf.tbl = table
-                colList = filter(None, sorted(columns))
+                colList = [_ for _ in columns if _]
 
                 if conf.exclude:
                     colList = [_ for _ in colList if _ not in conf.exclude.split(',')]
@@ -584,7 +584,7 @@ class Entries:
         choice = readInput(message, default='a')
 
         if not choice or choice.lower() == 'a':
-            dumpFromDbs = tables.keys()
+            dumpFromDbs = list(tables.keys())
         elif choice.lower() == 'q':
             return
         else:

plugins/generic/filesystem.py

@@ -28,6 +28,7 @@ from lib.core.enums import CHARSET_TYPE
 from lib.core.enums import EXPECTED
 from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapUndefinedMethod
+from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.core.settings import UNICODE_ENCODING
 from lib.request import inject
 
@@ -37,7 +38,7 @@ class Filesystem:
     """
 
     def __init__(self):
-        self.fileTblName = "sqlmapfile"
+        self.fileTblName = "%sfile" % TAKEOVER_TABLE_PREFIX
         self.tblField = "data"
 
     def _checkFileLength(self, localFile, remoteFile, fileRead=False):

plugins/generic/search.py

@@ -273,7 +273,7 @@ class Search:
                     dbName = "SQLite" if Backend.isDbms(DBMS.SQLITE) else "Firebird"
                     foundTbls["%s%s" % (dbName, METADB_SUFFIX)] = []
 
-                for db in foundTbls.keys():
+                for db in foundTbls:
                     db = safeSQLIdentificatorNaming(db)
 
                     infoMsg = "fetching number of table"
@@ -326,7 +326,7 @@ class Search:
                             foundTbl = safeSQLIdentificatorNaming(foundTbl, True)
                             foundTbls[db].append(foundTbl)
 
-        for db in foundTbls.keys():
+        for db in list(foundTbls.keys()):
             if isNoneValue(foundTbls[db]):
                 del foundTbls[db]
 

plugins/generic/takeover.py

@@ -24,6 +24,7 @@ from lib.core.exception import SqlmapNotVulnerableException
 from lib.core.exception import SqlmapSystemException
 from lib.core.exception import SqlmapUndefinedMethod
 from lib.core.exception import SqlmapUnsupportedDBMSException
+from lib.core.settings import TAKEOVER_TABLE_PREFIX
 from lib.takeover.abstraction import Abstraction
 from lib.takeover.icmpsh import ICMPsh
 from lib.takeover.metasploit import Metasploit
@@ -37,7 +38,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry, Miscellaneous):
     """
 
     def __init__(self):
-        self.cmdTblName = "sqlmapoutput"
+        self.cmdTblName = ("%soutput" % TAKEOVER_TABLE_PREFIX)
         self.tblField = "data"
 
         Abstraction.__init__(self)
@@ -140,7 +141,7 @@ class Takeover(Abstraction, Metasploit, ICMPsh, Registry, Miscellaneous):
                 try:
                     with open(filename, "wb") as f:
                         f.write("1")
-                except IOError, ex:
+                except IOError as ex:
                     errMsg = "there has been a file opening/writing error "
                     errMsg += "for filename '%s' ('%s')" % (filename, getSafeExString(ex))
                     raise SqlmapSystemException(errMsg)

plugins/generic/users.py

@@ -171,7 +171,7 @@ class Users:
         else:
             users = []
 
-        users = filter(None, users)
+        users = [_ for _ in users if _]
 
         if any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) or conf.direct:
             if Backend.isDbms(DBMS.MSSQL) and Backend.isVersionWithin(("2005", "2008")):
@@ -370,7 +370,7 @@ class Users:
         else:
             users = []
 
-        users = filter(None, users)
+        users = [_ for _ in users if _]
 
         # Set containing the list of DBMS administrators
         areAdmins = set()

sqlmap.conf

@@ -725,6 +725,10 @@ outputDir =
 # Valid: True or False
 parseErrors = False
 
+# Redump entries having unknown character marker (?).
+# Valid: True or False
+repair = False
+
 # Regular expression for filtering targets from provided Burp.
 # or WebScarab proxy log.
 # Example: (google|yahoo)

sqlmap.py

@@ -5,6 +5,8 @@ Copyright (c) 2006-2019 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from __future__ import print_function
+
 try:
     import sys
 
@@ -51,6 +53,7 @@ try:
     from lib.core.data import kb
     from lib.core.common import unhandledExceptionMessage
     from lib.core.common import MKSTEMP_PREFIX
+    from lib.core.common import setColor
     from lib.core.exception import SqlmapBaseException
     from lib.core.exception import SqlmapShellQuitException
     from lib.core.exception import SqlmapSilentQuitException
@@ -188,10 +191,10 @@ def main():
         raise SystemExit
 
     except KeyboardInterrupt:
-        print
+        print()
 
     except EOFError:
-        print
+        print()
 
         errMsg = "exit"
         logger.error(errMsg)
@@ -200,7 +203,7 @@ def main():
         pass
 
     except:
-        print
+        print()
         errMsg = unhandledExceptionMessage()
         excMsg = traceback.format_exc()
         valid = checkIntegrity()
@@ -210,13 +213,13 @@ def main():
             errMsg += "You should retrieve the latest development version from official GitHub "
             errMsg += "repository at '%s'" % GIT_PAGE
             logger.critical(errMsg)
-            print
+            print()
             dataToStdout(excMsg)
             raise SystemExit
 
         elif any(_ in excMsg for _ in ("tamper/", "waf/")):
             logger.critical(errMsg)
-            print
+            print()
             dataToStdout(excMsg)
             raise SystemExit
 
@@ -225,16 +228,21 @@ def main():
             logger.critical(errMsg)
             raise SystemExit
 
-        elif "MemoryError" in excMsg:
+        elif any(_ in excMsg for _ in ("MemoryError", "Cannot allocate memory")):
             errMsg = "memory exhaustion detected"
             logger.critical(errMsg)
             raise SystemExit
 
-        elif any(_ in excMsg for _ in ("No space left", "Disk quota exceeded")):
+        elif any(_ in excMsg for _ in ("No space left", "Disk quota exceeded", "Disk full while accessing")):
             errMsg = "no space left on output device"
             logger.critical(errMsg)
             raise SystemExit
 
+        elif any(_ in excMsg for _ in ("The paging file is too small",)):
+            errMsg = "no space left for paging file"
+            logger.critical(errMsg)
+            raise SystemExit
+
         elif all(_ in excMsg for _ in ("No such file", "_'", "self.get_prog_name()")):
             errMsg = "corrupted installation detected ('%s'). " % excMsg.strip().split('\n')[-1]
             errMsg += "You should retrieve the latest development version from official GitHub "
@@ -332,8 +340,7 @@ def main():
             logger.critical("%s\n%s" % (errMsg, excMsg))
         else:
             logger.critical(errMsg)
-            kb.stickyLevel = logging.CRITICAL
-            dataToStdout(excMsg)
+            dataToStdout("%s\n" % setColor(excMsg.strip(), level=logging.CRITICAL))
             createGithubIssue(errMsg, excMsg)
 
     finally:

thirdparty/beautifulsoup/beautifulsoup.py

@@ -370,7 +370,7 @@ class PageElement(object):
         g = generator()
         while True:
             try:
-                i = g.next()
+                i = next(g)
             except StopIteration:
                 break
             if i:
@@ -470,7 +470,7 @@ class NavigableString(unicode, PageElement):
         if attr == 'string':
             return self
         else:
-            raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr)
+            raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, attr))
 
     def __unicode__(self):
         return str(self).decode(DEFAULT_OUTPUT_ENCODING)
@@ -668,7 +668,7 @@ class Tag(PageElement):
             return self.find(tag[:-3])
         elif tag.find('__') != 0:
             return self.find(tag)
-        raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag)
+        raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__, tag))
 
     def __eq__(self, other):
         """Returns true iff this tag has the same name, the same attributes,
@@ -974,8 +974,8 @@ class SoupStrainer:
             if self._matches(markup, self.text):
                 found = markup
         else:
-            raise Exception, "I don't know how to match against a %s" \
-                  % markup.__class__
+            raise Exception("I don't know how to match against a %s" \
+                  % markup.__class__)
         return found
 
     def _matches(self, markup, matchAgainst):
@@ -1836,7 +1836,7 @@ class UnicodeDammit:
             u = self._toUnicode(markup, proposed)
             self.markup = u
             self.originalEncoding = proposed
-        except Exception, e:
+        except Exception as e:
             # print "That didn't work!"
             # print e
             return None

thirdparty/clientform/clientform.py

@@ -1100,7 +1100,7 @@ def _ParseFileEx(file, base_uri,
         data = file.read(CHUNK)
         try:
             fp.feed(data)
-        except ParseError, e:
+        except ParseError as e:
             e.base_uri = base_uri
             raise
         if len(data) != CHUNK: break
@@ -1141,7 +1141,7 @@ def _ParseFileEx(file, base_uri,
     for form in forms:
         try:
             form.fixup()
-        except AttributeError, ex:
+        except AttributeError as ex:
             if not any(_ in str(ex) for _ in ("is disabled", "is readonly")):
                 raise
     return forms
@@ -2902,7 +2902,7 @@ class HTMLForm:
         control = self.find_control(name)
         try:
             control.value = value
-        except AttributeError, e:
+        except AttributeError as e:
             raise ValueError(str(e))
 
     def get_value(self,

thirdparty/colorama/ansitowin32.py

@@ -180,7 +180,7 @@ class AnsiToWin32(object):
     def _write(self, text, retry=5):
         try:
             self.wrapped.write(text)
-        except IOError, err:
+        except IOError as err:
             if not (err.errno == 0 and retry > 0):
                 raise
             self._write(text, retry-1)

thirdparty/gprof2dot/gprof2dot.py

@@ -695,7 +695,7 @@ class XmlTokenizer:
             self.final = len(data) < size
             try:
                 self.parser.Parse(data, self.final)
-            except xml.parsers.expat.ExpatError, e:
+            except xml.parsers.expat.ExpatError as e:
                 #if e.code == xml.parsers.expat.errors.XML_ERROR_NO_ELEMENTS:
                 if e.code == 3:
                     pass
@@ -732,7 +732,7 @@ class XmlParser(Parser):
         self.consume()
 
     def consume(self):
-        self.token = self.tokenizer.next()
+        self.token = next(self.tokenizer)
 
     def match_element_start(self, name):
         return self.token.type == XML_ELEMENT_START and self.token.name_or_data == name
@@ -1719,7 +1719,7 @@ class XPerfParser(Parser):
             lineterminator = '\r\n',
             quoting = csv.QUOTE_NONE)
         it = iter(reader)
-        row = reader.next()
+        row = next(reader)
         self.parse_header(row)
         for row in it:
             self.parse_row(row)

thirdparty/keepalive/keepalive.py

@@ -238,7 +238,7 @@ class KeepAliveHandler:
                 self._cm.add(host, h, 0)
                 self._start_transaction(h, req)
                 r = h.getresponse()
-        except (socket.error, httplib.HTTPException), err:
+        except (socket.error, httplib.HTTPException) as err:
             raise urllib2.URLError(err)
 
         if DEBUG: DEBUG.info("STATUS: %s, %s", r.status, r.reason)
@@ -323,7 +323,7 @@ class KeepAliveHandler:
                     h.putrequest(req.get_method() or 'GET', req.selector, skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
                 else:
                     h.putrequest(req.get_method() or 'GET', req.get_selector(), skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
-        except (socket.error, httplib.HTTPException), err:
+        except (socket.error, httplib.HTTPException) as err:
             raise urllib2.URLError(err)
 
         if not req.headers.has_key('Connection'):
@@ -495,7 +495,7 @@ def error_handler(url):
             fo.close()
             try: status, reason = fo.status, fo.reason
             except AttributeError: status, reason = None, None
-        except IOError, e:
+        except IOError as e:
             print "  EXCEPTION: %s" % e
             raise
         else:
@@ -613,7 +613,7 @@ def test_timeout(url):
 def test(url, N=10):
     print "checking error hander (do this on a non-200)"
     try: error_handler(url)
-    except IOError, e:
+    except IOError as e:
         print "exiting - exception will prevent further tests"
         sys.exit()
     print

thirdparty/multipart/multipartpost.py

@@ -59,7 +59,7 @@ class MultipartPostHandler(urllib2.BaseHandler):
                         v_vars.append((key, value))
             except TypeError:
                 systype, value, traceback = sys.exc_info()
-                raise SqlmapDataException, "not a valid non-string sequence or mapping object", traceback
+                raise SqlmapDataException("not a valid non-string sequence or mapping object '%s'" % traceback)
 
             if len(v_files) == 0:
                 data = urllib.urlencode(v_vars, doseq)

thirdparty/odict/odict.py

@@ -609,7 +609,7 @@ class _OrderedDict(dict):
         TypeError: pop expected at most 2 arguments, got 3
         """
         if len(args) > 1:
-            raise TypeError, ('pop expected at most 2 arguments, got %s' %
+            raise TypeError('pop expected at most 2 arguments, got %s' %
                 (len(args) + 1))
         if key in self:
             val = self[key]
@@ -621,7 +621,7 @@ class _OrderedDict(dict):
                 raise KeyError(key)
         return val
 
-    def popitem(self, i=-1):
+    def popitem(self, last=True):
         """
         Delete and return an item specified by index, not a random one as in
         dict. The index is -1 by default (the last item).
@@ -643,6 +643,7 @@ class _OrderedDict(dict):
         if not self._sequence:
             raise KeyError('popitem(): dictionary is empty')
         try:
+            i = -1 if last else 0
             key = self._sequence[i]
         except IndexError:
             raise IndexError('popitem(): index %s not valid' % i)