[pull] master from sqlmapproject:master

data/txt/sha256sums.txt

@@ -168,40 +168,40 @@ e376093d4f6e42ee38b050af329179df9c1c136b7667b2f1cb559f5d4b69ebd9  lib/controller
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/controller/__init__.py
 2a96190ced25d8929861b13866101812fcadf5cac23dd1dd4b29b1a915918769  lib/core/agent.py
 1da4ec9cd9b67c8b54e4a3d314f8237d58778d8f3a00bc26a1e0540294dca30f  lib/core/bigarray.py
-3f1fbe7c15b929db81097d10c2897eb1e711f158de51ef9041b4c74e11f3f1d8  lib/core/common.py
+af24159b8ca5b8fe5e13cdfdedc2a758a2f4883361a601e0a550127cff368b3a  lib/core/common.py
 a6397b10de7ae7c56ed6b0fa3b3c58eb7a9dbede61bf93d786e73258175c981e  lib/core/compat.py
 a9997e97ebe88e0bf7efcf21e878bc5f62c72348e5aba18f64d6861390a4dcf2  lib/core/convert.py
 c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.py
 e396b7971d38896e0e20b973a3a6a3fbc3171d080a21bc6e66a65bee452fd69c  lib/core/datatype.py
 e18c0c2c5a57924a623792a48bfd36e98d9bc085f6db61a95fc0dc8a3bcedc0c  lib/core/decorators.py
 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d  lib/core/defaults.py
 76756ce056dd2e5363e5e1d2b2b4c3f3b36ff8c8ac215e83ee3b8d535ea6b741  lib/core/dicts.py
-143f0b5b30e0f24b3527945879497de1542ba1aee9e6f0f98865e626c4c9317e  lib/core/dump.py
+a033f92d136c707a25927c2383125ddb004d4283db62c004dcd67c3fc242bb1c  lib/core/dump.py
 1abf1edeacb85eaf5cffd35fcbde4eee2da6f5fc722a8dc1f9287fb55d138418  lib/core/enums.py
 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4  lib/core/exception.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/core/__init__.py
 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad  lib/core/log.py
 02a2264324caa249154e024a01bcd7cc40dbca4d647d5d10a50654b4415a6d77  lib/core/optiondict.py
-6576d40a66fa7871d3498c193f4e1e50a9fa9a380005d019c5c2266c1dc31c21  lib/core/option.py
+4dc6383b5d9cfff5b1bf8ffa8b0b02a7dbd8df8624b76030f8925c2dbdee9bf9  lib/core/option.py
 8171f6ee33e7742f06bb3014a28324496374beddee7b378ace10a26414a97762  lib/core/patch.py
 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec  lib/core/profiling.py
 03db48f02c3d07a047ddb8fe33a757b6238867352d8ddda2a83e4fec09a98d04  lib/core/readlineng.py
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 3574639db4942d16a2dc0a2f04bb7c0913c40c3862b54d34c44075a760e0c194  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-4267b95315e5351fa06aa27c883e187186adc3709e50acedf10079fd611dbb8d  lib/core/settings.py
+2503c9b9eca74ae3c2a6b245504046b623bf0e8a8f433105f41f61fa626aec48  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 d35650179816193164a5f177102f18379dfbe6bb6d40fbb67b78d907b41c8038  lib/core/target.py
 ddf8c5a3dbebd6cdf8b8ba4417e36652d1e040f025175cb6487f1aebc0208836  lib/core/testing.py
 cf4dca323645d623109a82277a8e8a63eb9abb3fff6c8a57095eb171c1ef91b3  lib/core/threads.py
 b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02  lib/core/unescaper.py
 10719f5ca450610ad28242017b2d8a77354ca357ffa26948c5f62d20cac29a8b  lib/core/update.py
-9ed5a0aef84f55d42894a006ff3616e8ee388a55790b04d968c80d1470c6d3bc  lib/core/wordlist.py
+ec11fd5a3f4efd10a1cae288157ac6eb6fb75da4666d76d19f6adf74ac338b5a  lib/core/wordlist.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/__init__.py
 54bfd31ebded3ffa5848df1c644f196eb704116517c7a3d860b5d081e984d821  lib/parse/banner.py
 a9f10a558684778bdb00d446cb88967fc1bfd413ae6a5f4bd582b3ea442baa87  lib/parse/cmdline.py
-cac08047db1168b24174496268af811cee555e7c3fdd528ef8d895ec49b91d36  lib/parse/configfile.py
+02d82e4069bd98c52755417f8b8e306d79945672656ac24f1a45e7a6eff4b158  lib/parse/configfile.py
 c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/handler.py
 97361d481a97b600a3086b7f228f54ffa68a78df8b63b76bfaa5495d66770b63  lib/parse/headers.py
 1ad9054cd8476a520d4e2c141085ae45d94519df5c66f25fac41fe7d552ab952  lib/parse/html.py
@@ -212,7 +212,7 @@ d2e771cdacef25ee3fdc0e0355b92e7cd1b68f5edc2756ffc19f75d183ba2c73  lib/parse/payl
 a1c638493ecdc5194db7186bbfed815c6eed2344f2607cac8c9fa50534824266  lib/request/basic.py
 bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e  lib/request/chunkedhandler.py
 2daf0ce19eacda64687f441c90ef8da51714c3e8947c993ba08fb4ecdc4f5287  lib/request/comparison.py
-626bb6f3316a906a4629c0feb8ecbbcf473fb59e5bc532603c35b6b8f63f1deb  lib/request/connect.py
+f83140c85be7f572f83c4ab4279fa1d8601243210cdfe4a44b2fc218befbcffd  lib/request/connect.py
 8e06682280fce062eef6174351bfebcb6040e19976acff9dc7b3699779783498  lib/request/direct.py
 cf019248253a5d7edb7bc474aa020b9e8625d73008a463c56ba2b539d7f2d8ec  lib/request/dns.py
 f56fc33251bd6214e3a6316c8f843eb192b2996aa84bd4c3e98790fdcf6e8cf0  lib/request/httpshandler.py
@@ -231,7 +231,7 @@ f522436fbd14bdab090a1d305fcac0361800cb8e36c8cbcb47933298376a71e0  lib/takeover/r
 f6e5d6e2ff368fa39943b2302982f33c47eb9a12d01419bef50fcf934b2bce34  lib/takeover/udf.py
 4b5ff4fcfa25454e6a93600d32af42a69bd59151639f569c01920c8610a99656  lib/takeover/web.py
 14179e5273378ec8d63660a87c5cb07a42b61a6fceb7f3bb494a7b5ce10ce2cb  lib/takeover/xp_cmdshell.py
-e29a4054bb5285ba63ae2c0b2c05c8a15b80ec5719ddc4559baa0772d70f24b9  lib/techniques/blind/inference.py
+69928272eed889033e106527f88454dc844bfbb375fcf7c22d5f76ee30c62c9b  lib/techniques/blind/inference.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/blind/__init__.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/dns/__init__.py
 3df9839fb92a81d46b6194d7adacb43f391efb78b071783c132e8d596ecbfaf1  lib/techniques/dns/test.py
@@ -241,10 +241,10 @@ f552b6140d4069be6a44792a08f295da8adabc1c4bb6a5e100f222f87144ca9d  lib/techniques
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/__init__.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/union/__init__.py
 30cae858e2a5a75b40854399f65ad074e6bb808d56d5ee66b94d4002dc6e101b  lib/techniques/union/test.py
-77d2404e5b23fa419113de963bf9eb207582d1548fb0d3f36876d198785c88c3  lib/techniques/union/use.py
+a17c1d201bd084de0093254bcd303aa859399891de13a7259e8c200e98294efb  lib/techniques/union/use.py
 67dff80a17503b91c8ff93788ccc037b6695aa18b0793894b42488cbb21c4c83  lib/utils/api.py
 ea5e14f8c9d74b0fb17026b14e3fb70ee90e4046e51ab2c16652d86b3ca9b949  lib/utils/brute.py
-3fa1b9fd57ff47c6a283e8381bf70259dce57bb2327f99d8cb56450f1acf2d46  lib/utils/crawler.py
+da5bcbcda3f667582adf5db8c1b5d511b469ac61b55d387cec66de35720ed718  lib/utils/crawler.py
 a94958be0ec3e9d28d8171813a6a90655a9ad7e6aa33c661e8d8ebbfcf208dbb  lib/utils/deps.py
 51cfab194cd5b6b24d62706fb79db86c852b9e593f4c55c15b35f175e70c9d75  lib/utils/getch.py
 853c3595e1d2efc54b8bfb6ab12c55d1efc1603be266978e3a7d96d553d91a52  lib/utils/gui.py
@@ -255,7 +255,7 @@ dc68b7fdb2ae0e958f2a553984a94a06832d216a3209f632ad9ff53f17554524  lib/utils/http
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/utils/__init__.py
 22ba65391b0a73b1925e5becf8ddab6ba73a196d86e351a2263509aad6676bd7  lib/utils/pivotdumptable.py
 c1dfc3bed0fed9b181f612d1d747955dd2b506dbe99bc9fd481495602371473a  lib/utils/progress.py
-b0f777739420f7207e57d7073fd94c630a456e4c419d440b98ed6521143952dc  lib/utils/purge.py
+27afe211030d06db28df85296bfbf698296c94440904c390cef0ff0c259dbbc5  lib/utils/purge.py
 c853aa08ab24a00a78969408d60684da0ccb33a2a6693492e0acb7c480ffbcd1  lib/utils/safe2bin.py
 2ee72e83500a1bf02fcd942564fca0053a0c46f736286f0c35dd6904e09f4734  lib/utils/search.py
 8258d0f54ad94e6101934971af4e55d5540f217c40ddcc594e2fba837b856d35  lib/utils/sgmllib.py
@@ -487,7 +487,7 @@ eb45fd711efa71ab9d91d815cc8abebc9abc4770311fbb827159008b000f4fc2  plugins/generi
 c6ad39bfd1810413402dedfc275fc805fa13f85fc490e236c1e725bde4e5100b  sqlmapapi.py
 4e993cfe2889bf0f86ad0abafd9a6a25849580284ea279b2115e99707e14bb97  sqlmapapi.yaml
 a40607ce164eb2d21865288d24b863edb1c734b56db857e130ac1aef961c80b9  sqlmap.conf
-e9d3d52d4c0698b956cc0dc92c177d432b1f97c5918f750baa3e737de4ae574b  sqlmap.py
+01457a41833b89b8bcdcae7b241a184bfcfdd8088575506154dd6ea1f4d71186  sqlmap.py
 eb37a88357522fd7ad00d90cdc5da6b57442b4fec49366aadb2944c4fbf8b804  tamper/0eunion.py
 a9785a4c111d6fee2e6d26466ba5efb3b229c00520b26e8024b041553b53efba  tamper/apostrophemask.py
 cf26bc8006519bd25ce06d347f72770cd75b61575cf65e5812274e8ab9392eb4  tamper/apostrophenullencode.py

lib/core/common.py

@@ -1411,7 +1411,7 @@ def parseJson(content):
     """
     This function parses POST_HINT.JSON and POST_HINT.JSON_LIKE content
 
-    >>> parseJson("{'id':1}")["id"] == 1
+    >>> parseJson("{'id':1, 'foo':[2,3,4]}")["id"] == 1
     True
     >>> parseJson('{"id":1}')["id"] == 1
     True
@@ -1429,10 +1429,10 @@ def parseJson(content):
         if quote == '"':
             retVal = json.loads(content)
         elif quote == "'":
-            content = content.replace('"', '\\"')
-            content = content.replace("\\'", BOUNDARY_BACKSLASH_MARKER)
-            content = content.replace("'", '"')
-            content = content.replace(BOUNDARY_BACKSLASH_MARKER, "'")
+            def _(match):
+                return '"%s"' % match.group(1).replace('"', '\\"')
+
+            content = re.sub(r"'((?:[^'\\]|\\.)*)'", _, content)
             retVal = json.loads(content)
     except:
         pass
@@ -2065,7 +2065,7 @@ def getCharset(charsetType=None):
 
     # Digits
     elif charsetType == CHARSET_TYPE.DIGITS:
-        asciiTbl.extend((0, 9))
+        asciiTbl.extend(xrange(0, 10))
         asciiTbl.extend(xrange(47, 58))
 
     # Hexadecimal
@@ -2465,7 +2465,7 @@ def getSQLSnippet(dbms, sfile, **variables):
 
     return retVal
 
-def readCachedFileContent(filename, mode="rb"):
+def readCachedFileContent(filename, mode='r'):
     """
     Cached reading of file content (avoiding multiple same file reading)
 
@@ -3609,7 +3609,7 @@ def saveConfig(conf, filename):
 
             config.set(family, option, value)
 
-    with openFile(filename, "wb") as f:
+    with openFile(filename, 'w') as f:
         try:
             config.write(f)
         except IOError as ex:
@@ -3815,6 +3815,7 @@ def openFile(filename, mode='r', encoding=UNICODE_ENCODING, errors="reversible",
     # Reference: https://stackoverflow.com/a/37462452
     if 'b' in mode:
         buffering = 0
+        encoding = None
 
     if filename == STDIN_PIPE_DASH:
         if filename not in kb.cache.content:
@@ -4022,7 +4023,7 @@ def createGithubIssue(errMsg, excMsg):
             logger.info(infoMsg)
 
             try:
-                with openFile(paths.GITHUB_HISTORY, "a+b") as f:
+                with openFile(paths.GITHUB_HISTORY, "a+") as f:
                     f.write("%s\n" % key)
             except:
                 pass
@@ -5103,7 +5104,7 @@ def resetCookieJar(cookieJar):
                 os.close(handle)
 
                 # Reference: http://www.hashbangcode.com/blog/netscape-http-cooke-file-parser-php-584.html
-                with openFile(filename, "w+b") as f:
+                with openFile(filename, "w+") as f:
                     f.write("%s\n" % NETSCAPE_FORMAT_HEADER_COOKIES)
                     for line in lines:
                         _ = line.split("\t")

lib/core/dump.py

@@ -110,7 +110,7 @@ def setOutputFile(self):
 
         self._outputFile = os.path.join(conf.outputPath, "log")
         try:
-            self._outputFP = openFile(self._outputFile, "ab" if not conf.flushSession else "wb")
+            self._outputFP = openFile(self._outputFile, 'a' if not conf.flushSession else 'w')
         except IOError as ex:
             errMsg = "error occurred while opening log file ('%s')" % getSafeExString(ex)
             raise SqlmapGenericException(errMsg)
@@ -453,7 +453,7 @@ def dbTableValues(self, tableValues):
             dumpFileName = conf.dumpFile or os.path.join(dumpDbPath, re.sub(r'[\\/]', UNSAFE_DUMP_FILEPATH_REPLACEMENT, "%s.%s" % (unsafeSQLIdentificatorNaming(table), conf.dumpFormat.lower())))
             if not checkFile(dumpFileName, False):
                 try:
-                    openFile(dumpFileName, "w+b").close()
+                    openFile(dumpFileName, "w+").close()
                 except SqlmapSystemException:
                     raise
                 except:
@@ -481,7 +481,7 @@ def dbTableValues(self, tableValues):
                         else:
                             count += 1
 
-            dumpFP = openFile(dumpFileName, "wb" if not appendToFile else "ab", buffering=DUMP_FILE_BUFFER_SIZE)
+            dumpFP = openFile(dumpFileName, 'w' if not appendToFile else 'a', buffering=DUMP_FILE_BUFFER_SIZE)
 
         count = int(tableValues["__infos__"]["count"])
         if count > TRIM_STDOUT_DUMP_SIZE:

lib/core/option.py

@@ -753,7 +753,7 @@ def _listTamperingFunctions():
         logger.info(infoMsg)
 
         for script in sorted(glob.glob(os.path.join(paths.SQLMAP_TAMPER_PATH, "*.py"))):
-            content = openFile(script, "rb").read()
+            content = openFile(script, 'r').read()
             match = re.search(r'(?s)__priority__.+"""(.+)"""', content)
             if match:
                 comment = match.group(1).strip()
@@ -1015,8 +1015,8 @@ def _setPostprocessFunctions():
                     handle, filename = tempfile.mkstemp(prefix=MKSTEMP_PREFIX.PREPROCESS, suffix=".py")
                     os.close(handle)
 
-                    openFile(filename, "w+b").write("#!/usr/bin/env\n\ndef postprocess(page, headers=None, code=None):\n    return page, headers, code\n")
-                    openFile(os.path.join(os.path.dirname(filename), "__init__.py"), "w+b").write("pass")
+                    openFile(filename, "w+").write("#!/usr/bin/env\n\ndef postprocess(page, headers=None, code=None):\n    return page, headers, code\n")
+                    openFile(os.path.join(os.path.dirname(filename), "__init__.py"), "w+").write("pass")
 
                     errMsg = "function 'postprocess(page, headers=None, code=None)' "
                     errMsg += "in postprocess script '%s' " % script
@@ -1593,7 +1593,7 @@ def _createHomeDirectories():
                 os.makedirs(directory)
 
             _ = os.path.join(directory, randomStr())
-            open(_, "w+b").close()
+            open(_, "w+").close()
             os.remove(_)
 
             if conf.get("outputDir") and context == "output":
@@ -1623,7 +1623,7 @@ def _createTemporaryDirectory():
 
             _ = os.path.join(conf.tmpDir, randomStr())
 
-            open(_, "w+b").close()
+            open(_, "w+").close()
             os.remove(_)
 
             tempfile.tempdir = conf.tmpDir

lib/core/settings.py

@@ -19,7 +19,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.1.47"
+VERSION = "1.10.1.52"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/core/wordlist.py

@@ -27,6 +27,7 @@ class Wordlist(six.Iterator):
     def __init__(self, filenames, proc_id=None, proc_count=None, custom=None):
         self.filenames = [filenames] if isinstance(filenames, six.string_types) else filenames
         self.fp = None
+        self.zip_file = None
         self.index = 0
         self.counter = -1
         self.current = None
@@ -49,16 +50,16 @@ def adjust(self):
             self.current = self.filenames[self.index]
             if isZipFile(self.current):
                 try:
-                    _ = zipfile.ZipFile(self.current, 'r')
+                    self.zip_file = zipfile.ZipFile(self.current, 'r')
                 except zipfile.error as ex:
                     errMsg = "something appears to be wrong with "
                     errMsg += "the file '%s' ('%s'). Please make " % (self.current, getSafeExString(ex))
                     errMsg += "sure that you haven't made any changes to it"
                     raise SqlmapInstallationException(errMsg)
-                if len(_.namelist()) == 0:
+                if len(self.zip_file.namelist()) == 0:
                     errMsg = "no file(s) inside '%s'" % self.current
                     raise SqlmapDataException(errMsg)
-                self.fp = _.open(_.namelist()[0])
+                self.fp = self.zip_file.open(self.zip_file.namelist()[0])
             else:
                 self.fp = open(self.current, "rb")
             self.iter = iter(self.fp)
@@ -70,6 +71,10 @@ def closeFP(self):
             self.fp.close()
             self.fp = None
 
+        if self.zip_file:
+            self.zip_file.close()
+            self.zip_file = None
+
     def __next__(self):
         retVal = None
         while True:

lib/parse/configfile.py

@@ -64,7 +64,7 @@ def configFileParser(configFile):
     logger.debug(debugMsg)
 
     checkFile(configFile)
-    configFP = openFile(configFile, "rb")
+    configFP = openFile(configFile, 'r')
 
     try:
         config = UnicodeRawConfigParser()

lib/request/connect.py

@@ -490,7 +490,7 @@ def getPage(**kwargs):
                 headers = forgeHeaders(auxHeaders, headers)
 
             if kb.headersFile:
-                content = openFile(kb.headersFile, "rb").read()
+                content = openFile(kb.headersFile, 'r').read()
                 for line in content.split("\n"):
                     line = getText(line.strip())
                     if ':' in line:

lib/techniques/blind/inference.py

@@ -415,7 +415,7 @@ def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None,
                             # list
                             if expand and shiftTable:
                                 charTbl = xrange(maxChar + 1, (maxChar + 1) << shiftTable.pop())
-                                originalTbl = xrange(charTbl)
+                                originalTbl = xrange(charTbl[0], charTbl[-1] + 1)
                                 maxChar = maxValue = charTbl[-1]
                                 minValue = charTbl[0]
                             else:
@@ -471,13 +471,16 @@ def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None,
                 bit = 0
                 while len(candidates) > 1:
                     bits = {}
+                    maxCandidate = max(candidates)
+                    maxBits = maxCandidate.bit_length() if maxCandidate > 0 else 1
+
                     for candidate in candidates:
-                        bit = 0
-                        while candidate:
+                        for bit in xrange(maxBits):
                             bits.setdefault(bit, 0)
-                            bits[bit] += 1 if candidate & 1 else -1
-                            candidate >>= 1
-                            bit += 1
+                            if candidate & (1 << bit):
+                                bits[bit] += 1
+                            else:
+                                bits[bit] -= 1
 
                     choice = sorted(bits.items(), key=lambda _: abs(_[1]))[0][0]
                     mask = 1 << choice
@@ -499,7 +502,10 @@ def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None,
                     incrementCounter(getTechnique())
 
                     if result:
-                        return decodeIntToUnicode(candidates[0])
+                        if candidates[0] == 0:      # Trailing zeros
+                            return None
+                        else:
+                            return decodeIntToUnicode(candidates[0])
 
         # Go multi-threading (--threads > 1)
         if numThreads > 1 and isinstance(length, int) and length > 1: