General boolean inference improvements

Author: stamparm

data/txt/catalog-identifiers.txt

@@ -23,9 +23,9 @@ a65269dcf3cecd4be0bf6b657cbf49ac77814ac7b0e30afa1cd44bc2fed64c33  data/shell/sta
 c52c17f3344707cae4c3694a979e073202bd46866fcc51d99f7e4d0c21cf335b  data/shell/stagers/stager.cfm_
 8cb4a001efc15bd8022d44df6eb9b2f5f5af1c64caba8f7dffde563ccba76347  data/shell/stagers/stager.jsp_
 af4e1f87ec7afd12b7ddb39ff07bf24cd31be2b1de11e1be064e1dd96ff43eac  data/shell/stagers/stager.php_
+aa4d6396df0abde7560ced7d8f7625dd70d57401db86d205f80064609c4b2772  data/txt/catalog-identifiers.txt
 eb86f6ad21e597f9283bb4360129ebc717bc8f063d7ab2298f31118275790484  data/txt/common-columns.txt
 63ba15f2ba3df6e55600a2749752c82039add43ed61129febd9221eb1115f240  data/txt/common-files.txt
-4d6a32155dd6b570e5cdae8036efd69d8f8ebab79cb82a4d094c15f35af8b13d  data/txt/common-outputs.txt
 44047281263ef297f27fdd8fa98a0b0438a25989f897ce184cb0e2e442fb6c11  data/txt/common-tables.txt
 ccba96624a0176b4c5acd8824db62a8c6856dafa7d32424807f38efed22a6c29  data/txt/keywords.txt
 522cce0327de8a5dfb5ade505e8a23bbd37bcabcbb2993f4f787ccdecf24997e  data/txt/smalldict.txt
@@ -168,7 +168,7 @@ d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f  lib/controller
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/controller/__init__.py
 48ffe93d61734e16c3b20153b51595853d9ac1fbcf0b537e0e61e957b0c0bfa6  lib/core/agent.py
 c51c33501cc905586a9aaac93b06f2ac6f71628d032a7dc39fd0ef05d7ee3856  lib/core/bigarray.py
-19989ca19194bf3f7a42a929b153e45c9a2177e01ab6ab63a5372daa5989c0e8  lib/core/common.py
+2b0e014869b071a2b6aa4e0c9a23427abdc61a92f09adbcce123fd0fc7ec3aa6  lib/core/common.py
 8f1272487e1adfcc8c755a2f56f0c6d21eac5e685a73a9a159482f9dc9142bc5  lib/core/compat.py
 5301ba2204404d086e9a67271cde00fc10214c63b018a95fc5aa90ff9e0b2ad9  lib/core/convert.py
 c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.py
@@ -181,26 +181,26 @@ c2db614a3ce7dda889152bea8bd6d709e5d8c2b556741fdbfe44469f27ce266b  lib/core/enums
 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4  lib/core/exception.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/core/__init__.py
 914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad  lib/core/log.py
-23852bdfadfb4bd5663302a63bdcc7227c0314fbdea884167d58ca21cda9fb09  lib/core/optiondict.py
-0caac9b4af2cc50321a4d8126d92481ad0b092af2075e7efa19bccef529986fb  lib/core/option.py
+0ba8838a8a8774a8a6baaf75d8122c3fa0cb9a6e4f468a4a94f32eb894feb754  lib/core/optiondict.py
+7229352618491ee1d23bbbfd6e7f160b489ce4b1a8d99c1e873d9664382c3f8c  lib/core/option.py
 21b2b1745107c211fc7593923a3da7a808d40763c00091c28de5f7c129bcf3bc  lib/core/patch.py
 49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec  lib/core/profiling.py
 0c36a65b6237732eb001d333f80f0c58c088ff01ae80cf07e4dcc6da2a806364  lib/core/readlineng.py
 9bf174058f15d14e24e94f9aaf42df045119d3617c6c54bd2f3af79b462f331d  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-df067f981efe10f6743eba13c48c9c1db158ff4e9d015831e5dbfa2ece80f7bf  lib/core/settings.py
+b2ce6d9948284621cbfa73d81c97416a98e6cbf6f87e2c9dd0b493030ff85340  lib/core/settings.py
 c7804223319e18eb0b8e2cbf0a8b6896d1cefb7b0b1a2e9f1cf826a8a3b56750  lib/core/shell.py
 a2e98a94b231432736d6b304fc75525c8b5fdb4768c418387c5b4c1a610dad64  lib/core/subprocessng.py
 69a68894db04695234369eedac71b5a89efc1b4ce89ef0e61ebbbc1895ff32b2  lib/core/target.py
-96d107a31bb9647a9b7c26f10beac528bf4edc6e607c8b776c624d494332c7f8  lib/core/testing.py
+e08683ba2558b734562a3490caf0bdde4ae8767b81b0d89fe6db346a13a452c1  lib/core/testing.py
 95656c44bab1771f4808030dd6a17eae5b129cb1234443f00b19695c7b712b86  lib/core/threads.py
 b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02  lib/core/unescaper.py
 53e396902cb2546eaa09e77073fcba8be8827ee9ce055cfc899e81b0e6ad4d6d  lib/core/update.py
 2400e465fa4d13e4c32795910878c71ff212e4361b46428d57ce43983f5e997c  lib/core/wordlist.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/__init__.py
 54bfd31ebded3ffa5848df1c644f196eb704116517c7a3d860b5d081e984d821  lib/parse/banner.py
-c9d38a60a85691cdb540e33510dd16228d6afcce0fd2ba39780f71b6da57ebb5  lib/parse/cmdline.py
+ea85d5eccc6d6f6f9e4423ea97dc607ef2ca10b8264e7877c128b2252d1340e3  lib/parse/cmdline.py
 925a068efa1885fa40671414a887c088f2aafbe8cb76f01286e6bde3f624dac1  lib/parse/configfile.py
 c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/handler.py
 5c9a9caee948843d5537745640cc7b98d70a0412cc0949f59d4ebe8b2907c06c  lib/parse/headers.py
@@ -219,7 +219,7 @@ c968a04d3de9256d56c423d46556441223607e4573627f2af4e772e084aef5fc  lib/request/dn
 7344978ac1c52060716b7837c88a62768c6a445eafe189ea3232b8a498fdd038  lib/request/http2.py
 92c81cc31ff4a396723242058fb2152c9e9745f8412d01ea74480b048a53af6c  lib/request/httpshandler.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/request/__init__.py
-7a0ac2522213e756348fd871a7af74cc963bdc82f9d7ade57be5de42b5bf7cab  lib/request/inject.py
+7ad7ac3c7126b3bad5898803a87769f199f6e8ecfb8abdca4fc4a29e63932595  lib/request/inject.py
 df97f7ccb437f9fda76b3d87cb5c11a01d09a0fa395c0d6bd555812cf92b70e6  lib/request/interactsh.py
 ff15723c82e343eb95f4599d251165d478ca720afc8f5daaed3da44ea923df44  lib/request/keepalive.py
 ada4d305d6ce441f79e52ec3f2fc23869ee2fa87c017723e8f3ed0dfa61cdab4  lib/request/methodrequest.py
@@ -236,7 +236,7 @@ f522436fbd14bdab090a1d305fcac0361800cb8e36c8cbcb47933298376a71e0  lib/takeover/r
 0787f78e6bd9bb21d4267c95c4c99806711bb57c5518485c2e25f10fcf9c41fc  lib/takeover/udf.py
 23d73af417604dab460b74cdc230896153f018a6c00d144019491053640a172f  lib/takeover/web.py
 8cc1e226d4150fe8aa1a056e5d32d858ed6444d3d4e2af7fb4bc08f0bbe9d527  lib/takeover/xp_cmdshell.py
-a66a4b9df6207dce722c9b71d290ea426723cb4b697b416065dc7dd5db96fe8e  lib/techniques/blind/inference.py
+da4930b2499270172140ae1f12c4666d42b2f1c0c348fcb021d4dccedc91d0ac  lib/techniques/blind/inference.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/blind/__init__.py
 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3  lib/techniques/dns/__init__.py
 3df9839fb92a81d46b6194d7adacb43f391efb78b071783c132e8d596ecbfaf1  lib/techniques/dns/test.py
@@ -499,8 +499,8 @@ e2e20e4707abe9ed8b6208837332d2daa4eaca282f847412063f2484dcca8fbd  plugins/dbms/v
 2b2dad6ba1d344215cad11b629546eb9f259d7c996c202edf3de5ab22418787e  plugins/dbms/virtuoso/takeover.py
 51c44048e4b335b306f8ed1323fd78ad6935a8c0d6e9d6efe195a9a5a24e46dc  plugins/generic/connector.py
 a967f4ebd101c68a5dcc10ff18c882a8f44a5c3bf06613d951a739ecc3abb9b3  plugins/generic/custom.py
-6f77b5cae6781a746f8490fe3e85456e575165b38edd280a69c9327af8bee85f  plugins/generic/databases.py
-13086bfae6022edc2bbd35512fa3bda3402c269e9d6148ffe386ba5b8b4ba461  plugins/generic/entries.py
+dc52b79735a2e349dbc599bc7bd3f57b58560aed977f8b053f745e93532d4e13  plugins/generic/databases.py
+93c4833046dce00a60cf6ca118e6637d4be23a67aa45714f91d23215d544b023  plugins/generic/entries.py
 d2de7fc135cf0db3eb4ac4a509c23ebec5250a5d8043face7f8c546a09f301b5  plugins/generic/enumeration.py
 8d5e3eacbd2a3cfec63fcf5bdcc8efc77656f29b11ca652c4ee60c72daea04ab  plugins/generic/filesystem.py
 efd7177218288f32881b69a7ba3d667dc9178f1009c06a3e1dd4f4a4ee6980db  plugins/generic/fingerprint.py

data/txt/common-outputs.txt

@@ -1582,10 +1582,10 @@ def setPaths(rootPath):
     paths.SQLMAP_XML_PAYLOADS_PATH = os.path.join(paths.SQLMAP_XML_PATH, "payloads")
 
     # sqlmap files
+    paths.CATALOG_IDENTIFIERS = os.path.join(paths.SQLMAP_TXT_PATH, "catalog-identifiers.txt")
     paths.COMMON_COLUMNS = os.path.join(paths.SQLMAP_TXT_PATH, "common-columns.txt")
     paths.COMMON_FILES = os.path.join(paths.SQLMAP_TXT_PATH, "common-files.txt")
     paths.COMMON_TABLES = os.path.join(paths.SQLMAP_TXT_PATH, "common-tables.txt")
-    paths.COMMON_OUTPUTS = os.path.join(paths.SQLMAP_TXT_PATH, 'common-outputs.txt')
     paths.DIGEST_FILE = os.path.join(paths.SQLMAP_TXT_PATH, "sha256sums.txt")
     paths.SQL_KEYWORDS = os.path.join(paths.SQLMAP_TXT_PATH, "keywords.txt")
     paths.SMALL_DICT = os.path.join(paths.SQLMAP_TXT_PATH, "smalldict.txt")
@@ -2605,42 +2605,23 @@ def calculateDeltaSeconds(start):
 
 def initCommonOutputs():
     """
-    Initializes dictionary containing common output values used by "good samaritan" feature
+    Initializes the per-context dictionary of common identifier names used by the
+    predictive-inference feature to shortcut blind table/column name enumeration.
+    Sourced directly from the curated '--common-tables'/'--common-columns' wordlists
+    (real-world, app-focused names); prediction only ever reorders the charset or
+    confirms a whole value, so it never penalizes a miss.
 
-    >>> initCommonOutputs(); "information_schema" in kb.commonOutputs["Databases"]
+    >>> initCommonOutputs(); "users" in kb.commonOutputs["Tables"]
     True
     """
 
     kb.commonOutputs = {}
-    key = None
-
-    with openFile(paths.COMMON_OUTPUTS, 'r') as f:
-        for line in f:
-            if line.find('#') != -1:
-                line = line[:line.find('#')]
-
-            line = line.strip()
-
-            if len(line) > 1:
-                if line.startswith('[') and line.endswith(']'):
-                    key = line[1:-1]
-                elif key:
-                    if key not in kb.commonOutputs:
-                        kb.commonOutputs[key] = set()
-
-                    if line not in kb.commonOutputs[key]:
-                        kb.commonOutputs[key].add(line)
-
-    # The curated '--common-tables'/'--common-columns' brute-force wordlists are far larger and much
-    # more app-focused than the built-in [Tables]/[Columns] prediction sections (which are mostly
-    # system objects), so fold them into the good-samaritan prediction to raise its real-world hit rate.
-    # The mechanism only reorders the charset, so extra coverage never penalizes a miss.
-    for _key, _path in (("Tables", paths.COMMON_TABLES), ("Columns", paths.COMMON_COLUMNS)):
+
+    for key, path in (("Tables", paths.COMMON_TABLES), ("Columns", paths.COMMON_COLUMNS)):
         try:
-            for _ in getFileItems(_path):
-                kb.commonOutputs.setdefault(_key, set()).add(_)
+            kb.commonOutputs[key] = set(getFileItems(path))
         except SqlmapSystemException:
-            pass
+            kb.commonOutputs[key] = set()
 
 def getFileItems(filename, commentPrefix='#', unicoded=True, lowercase=False, unique=False):
     """
@@ -2684,19 +2665,17 @@ def getFileItems(filename, commentPrefix='#', unicoded=True, lowercase=False, un
 
     return retVal if not unique else list(retVal.keys())
 
-def goGoodSamaritan(prevValue, originalCharset):
+def predictValue(prevValue, originalCharset):
     """
-    Function for retrieving parameters needed for common prediction (good
-    samaritan) feature.
+    Predictive-inference helper: given the value retrieved so far (prefix), consult the
+    per-context common-identifier set (kb.commonOutputs[kb.partRun], from the common-
+    tables/common-columns wordlists) to shortcut blind extraction.
 
     prevValue: retrieved query output so far (e.g. 'i').
 
-    Returns commonValue if there is a complete single match (in kb.partRun
-    of txt/common-outputs.txt under kb.partRun) regarding parameter
-    prevValue. If there is no single value match, but multiple, commonCharset is
-    returned containing more probable characters (retrieved from matched
-    values in txt/common-outputs.txt) together with the rest of charset as
-    otherCharset.
+    Returns commonValue when a single wordlist entry matches the prefix (the whole value
+    can be confirmed in one request); otherwise commonCharset holds the more probable
+    next characters (reordered ahead of otherCharset) so the bisection converges faster.
     """
 
     if kb.commonOutputs is None:
@@ -2755,7 +2734,7 @@ def goGoodSamaritan(prevValue, originalCharset):
 def getPartRun(alias=True):
     """
     Goes through call stack and finds constructs matching
-    conf.dbmsHandler.*. Returns it or its alias used in 'txt/common-outputs.txt'
+    conf.dbmsHandler.*. Returns it or its predictive-inference context alias (e.g. 'Tables'/'Columns')
     """
 
     retVal = None
@@ -3692,8 +3671,8 @@ def setOptimize():
     Sets options turned on by switch '-o'
     """
 
-    # conf.predictOutput = True
-    # Note: persistent (Keep-Alive) connections are now used by default (see _setHTTPHandlers)
+    # Note: persistent (Keep-Alive) connections are now used by default (see _setHTTPHandlers); predictive
+    # inference is now an inherent, always-on part of blind name enumeration (no longer a switch)
     conf.threads = 3 if conf.threads < 3 and cmdLineOptions.threads is None else conf.threads
     conf.nullConnection = not any((conf.data, conf.textOnly, conf.titles, conf.string, conf.notString, conf.regexp, conf.tor))
 

data/txt/sha256sums.txt

@@ -2243,6 +2243,11 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.disableHuffman = False
     kb.huffmanProbes = 0
     kb.huffmanEscapes = 0
+    kb.lowCardCache = {}
+    kb.dumpCharset = {}
+    kb.dumpCharsetStable = {}
+    kb.litmusCounter = 0
+    kb.reliabilityAlarm = False
     kb.httpErrorCodes = {}
     kb.inferenceMode = False
     kb.ignoreCasted = None
@@ -2256,7 +2261,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.lastParserStatus = None
 
     kb.locks = AttribDict()
-    for _ in ("cache", "connError", "count", "handlers", "hint", "identYwaf", "index", "io", "limit", "liveCookies", "log", "socket", "redirect", "request", "value"):
+    for _ in ("cache", "connError", "count", "handlers", "hint", "identYwaf", "index", "io", "limit", "liveCookies", "log", "prediction", "socket", "redirect", "request", "value"):
         kb.locks[_] = threading.Lock()
 
     kb.matchRatio = None
@@ -2914,10 +2919,6 @@ def _basicOptionValidation():
         errMsg = "switch '--dump' is incompatible with switch '--dump-all'"
         raise SqlmapSyntaxException(errMsg)
 
-    if conf.predictOutput and (conf.threads > 1 or conf.optimize):
-        errMsg = "switch '--predict-output' is incompatible with option '--threads' and switch '-o'"
-        raise SqlmapSyntaxException(errMsg)
-
     if conf.threads > MAX_NUMBER_OF_THREADS and not conf.get("skipThreadCheck"):
         errMsg = "maximum number of used threads is %d avoiding potential connection issues" % MAX_NUMBER_OF_THREADS
         raise SqlmapSyntaxException(errMsg)

lib/core/common.py

@@ -79,7 +79,6 @@
 
     "Optimization": {
         "optimize": "boolean",
-        "predictOutput": "boolean",
         "keepAlive": "boolean",
         "noKeepAlive": "boolean",
         "nullConnection": "boolean",

lib/core/option.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.30"
+VERSION = "1.10.7.31"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -560,11 +560,52 @@
     for _char in _chars:
         HUFFMAN_PRIOR_WEIGHTS[ord(_char)] = _weight
 
-# Bounds for feeding extracted values back into the "good samaritan" (--predict-output) common-output
-# pool for their enumeration context, so later same-context items that share structure (e.g.
-# wp_posts / wp_users / wp_options ...) are predicted faster. MAX_LENGTH keeps large data cells from
-# bloating/polluting the pool (identifiers are short); MAX_ITEMS bounds per-context growth so a huge
-# enumeration cannot make the per-character prediction scan costly. Misses always fall back to bisection.
+# Enumeration contexts (kb.partRun) for which predictive inference is active by default: the identifier
+# names retrieved here are drawn from a known, skewed distribution captured by the common-tables/
+# common-columns wordlists, so whole-value prediction / charset reordering pays off. Deliberately NOT
+# applied to arbitrary dumped data (unknown distribution) or one-shot values (banner, current-user).
+NAME_PREDICTION_CONTEXTS = ("Tables", "Columns")
+
+# Order of the character-level Markov model used to seed the Huffman set-membership tree during blind
+# name enumeration: warmed from the shipped identifier corpus so it predicts a name from the first
+# character (identifiers are short, structured and low-entropy). CATALOG_IDENTIFIERS_PRIOR_PEAK is the
+# weight the corpus prior is scaled to (higher -> the predicted next character sits nearer the tree
+# root -> closer to one request per character). Data dumps keep the classic order-0 adaptive model.
+NAME_MARKOV_ORDER = 3
+CATALOG_IDENTIFIERS_PRIOR_PEAK = 20
+
+# Maximum number of distinct values a dumped column may show before it is treated as high-cardinality
+# and whole-value guessing is abandoned for it. At or below this, each new cell is first confirmed by
+# equality against the values already seen for that column (one request on a hit) before per-character
+# extraction. Self-verifying, so it never returns a wrong value; the bound keeps misses cheap.
+LOW_CARDINALITY_THRESHOLD = 32
+
+# Oracle-reliability litmus: during bulk blind extraction (dumps / name enumeration) a known-answer
+# differential is fired every this-many extracted values - one probe that MUST be TRUE (the value we just
+# read equals itself) and one that MUST be FALSE (it equals a deliberately corrupted copy). A healthy
+# oracle always answers T/F; an always-true channel (WAF/200-for-everything, reads-everything-true) or a
+# flaky/degraded one (timing jitter, lease near end-of-life) trips it - converting SILENT data corruption
+# into a one-time "results may be unreliable" warning. The first value is always checked (catch it before
+# a whole garbage dump), then every Nth. Cheap and amortized; set to 0 to disable.
+ORACLE_LITMUS_CHECK_EVERY = 25
+
+# Whole-value guessing only starts once some value has repeated (proof the column is low-cardinality), so
+# an all-unique column - primary key, hash, free text - never wastes a probe. Once armed, at most this
+# many candidates (most-frequent first) are tried per cell, so even a column that trips the threshold with
+# many near-unique values can only ever waste a small, bounded number of probes before falling back.
+LOW_CARDINALITY_MAX_GUESSES = 8
+
+# Number of consecutive dumped rows a column's observed character set must stay unchanged before it is
+# trusted as closed and used to restrict the time-based bisection alphabet. A column whose alphabet keeps
+# growing (e.g. a monotonic primary key or high-entropy text) never reaches this, so it is never charged
+# the speculative restricted-search-then-escalate cost.
+DUMP_CHARSET_STABLE_ROWS = 3
+
+# Bounds for feeding extracted values back into the predictive-inference pool for their enumeration
+# context, so later same-context items that share structure (e.g. wp_posts / wp_users / wp_options ...)
+# are predicted faster. MAX_LENGTH keeps large data cells from bloating/polluting the pool (identifiers
+# are short); MAX_ITEMS bounds per-context growth so a huge enumeration cannot make the per-character
+# prediction scan costly. Only fed single-threaded (never mutated under value-parallel enumeration).
 PREDICTION_FEEDBACK_MAX_LENGTH = 128
 PREDICTION_FEEDBACK_MAX_ITEMS = 10000