Fixing desync error from SLOW_ORDER_COUNT_THRESHOLD

Author: stamparm

data/xml/queries.xml

@@ -222,7 +222,7 @@
         </columns>
         <dump_table>
             <inband query="SELECT %s FROM %s.%s"/>
-            <blind query="SELECT MIN(%s) FROM %s WHERE CONVERT(NVARCHAR(4000),%s)>'%s'" query2="SELECT MAX(%s) FROM %s WHERE CONVERT(NVARCHAR(4000),%s) LIKE '%s'" query3="SELECT %s FROM (SELECT %s, ROW_NUMBER() OVER (ORDER BY (SELECT 1)) AS CAP FROM %s)x WHERE CAP=%d" count="SELECT LTRIM(STR(COUNT(*))) FROM %s" count2="SELECT LTRIM(STR(COUNT(DISTINCT(%s)))) FROM %s" keyset_first="SELECT MIN(%s) FROM %s" keyset_next="SELECT MIN(%s) FROM %s WHERE %s>'%s'" keyset_by="SELECT MAX(%s) FROM %s WHERE %s='%s'" keyset_seed="SELECT %s FROM %s ORDER BY %s OFFSET %d ROWS FETCH NEXT 1 ROWS ONLY" keyset_ordered="SELECT TOP 1 %s FROM %s WHERE %s ORDER BY %s" keyset_where="SELECT MAX(%s) FROM %s WHERE %s"/>
+            <blind query="SELECT MIN(%s) FROM %s WHERE CONVERT(NVARCHAR(4000),%s)>'%s'" query2="SELECT MAX(%s) FROM %s WHERE CONVERT(NVARCHAR(4000),%s) LIKE '%s'" query3="SELECT %s FROM (SELECT %s, ROW_NUMBER() OVER (ORDER BY %s) AS CAP FROM %s)x WHERE CAP=%d" count="SELECT LTRIM(STR(COUNT(*))) FROM %s" count2="SELECT LTRIM(STR(COUNT(DISTINCT(%s)))) FROM %s" keyset_first="SELECT MIN(%s) FROM %s" keyset_next="SELECT MIN(%s) FROM %s WHERE %s>'%s'" keyset_by="SELECT MAX(%s) FROM %s WHERE %s='%s'" keyset_seed="SELECT %s FROM %s ORDER BY %s OFFSET %d ROWS FETCH NEXT 1 ROWS ONLY" keyset_ordered="SELECT TOP 1 %s FROM %s WHERE %s ORDER BY %s" keyset_where="SELECT MAX(%s) FROM %s WHERE %s"/>
             <primary_key count="SELECT COUNT(*) FROM sys.indexes i JOIN sys.index_columns ic ON i.object_id=ic.object_id AND i.index_id=ic.index_id WHERE i.is_primary_key=1 AND i.object_id=OBJECT_ID('%s.dbo.%s')" query="SELECT name FROM (SELECT c.name AS name, ROW_NUMBER() OVER (ORDER BY ic.key_ordinal) AS rn FROM sys.indexes i JOIN sys.index_columns ic ON i.object_id=ic.object_id AND i.index_id=ic.index_id JOIN sys.columns c ON ic.object_id=c.object_id AND c.column_id=ic.column_id WHERE i.is_primary_key=1 AND i.object_id=OBJECT_ID('%s.dbo.%s')) x WHERE rn=%d+1"/>
         </dump_table>
         <search_db>

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.74"
+VERSION = "1.10.7.75"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -879,9 +879,6 @@
 # Warn user of possible delay due to large page dump in full UNION query injections
 LARGE_OUTPUT_THRESHOLD = 1024 ** 2
 
-# On huge tables there is a considerable slowdown if every row retrieval requires ORDER BY (most noticable in table dumping using ERROR injections)
-SLOW_ORDER_COUNT_THRESHOLD = 10000
-
 # Give up on hash recognition if nothing was found in first given number of rows
 HASH_RECOGNITION_QUIT_THRESHOLD = 1000
 

lib/techniques/error/use.py

@@ -29,7 +29,6 @@
 from lib.core.common import isListLike
 from lib.core.common import isNumPosStrValue
 from lib.core.common import listToStrValue
-from lib.core.common import readInput
 from lib.core.common import unArrayizeValue
 from lib.core.common import wasLastResponseHTTPError
 from lib.core.compat import xrange
@@ -51,7 +50,6 @@
 from lib.core.settings import NULL
 from lib.core.settings import PARTIAL_VALUE_MARKER
 from lib.core.settings import ROTATING_CHARS
-from lib.core.settings import SLOW_ORDER_COUNT_THRESHOLD
 from lib.core.settings import SQL_SCALAR_REGEX
 from lib.core.settings import TURN_OFF_RESUME_INFO_LIMIT
 from lib.core.threads import getCurrentThreadData
@@ -322,7 +320,7 @@ def errorUse(expression, dump=False):
     # entry at a time
     # NOTE: we assume that only queries that get data from a table can
     # return multiple entries
-    if (dump and (conf.limitStart or conf.limitStop)) or (" FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and ("(CASE" not in expression.upper() or ("(CASE" in expression.upper() and "WHEN use" in expression))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+    if not re.search(SQL_SCALAR_REGEX, expression, re.I) and ((dump and (conf.limitStart or conf.limitStop)) or (" FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and ("(CASE" not in expression.upper() or ("(CASE" in expression.upper() and "WHEN use" in expression)))):
         expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)
 
         if limitCond:
@@ -367,13 +365,10 @@ def errorUse(expression, dump=False):
                 return value
 
             if isNumPosStrValue(count) and int(count) > 1:
-                if " ORDER BY " in expression and (stopLimit - startLimit) > SLOW_ORDER_COUNT_THRESHOLD:
-                    message = "due to huge table size do you want to remove "
-                    message += "ORDER BY clause gaining speed over consistency? [y/N] "
-
-                    if readInput(message, default='N', boolean=True):
-                        expression = expression[:expression.index(" ORDER BY ")]
-
+                # NOTE: the ORDER BY clause is deliberately NOT stripped here. This path fetches each column
+                # of a row in a separate LIMIT/OFFSET query, so dropping ORDER BY would let the per-column
+                # offsets resolve to different physical rows and silently misalign cells across the row. Huge
+                # tables are handled cheaply and safely by keyset (seek) pagination (see plugins/generic/entries.py).
                 numThreads = min(conf.threads, (stopLimit - startLimit))
 
                 threadData = getCurrentThreadData()

lib/utils/keysetdump.py

@@ -189,12 +189,12 @@ def _dumpSingle(tbl, colList, count, cursor, tableRef, entries, lengths):
     produced = 0
 
     while produced < target:
-        if pivotValue is None:
-            query = blind.keyset_first % (field, tableRef)
-        else:
-            query = _embed(blind.keyset_next, pivotValue, field, tableRef, field)
-
-        query = agent.whereQuery(query)
+        # Advance with ORDER BY ... LIMIT 1 (like the composite path), NOT MIN(): the value-extraction
+        # casts the aggregated column to VARCHAR *inside* MIN(), yielding a LEXICAL minimum ('10' after
+        # '1') that disagrees with the numeric '>' comparison and silently skips rows (2..9, 11..). The
+        # ORDER BY is on the raw (numeric) column, so the next cursor value is the true successor.
+        condition = "1=1" if pivotValue is None else "%s>%s" % (field, _lit(pivotValue))
+        query = agent.whereQuery(blind.keyset_ordered % (field, tableRef, condition, field))
         value = unArrayizeValue(inject.getValue(query))
 
         if isNoneValue(value) or value == NULL:

plugins/generic/entries.py

@@ -184,7 +184,49 @@ def dumpTable(self, foundData=None):
 
                 entriesCount = 0
 
-                if any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) or conf.direct:
+                def _dumpCountQuery():
+                    if Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2, DBMS.DERBY, DBMS.ALTIBASE, DBMS.MIMERSQL, DBMS.SNOWFLAKE):
+                        _ = rootQuery.blind.count % (tbl.upper() if not conf.db else ("%s.%s" % (conf.db.upper(), tbl.upper())))
+                    elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.MAXDB, DBMS.ACCESS, DBMS.FIREBIRD, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.RAIMA):
+                        _ = rootQuery.blind.count % tbl
+                    elif Backend.getIdentifiedDbms() in (DBMS.SYBASE, DBMS.MSSQL):
+                        _ = rootQuery.blind.count % ("%s.%s" % (conf.db, tbl)) if conf.db else tbl
+                    elif Backend.isDbms(DBMS.INFORMIX):
+                        _ = rootQuery.blind.count % (conf.db, tbl)
+                    else:
+                        _ = rootQuery.blind.count % (conf.db, tbl)
+                    return agent.whereQuery(_)
+
+                # Keyset (seek) pagination for the error/query (inband) path too, not just the blind path below.
+                # It fetches every cell value-anchored (MAX(col) WHERE key=K_r on a unique integer key), so the
+                # per-cell retrievals of one row are pinned to the SAME physical row regardless of scan order,
+                # threads or MVCC - unlike ORDER BY ... LIMIT/OFFSET, which silently misaligns cells once a
+                # stable order is unavailable. UNION is intrinsically aligned (whole-row), so it is never preempted.
+                keysetDone = False
+                if not conf.direct and not conf.noKeyset and not isTechniqueAvailable(PAYLOAD.TECHNIQUE.UNION) \
+                        and any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)):
+                    count = inject.getValue(_dumpCountQuery(), expected=EXPECTED.INT, charsetType=CHARSET_TYPE.DIGITS)
+                    if isNumPosStrValue(count) and (conf.keyset or int(count) >= KEYSET_MIN_ROWS):
+                        keysetCursor = resolveKeysetCursor(tbl, colList)
+                        if keysetCursor:
+                            infoMsg = "using keyset (seek) pagination on column(s) '%s' " % ', '.join(keysetCursor)
+                            infoMsg += "for table '%s'" % unsafeSQLIdentificatorNaming(tbl)
+                            logger.info(infoMsg)
+
+                            try:
+                                entries, lengths = keysetDumpTable(tbl, colList, int(count), keysetCursor)
+                                for column, columnEntries in entries.items():
+                                    length = max(lengths[column], getConsoleLength(column))
+                                    kb.data.dumpedTable[column] = {"length": length, "values": columnEntries}
+                                    entriesCount = len(columnEntries)
+                                keysetDone = bool(kb.data.dumpedTable)
+                            except KeyboardInterrupt:
+                                kb.dumpKeyboardInterrupt = True
+                                clearConsoleLine()
+                                warnMsg = "Ctrl+C detected in dumping phase"
+                                logger.warning(warnMsg)
+
+                if not keysetDone and (any(isTechniqueAvailable(_) for _ in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)) or conf.direct):
                     entries = []
                     query = None
 
@@ -213,7 +255,7 @@ def dumpTable(self, foundData=None):
                                         for index in indexRange:
                                             row = []
                                             for column in colList:
-                                                query = rootQuery.blind.query3 % (column, column, table, index)
+                                                query = rootQuery.blind.query3 % (column, column, prioritySortColumns(colList)[0], table, index)
                                                 query = agent.whereQuery(query)
                                                 value = inject.getValue(query, blind=False, time=False, dump=True) or ""
                                                 row.append(value)
@@ -369,7 +411,7 @@ def dumpTable(self, foundData=None):
 
                                 for index in indexRange:
                                     for column in colList:
-                                        query = rootQuery.blind.query3 % (column, column, table, index)
+                                        query = rootQuery.blind.query3 % (column, column, prioritySortColumns(colList)[0], table, index)
                                         query = agent.whereQuery(query)
 
                                         value = inject.getValue(query, union=False, error=False, dump=True) or ""