Minor patching for binary fields and casting

Author: stamparm

data/xml/queries.xml

@@ -83,7 +83,7 @@
     </dbms>
 
     <dbms value="PostgreSQL">
-        <cast query="CAST(%s AS VARCHAR(10000))"/>
+        <cast query="CAST(%s AS TEXT)"/>
         <length query="LENGTH(%s)"/>
         <!-- NOTE: PostgreSQL does not like COALESCE with different data-types (e.g. COALESCE(id,' ')) -->
         <isnull query="COALESCE(%s::text,' ')"/>
@@ -119,8 +119,8 @@
             <blind query="SELECT DISTINCT(passwd) FROM pg_shadow WHERE usename='%s' OFFSET %d LIMIT 1" count="SELECT COUNT(DISTINCT(passwd)) FROM pg_shadow WHERE usename='%s'"/>
         </passwords>
         <privileges>
-            <inband query="SELECT usename,(CASE WHEN usecreatedb THEN 1 ELSE 0 END),(CASE WHEN usesuper THEN 1 ELSE 0 END),(CASE WHEN usecatupd THEN 1 ELSE 0 END) FROM pg_user" condition="usename"/>
-            <blind query="SELECT (CASE WHEN usecreatedb THEN 1 ELSE 0 END),(CASE WHEN usesuper THEN 1 ELSE 0 END),(CASE WHEN usecatupd THEN 1 ELSE 0 END) FROM pg_user WHERE usename='%s' OFFSET %d LIMIT 1" count="SELECT COUNT(DISTINCT(usename)) FROM pg_user WHERE usename='%s'"/>
+            <inband query="SELECT usename,(CASE WHEN usecreatedb THEN 1 ELSE 0 END),(CASE WHEN usesuper THEN 1 ELSE 0 END),(CASE WHEN userepl THEN 1 ELSE 0 END) FROM pg_user" condition="usename"/>
+            <blind query="SELECT (CASE WHEN usecreatedb THEN 1 ELSE 0 END),(CASE WHEN usesuper THEN 1 ELSE 0 END),(CASE WHEN userepl THEN 1 ELSE 0 END) FROM pg_user WHERE usename='%s' OFFSET %d LIMIT 1" count="SELECT COUNT(DISTINCT(usename)) FROM pg_user WHERE usename='%s'"/>
         </privileges>
         <roles/>
         <statements>
@@ -163,7 +163,7 @@
     </dbms>
 
     <dbms value="Microsoft SQL Server">
-        <cast query="CAST(%s AS NVARCHAR(4000))"/>
+        <cast query="CAST(%s AS NVARCHAR(MAX))"/>
         <length query="LTRIM(STR(LEN(%s+'.')-1))"/>
         <isnull query="ISNULL(%s,' ')"/>
         <delimiter query="+"/>
@@ -384,7 +384,7 @@
         </columns>
         <dump_table>
             <inband query="SELECT %s FROM %s"/>
-            <blind query="SELECT %s FROM %s LIMIT %d,1" count="SELECT COUNT(*) FROM %s" rowid="rowid" keyset_first="SELECT MIN(%s) FROM %s" keyset_next="SELECT MIN(%s) FROM %s WHERE %s>'%s'" keyset_by="SELECT MAX(%s) FROM %s WHERE %s='%s'" keyset_seed="SELECT %s FROM %s ORDER BY %s LIMIT 1 OFFSET %d" keyset_ordered="SELECT %s FROM %s WHERE %s ORDER BY %s LIMIT 1" keyset_where="SELECT MAX(%s) FROM %s WHERE %s"/>
+            <blind query="SELECT %s FROM %s ORDER BY rowid LIMIT %d,1" count="SELECT COUNT(*) FROM %s" rowid="rowid" keyset_first="SELECT MIN(%s) FROM %s" keyset_next="SELECT MIN(%s) FROM %s WHERE %s>'%s'" keyset_by="SELECT MAX(%s) FROM %s WHERE %s='%s'" keyset_seed="SELECT %s FROM %s ORDER BY %s LIMIT 1 OFFSET %d" keyset_ordered="SELECT %s FROM %s WHERE %s ORDER BY %s LIMIT 1" keyset_where="SELECT MAX(%s) FROM %s WHERE %s"/>
         </dump_table>
         <search_db/>
         <search_table>
@@ -959,8 +959,8 @@
 
     <dbms value="Apache Derby">
         <!-- NOTE: CHAR(%s) causes 'A truncation error was encountered trying to shrink CHAR' -->
-        <cast query="RTRIM(CAST(%s AS CHAR(254)))"/>
-        <length query="LENGTH(RTRIM(CAST(%s AS CHAR(254))))"/>
+        <cast query="CAST(%s AS VARCHAR(32672))"/>
+        <length query="LENGTH(CAST(%s AS VARCHAR(32672)))"/>
         <isnull query="COALESCE(%s,' ')"/>
         <delimiter query="||"/>
         <limit query="OFFSET %d ROWS FETCH FIRST %d ROWS ONLY"/>
@@ -1158,7 +1158,7 @@
         <substring query="SUBSTR(%s,%d,%d)"/>
         <concatenate query="%s||%s"/>
         <case query="SELECT (CASE WHEN (%s) THEN '1' ELSE '0' END)"/>
-        <hex query="TO_HEX(%s)"/>
+        <hex query="TO_HEX(CAST(%s AS VARBINARY))"/>
         <inference query="CODEPOINT(SUBSTR((%s),%d,1))>%d" dbms_version="&gt;=0.178" query2="SUBSTR((%s),%d,1)>'%c'"/>/>
         <banner query="version()"/>
         <current_user query="CURRENT_USER"/>

lib/core/agent.py

@@ -508,10 +508,26 @@ def nullAndCastField(self, field):
         if field and Backend.getIdentifiedDbms():
             rootQuery = queries[Backend.getIdentifiedDbms()]
 
+            kb.binaryField = conf.binaryFields and field in conf.binaryFields
+            hexConvert = conf.hexConvert or kb.binaryField
+
+            # For BINARY fields, wrap the RAW column with the hex function rather than the text-casted one:
+            # text-casting binary first (e.g. CAST(<binary> AS NCHAR) on MySQL) NULLs non-text bytes, so HEX()
+            # would encode the NULL placeholder and silently lose the value (e.g. binary-stored password
+            # hashes). This is limited to binary fields: the blanket '--hex' keeps the cast-first path because
+            # raw-hexing a numeric/temporal column uses the DBMS's numeric HEX semantics (MySQL HEX(255)='FF'),
+            # which is NOT the hex of its string representation. Every DBMS hex function takes binary directly
+            # EXCEPT PostgreSQL's CONVERT_TO() (needs text), so PostgreSQL stays on the cast-first path too.
+            hexRaw = kb.binaryField and not Backend.isDbms(DBMS.PGSQL)
+
             if field.startswith("(CASE") or field.startswith("(IIF") or conf.noCast and not (field.startswith("COUNT(") and Backend.getIdentifiedDbms() == DBMS.MSSQL):
                 nulledCastedField = field
+                if hexConvert:
+                    nulledCastedField = self.hexConvertField(nulledCastedField)
             else:
-                if not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')):
+                if hexRaw:
+                    nulledCastedField = self.hexConvertField(field)
+                elif not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')):
                     nulledCastedField = rootQuery.cast.query % field
 
                 if re.search(r"COUNT\(", field) and Backend.getIdentifiedDbms() in (DBMS.RAIMA,):
@@ -521,9 +537,8 @@ def nullAndCastField(self, field):
                 else:
                     nulledCastedField = rootQuery.isnull.query % nulledCastedField
 
-            kb.binaryField = conf.binaryFields and field in conf.binaryFields
-            if conf.hexConvert or kb.binaryField:
-                nulledCastedField = self.hexConvertField(nulledCastedField)
+                if hexConvert and not hexRaw:
+                    nulledCastedField = self.hexConvertField(nulledCastedField)
 
         if suffix:
             nulledCastedField += suffix

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.75"
+VERSION = "1.10.7.76"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -885,6 +885,11 @@
 # Regular expression used for automatic hex conversion and hash cracking of (RAW) binary column values
 HASH_BINARY_COLUMNS_REGEX = r"(?i)pass|psw|hash"
 
+# Regular expression matching (declared) binary column types, used to auto-hex their values during dumping
+# so raw bytes (e.g. password hashes stored in binary form) are not silently truncated at NUL / mangled by
+# the text extraction channel (mirrors a manual '--binary-fields', using the already-fetched column type)
+BINARY_FIELDS_TYPE_REGEX = r"(?i)binary|blob|bytea|image|\braw\b"
+
 # Maximum number of redirections to any single URL - this is needed because of the state that cookies introduce
 MAX_SINGLE_URL_REDIRECTIONS = 4
 

plugins/generic/entries.py

@@ -40,6 +40,7 @@
 from lib.core.exception import SqlmapMissingMandatoryOptionException
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapUnsupportedFeatureException
+from lib.core.settings import BINARY_FIELDS_TYPE_REGEX
 from lib.core.settings import CHECK_ZERO_COLUMNS_THRESHOLD
 from lib.core.settings import CURRENT_DB
 from lib.core.settings import KEYSET_MIN_ROWS
@@ -114,6 +115,8 @@ def dumpTable(self, foundData=None):
         for tbl in tblList:
             tblList[tblList.index(tbl)] = safeSQLIdentificatorNaming(tbl, True)
 
+        binaryFields = conf.binaryFields  # user-provided '--binary-fields' (auto-detected ones are added per-table)
+
         for tbl in tblList:
             if kb.dumpKeyboardInterrupt:
                 break
@@ -169,6 +172,18 @@ def dumpTable(self, foundData=None):
                 colNames = colString = ','.join(column for column in colList)
                 rootQuery = queries[Backend.getIdentifiedDbms()].dump_table
 
+                # Auto-treat binary-typed columns (blob/varbinary/bytea/image/raw) as '--binary-fields', so their
+                # raw bytes (e.g. password hashes stored in binary form) are hex-extracted instead of being
+                # silently truncated at a NUL / mangled by the text channel (issues #8, #582, #2827). The column
+                # type is already known from the enumeration above, so this costs no extra request.
+                # (PostgreSQL excluded: its bytea already renders as readable '\xHEX' through the default text
+                # cast, and its hex path needs text input, so auto-hexing would double-encode.)
+                autoBinary = [] if Backend.isDbms(DBMS.PGSQL) else [column for column in colList if columns.get(column) and re.search(BINARY_FIELDS_TYPE_REGEX, getUnicode(columns[column]))]
+                conf.binaryFields = (list(binaryFields) if binaryFields else []) + [_ for _ in autoBinary if not (binaryFields and _ in binaryFields)]
+                if autoBinary:
+                    debugMsg = "auto-treating binary column(s) '%s' as binary fields" % ', '.join(unsafeSQLIdentificatorNaming(_) for _ in autoBinary)
+                    logger.debug(debugMsg)
+
                 infoMsg = "fetching entries"
                 if conf.col:
                     infoMsg += " of column(s) '%s'" % colNames
@@ -572,6 +587,7 @@ def cellQuery(column, index):
             finally:
                 kb.dumpColumns = None
                 kb.dumpTable = None
+                conf.binaryFields = binaryFields  # restore user-provided value (drop this table's auto-detected ones)
 
     def dumpAll(self):
         if conf.db is not None and conf.tbl is None:

tests/test_dialect.py

@@ -68,9 +68,10 @@ class TestNullAndCastField(unittest.TestCase):
 
     CASES = {
         DBMS.MYSQL:  "IFNULL(CAST(col AS NCHAR),' ')",
-        DBMS.MSSQL:  "ISNULL(CAST(col AS NVARCHAR(4000)),' ')",
+        # MSSQL/PGSQL casts are unbounded (NVARCHAR(MAX)/TEXT) so long values are not silently truncated
+        DBMS.MSSQL:  "ISNULL(CAST(col AS NVARCHAR(MAX)),' ')",
         DBMS.SYBASE: "ISNULL(CONVERT(VARCHAR(4000),col),' ')",
-        DBMS.PGSQL:  "COALESCE(CAST(col AS VARCHAR(10000))::text,' ')",
+        DBMS.PGSQL:  "COALESCE(CAST(col AS TEXT)::text,' ')",
         DBMS.ORACLE: "NVL(CAST(col AS VARCHAR(4000)),' ')",
     }