Adding unit tests

Author: stamparm

.github/workflows/tests.yml

@@ -40,6 +40,9 @@ jobs:
       - name: Basic import test
         run: python -c "import sqlmap; import sqlmapapi"
 
+      - name: Unit tests
+        run: python -m unittest discover -s tests -p "test_*.py"
+
       - name: Smoke test
         run: python sqlmap.py --smoke
 

data/txt/sha256sums.txt

@@ -188,7 +188,7 @@ ccc4a717e887652b1fcce073d9409d9c59a3b28548c703a9e453d15845f90cd7  lib/core/patch
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-b0e5477bbbf2eb673fa6b99829c2f51e108bebd3f572d0527e90684c157ba3c6  lib/core/settings.py
+a910686c6eba592ba3f6fc5cbb8bed1bd6c330b0165c7c5dc927a71c5ae8be88  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6  lib/core/target.py
@@ -564,6 +564,42 @@ dcdeed9ee285e63cf06baf8347e3db7f210ef25a63869bab78ce1ec6898ae191  tamper/unional
 7afc4d262b97773e67dcfa3e253a9a060dc964750f01d739636d17ee069f1512  tamper/versionedkeywords.py
 0694e721b07b8242245688be5c7951a3a22f512ed73776a998885e4b1bc82bc7  tamper/versionedmorekeywords.py
 ce1b6bf8f296de27014d6f21aa8b3df9469d418740cd31c93d1f5e36d6c509cf  tamper/xforwardedfor.py
+44401cad3e39ae9fb899ed5d0e2fdd0879561de05c3117f17f3b0db54f4e3724  tests/__init__.py
+bfb553602eb5d20b4ab5928dbcf8e6a3e7e5ff69f7d30d1f53ef6d323c237f6c  tests/test_agent.py
+d4d7d3525d25ce72bf38bd38b5fdf61144e381993d63be7dc72b2b4811ffab67  tests/test_bigarray.py
+27ad87c0ea377e0657bd6f6a4eaa0e9756aa9d28ec0483bdadeb3f66dcc4660d  tests/test_charset.py
+9e678a56e16211c49ab4995b6c658d3f122bfa3b357d9e17ff38f5a489ace6ad  tests/test_cloak.py
+a48c411fea864e6bcd6a1c7e1a35094b8cda8d15088fd9e7b0270542ae20daa9  tests/test_common_helpers.py
+7b72d4f850bbd059b8e95fceb45a58470354cb7270c99b0e9981aaa189af20d1  tests/test_comparison.py
+8593f14a18c4445c58b2e59462adcb761074ac7217cd7c3808519a90ba279bda  tests/test_convert.py
+5016119bdb57094381afdca35ef29a4a6641e26e4b48a9119f1db633e6123d29  tests/test_datafiles.py
+9c240d4f796e56376374d4ce46f358ceb7d48cc6a7427760c5bfb89ff01cb545  tests/test_datatypes.py
+3804eb2d730220360f9dc07d5994eb64e9f65acf3b0d8648df8df2a2177ba8fd  tests/test_decodepage.py
+e40a49cfa73c45b3c3c6d1d1d00738861e270cb7a07b28f5a5356f9c7c800cf2  tests/test_dialect.py
+993a2d4d87c4fbaf261663b069629acc95ee4405aa0c42cf5a8f39649fdb0fff  tests/test_dicts.py
+2bbe4b01f79992cfa8884651fc0a28dbd0e3abb0cbea9eb7eadf1f98ca3c3420  tests/test_encoding.py
+bb6991260a994fcbe79e05febaa34affd5631d02299fbc626820addd5f6ea4f4  tests/test_error_engine.py
+8105de9978fe286a29f6b635a58db1e9998d86e8dded54d7efdfb9d52a121094  tests/test_hashdb.py
+c04e8358fb6df45f69f2f26435c971acde280535bf304e84d30cf2681158c6a7  tests/test_hash.py
+205e84827461101a78b2cffaa3de49795a1214e92276fc7fd40f3456657062b9  tests/test_identifiers_output.py
+5372270b7ed82b62f273c2e9bd1f7ecd8605371e66cd0ad70663762cb08d42f1  tests/test_inference_engine.py
+caa06fed7323b2bb6d0f2443ce343de94f75bf8ad012c055d5e07741d908ebad  tests/test_misc.py
+cde0bea1263ae857561f91ed2bd515e972b716743f017d31b1718a8546c72759  tests/test_pagecontent.py
+4bac34af2abddce003756d6776e89b2fda220bb7603ef3761f4f37ee29f9c369  tests/test_payload_marking.py
+6bfc8201724078bd9d6d559916ef73c9ff97e19b0f2948f37e588a49b027795f  tests/test_payloads_structure.py
+5c95e7863190e440234f231864fb1219c35207132762858cc95181c57086bafc  tests/test_replication.py
+cec98d72992c0799229a780fa7f0d7f3fb01ec2d708187ce0e4a05c8612f291b  tests/test_safe2bin.py
+a1c6cda1e5b483f61e6a4f8ddd0b06a15ddaa3fd2119bfb9dbd9cc970d7a751d  tests/test_settings_regex.py
+d3d991331096e16e5019de3d652e9fff92c09bd9f97c50b1c2c3ceb0ed49b17e  tests/test_sqlparse.py
+41907c873663401f979b87eaff3efc8d52e0ce96cbe1eef7aa70c6d3af8cd5cf  tests/test_strings.py
+f3a628db8a3e05baee580c02132e95b164695e4b3ee1785707e3ea148702449a  tests/test_tamper.py
+b3e13febe9e0ff6f97334f2868655bfdbaa18755e464a6dc4c6d424f513bad02  tests/test_targeturl.py
+639851dc68f62b559b200b09c308e64e453f414969940005bac75dc0ab07a6b6  tests/test_texthelpers.py
+708b3c040f8b677a84020dd6f7c4242f77260b3c6d2697fe8189e1881b0e1365  tests/test_union_engine.py
+4b646f513c6da1e33200184ed6eabe0aa345eb2e2a19598dc123e191168591bf  tests/test_urls.py
+4f095ebda1b9bddde082ed464e863400cf23e9bf26f081948706213b35069195  tests/_testutils.py
+2364db35025a53ea4e5a0a80c034997642785f7e6d1566d0d0f1db959fe3c82e  tests/test_utils.py
+81bb6d7449f224fa337734ae361c1a340bf9a51768a854d6a1a6e718ed1263ca  tests/test_wordlist.py
 55eaefc664bd8598329d535370612351ec8443c52465f0a37172ea46a97c458a  thirdparty/ansistrm/ansistrm.py
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  thirdparty/ansistrm/__init__.py
 f597b49ef445bfbfb8f98d1f1a08dcfe4810de5769c0abfab7cdce4eebbfcae7  thirdparty/beautifulsoup/beautifulsoup.py

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.6.105"
+VERSION = "1.10.6.106"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

tests/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
+See the file 'LICENSE' for copying permission
+"""

tests/_testutils.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
+See the file 'LICENSE' for copying permission
+
+Shared bootstrap for the sqlmap unit/regression test suite.
+
+Brings sqlmap's global state (conf/kb, the 'reversible' codec, cross-references,
+option defaults) up far enough that pure/near-pure library functions can be
+exercised in isolation - WITHOUT a live target, network, or DBMS.
+
+stdlib unittest only (no pytest / no pip); works on Python 2.7 and 3.x.
+"""
+
+import os
+import sys
+import warnings
+
+# Quieten import-time noise before any sqlmap/3rd-party module is imported by bootstrap():
+# e.g. cryptography's "Python 2 is no longer supported" CryptographyDeprecationWarning via pymysql.
+warnings.filterwarnings("ignore", message=".*Python 2 is no longer supported.*")
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+# sqlmap reconfigures stdout at startup; py3 emits a benign RuntimeWarning about line buffering
+warnings.filterwarnings("ignore", message=".*line buffering.*binary mode.*")
+
+_BOOTSTRAPPED = False
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def bootstrap():
+    """Idempotently initialize sqlmap global state for testing."""
+    global _BOOTSTRAPPED
+    if _BOOTSTRAPPED:
+        return
+
+    if ROOT not in sys.path:
+        sys.path.insert(0, ROOT)
+    # a dummy target so cmdLineParser() populates ALL option defaults without erroring;
+    # save/restore the real argv so the unittest runner isn't confused by it
+    _orig_argv = list(sys.argv)
+    sys.argv = ["sqlmap.py", "-u", "http://test.invalid/?id=1"]
+
+    from lib.core.common import setPaths
+    from lib.core.patch import dirtyPatches, resolveCrossReferences
+    setPaths(ROOT)
+    dirtyPatches()                 # registers the 'reversible' codec error handler, etc.
+    resolveCrossReferences()
+
+    from lib.core.option import _setConfAttributes, _setKnowledgeBaseAttributes, _loadQueries
+    _setConfAttributes()
+    _setKnowledgeBaseAttributes()
+    _loadQueries()                 # populate the `queries` dict from queries.xml (needed by dialect builders)
+
+    from lib.core.data import conf, kb
+    from lib.core.defaults import defaults
+    from lib.parse.cmdline import cmdLineParser
+
+    args = cmdLineParser()
+    parsed = args.__dict__ if hasattr(args, "__dict__") else dict(args)
+    for k, v in parsed.items():
+        conf[k] = v
+    # overlay canonical defaults for options left None (sqlmap does this during init)
+    for k, v in defaults.items():
+        if conf.get(k) is None:
+            conf[k] = v
+
+    kb.binaryField = False         # normally set lazily during extraction
+
+    # Silence sqlmap's application logger - tests assert on results, not log output, and the
+    # INFO/WARNING/ERROR chatter (column counts, reflective-value notices, an intentionally
+    # malformed-deflate error, etc.) just clutters the unittest report.
+    import logging
+    logging.getLogger("sqlmapLog").setLevel(logging.CRITICAL + 1)
+
+    sys.argv = _orig_argv          # restore so unittest's arg parsing works
+    _BOOTSTRAPPED = True
+
+
+def set_dbms(name):
+    """Force the identified back-end DBMS for dialect-dependent functions.
+
+    Uses forceDbms (not setDbms) so switching DBMS repeatedly in one process does
+    not trigger the interactive fingerprint-mismatch prompt.
+    """
+    from lib.core.common import Backend
+    from lib.core.data import kb
+    kb.stickyDBMS = False
+    Backend.forceDbms(name)

tests/test_agent.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
+See the file 'LICENSE' for copying permission
+
+Payload assembly helpers in lib/core/agent.py.
+
+These are the (mostly) DBMS-independent string transforms that wrap, fold and
+clean a payload on its way to the wire: prefix/suffix, payload delimiters,
+field extraction, CONCAT folding, and RAND-marker cleanup. All values below
+were probed from real output, not assumed.
+"""
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from _testutils import bootstrap, set_dbms
+bootstrap()
+
+from lib.core.agent import agent
+from lib.core.enums import DBMS
+from lib.core.settings import PAYLOAD_DELIMITER
+
+
+class TestPayloadDelimiters(unittest.TestCase):
+    def test_add(self):
+        self.assertEqual(agent.addPayloadDelimiters("1 AND 1=1"),
+                         "%s1 AND 1=1%s" % (PAYLOAD_DELIMITER, PAYLOAD_DELIMITER))
+
+    def test_remove(self):
+        wrapped = "%spayload%s" % (PAYLOAD_DELIMITER, PAYLOAD_DELIMITER)
+        self.assertEqual(agent.removePayloadDelimiters(wrapped), "payload")
+
+    def test_remove_none_is_none(self):
+        self.assertIsNone(agent.removePayloadDelimiters(None))
+
+    def test_roundtrip(self):
+        for p in ["1=1", "1 AND SLEEP(5)", "' OR '1'='1", "", "a%sb" % "x"]:
+            self.assertEqual(agent.removePayloadDelimiters(agent.addPayloadDelimiters(p)), p,
+                             msg="delimiter round-trip for %r" % p)
+
+
+class TestPrefixSuffix(unittest.TestCase):
+    def test_prefix_default_pads_space(self):
+        # with no configured prefix, a single leading space is prepended
+        self.assertEqual(agent.prefixQuery("1=1"), " 1=1")
+
+    def test_suffix_default_identity(self):
+        self.assertEqual(agent.suffixQuery("1=1"), "1=1")
+
+
+class TestGetFields(unittest.TestCase):
+    def test_extracts_select_list(self):
+        # getFields(query) returns an 8-tuple; the fields-bearing slots are:
+        #   [0],[1] = regex match objects for the SELECT/expression (must be found, not None)
+        #   [5]     = parsed field list, [6] = raw fields string
+        # (asserting the match objects guards against a refactor that silently shifts the tuple)
+        result = agent.getFields("SELECT a,b FROM t")
+        self.assertIsNotNone(result[0], msg="getFields did not match the SELECT")
+        self.assertEqual(result[5], ["a", "b"])
+        self.assertEqual(result[6], "a,b")
+
+
+class TestConcatQuery(unittest.TestCase):
+    def test_mysql_concat_folding(self):
+        set_dbms(DBMS.MYSQL)
+        q = agent.concatQuery("SELECT a FROM t")
+        # folds the field through CONCAT with the start/stop delimiters and keeps the FROM
+        self.assertTrue(q.startswith("CONCAT("), msg=q)
+        self.assertIn("IFNULL(CAST(a AS NCHAR),' ')", q)
+        self.assertTrue(q.endswith("FROM t"), msg=q)
+
+
+class TestCleanupPayload(unittest.TestCase):
+    def test_randnum_marker_replaced_with_digits(self):
+        out = agent.cleanupPayload("SELECT [RANDNUM]")
+        self.assertNotIn("[RANDNUM]", out, msg="marker not replaced: %r" % out)   # actually substituted
+        self.assertTrue(out.startswith("SELECT "), msg=out)
+        self.assertTrue(out.split()[-1].isdigit(), msg=out)   # ...and replaced with a concrete number
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

tests/test_bigarray.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+"""
+Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
+See the file 'LICENSE' for copying permission
+
+BigArray disk-spill semantics (lib/core/bigarray.py).
+
+BigArray is the structure that lets sqlmap dump tables far larger than RAM: once
+the in-memory chunk exceeds chunk_size it is pickled to a temp file and a new
+chunk starts. The tricky, easy-to-break part is that indexing / iteration /
+pop / pickling must stay correct ACROSS the in-memory<->on-disk boundary.
+
+These force a spill with a tiny chunk_size and assert the data survives intact.
+"""
+
+import os
+import pickle
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from _testutils import bootstrap
+bootstrap()
+
+from lib.core.bigarray import BigArray
+
+N = 5000
+
+
+def _make_spilled():
+    # tiny chunk_size guarantees many on-disk chunks for N items
+    ba = BigArray(chunk_size=1024)
+    for i in range(N):
+        ba.append("item-%d" % i)
+    return ba
+
+
+class TestSpill(unittest.TestCase):
+    def test_actually_spilled_to_disk(self):
+        ba = _make_spilled()
+        self.assertGreater(len(ba.chunks), 1, msg="expected multiple chunks (a disk spill)")
+        # stronger than "more than one chunk": at least one chunk must be a real on-disk file
+        # (spilled chunks are stored as filenames). Otherwise this could pass while everything
+        # stayed in RAM.
+        disk_chunks = [c for c in ba.chunks if isinstance(c, str)]
+        self.assertTrue(disk_chunks, msg="no chunk was spilled to disk")
+        self.assertTrue(os.path.exists(disk_chunks[0]), msg="spilled chunk file missing on disk")
+
+    def test_len(self):
+        self.assertEqual(len(_make_spilled()), N)
+
+    def test_random_access_across_boundary(self):
+        ba = _make_spilled()
+        for i in (0, 1, 499, 500, 2500, N - 1):
+            self.assertEqual(ba[i], "item-%d" % i, msg="ba[%d]" % i)
+
+    def test_negative_index(self):
+        ba = _make_spilled()
+        self.assertEqual(ba[-1], "item-%d" % (N - 1))
+
+    def test_iteration_order_preserved(self):
+        ba = _make_spilled()
+        for idx, value in enumerate(ba):
+            if value != "item-%d" % idx:
+                self.fail("iteration order broke at %d: %r" % (idx, value))
+        self.assertEqual(idx, N - 1)
+
+    def test_pop_from_end(self):
+        ba = _make_spilled()
+        self.assertEqual(ba.pop(), "item-%d" % (N - 1))
+        self.assertEqual(len(ba), N - 1)
+
+    def test_pickle_roundtrip_across_spill(self):
+        ba = _make_spilled()
+        restored = pickle.loads(pickle.dumps(ba))
+        self.assertIsInstance(restored, BigArray)
+        self.assertEqual(len(restored), N)
+        self.assertEqual(restored[0], "item-0")
+        self.assertEqual(restored[N - 1], "item-%d" % (N - 1))
+
+
+class TestInMemorySmall(unittest.TestCase):
+    def test_no_spill_for_small(self):
+        ba = BigArray([1, 2, 3])
+        self.assertEqual(len(ba), 3)
+        self.assertEqual(list(ba), [1, 2, 3])
+        # the actual point of this test (the name promised it): a tiny array stays in ONE
+        # in-memory chunk and never touches disk
+        self.assertEqual(len(ba.chunks), 1, msg="small array unexpectedly spilled: %r" % (ba.chunks,))
+        self.assertFalse(any(isinstance(c, str) for c in ba.chunks), msg="small array wrote a disk chunk")
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)