|
| 1 | +#!/usr/bin/env python |
| 2 | + |
| 3 | +""" |
| 4 | +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) |
| 5 | +See the file 'LICENSE' for copying permission |
| 6 | +
|
| 7 | +JSONL output of the per-table dumper (Dump.dbTableValues in lib/core/dump.py). |
| 8 | +
|
| 9 | +--dump-format=JSONL writes one self-describing JSON object per row to a |
| 10 | +<host>/dump/<db>/<table>.jsonl file, streaming-safe (one independent line per |
| 11 | +row, no surrounding array/header/footer). These tests pin the contract that an |
| 12 | +automated consumer relies on: column order preserved (so it matches the CSV |
| 13 | +column order and is reproducible on Python 2's unordered dict), the DB-NULL |
| 14 | +marker (" ") mapped to JSON null exactly like --report-json, the empty string |
| 15 | +left intact (NOT collapsed to null), and a strict one-object-per-line layout. |
| 16 | +""" |
| 17 | + |
| 18 | +import json |
| 19 | +import os |
| 20 | +import shutil |
| 21 | +import sys |
| 22 | +import tempfile |
| 23 | +import unittest |
| 24 | + |
| 25 | +from collections import OrderedDict |
| 26 | + |
| 27 | +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| 28 | +from _testutils import bootstrap |
| 29 | +bootstrap() |
| 30 | + |
| 31 | +from lib.core.common import Backend |
| 32 | +from lib.core.data import conf, kb |
| 33 | +from lib.core.dump import Dump |
| 34 | +from lib.core.enums import DUMP_FORMAT |
| 35 | + |
| 36 | + |
| 37 | +class _JsonlDumpCase(unittest.TestCase): |
| 38 | + def setUp(self): |
| 39 | + self._saved = dict((k, conf.get(k)) for k in ("dumpFormat", "dumpPath", "dumpFile", "col", "api", "reportCollector", "limitStart", "limitStop", "csvDel", "forceDbms", "dbms")) |
| 40 | + self._savedKb = dict((k, kb.get(k)) for k in ("forcedDbms", "dbms")) |
| 41 | + # A DBMS leaked from an earlier test (e.g. one that uppercases identifiers) would change |
| 42 | + # both the on-disk filename and the JSON keys, so pin a neutral, case-preserving back-end. |
| 43 | + conf.forceDbms = conf.dbms = None |
| 44 | + kb.dbms = None |
| 45 | + Backend.forceDbms("MySQL") |
| 46 | + self.tmp = tempfile.mkdtemp(prefix="sqlmap-jsonl-test") |
| 47 | + conf.dumpFormat = DUMP_FORMAT.JSONL |
| 48 | + conf.dumpPath = self.tmp |
| 49 | + conf.dumpFile = None |
| 50 | + conf.col = None |
| 51 | + conf.api = False |
| 52 | + conf.reportCollector = None |
| 53 | + conf.limitStart = conf.limitStop = None |
| 54 | + conf.csvDel = "," |
| 55 | + self.d = Dump() |
| 56 | + self.d._write = lambda *a, **k: None # silence the console table |
| 57 | + |
| 58 | + def tearDown(self): |
| 59 | + for k, v in self._saved.items(): |
| 60 | + conf[k] = v |
| 61 | + for k, v in self._savedKb.items(): |
| 62 | + kb[k] = v |
| 63 | + shutil.rmtree(self.tmp, ignore_errors=True) |
| 64 | + |
| 65 | + def _dump(self, table_values): |
| 66 | + self.d.dbTableValues(table_values) |
| 67 | + db = table_values["__infos__"]["db"] or "All" |
| 68 | + path = os.path.join(self.tmp, db, "%s.jsonl" % table_values["__infos__"]["table"]) |
| 69 | + with open(path) as f: |
| 70 | + content = f.read() |
| 71 | + return content |
| 72 | + |
| 73 | + def _rows(self, content): |
| 74 | + return [json.loads(line) for line in content.splitlines() if line.strip()] |
| 75 | + |
| 76 | + |
| 77 | +class TestJsonlContract(_JsonlDumpCase): |
| 78 | + def test_one_object_per_row(self): |
| 79 | + content = self._dump({ |
| 80 | + "__infos__": {"count": 2, "db": "testdb", "table": "users"}, |
| 81 | + "id": {"length": 2, "values": ["1", "2"]}, |
| 82 | + "name": {"length": 6, "values": ["luther", "fluffy"]}, |
| 83 | + }) |
| 84 | + # exactly N non-empty lines, each terminated by a newline, each a standalone object |
| 85 | + lines = content.splitlines() |
| 86 | + self.assertEqual(len(lines), 2) |
| 87 | + self.assertTrue(content.endswith("\n")) |
| 88 | + rows = self._rows(content) |
| 89 | + self.assertEqual(rows[0], {"id": "1", "name": "luther"}) |
| 90 | + self.assertEqual(rows[1], {"id": "2", "name": "fluffy"}) |
| 91 | + |
| 92 | + def test_no_header_or_footer(self): |
| 93 | + # unlike CSV (header row) / HTML (doc scaffold), JSONL must be pure data lines |
| 94 | + content = self._dump({ |
| 95 | + "__infos__": {"count": 1, "db": "testdb", "table": "t"}, |
| 96 | + "id": {"length": 2, "values": ["1"]}, |
| 97 | + }) |
| 98 | + lines = [l for l in content.splitlines() if l.strip()] |
| 99 | + self.assertEqual(len(lines), 1) |
| 100 | + self.assertEqual(json.loads(lines[0]), {"id": "1"}) |
| 101 | + |
| 102 | + def test_db_null_becomes_json_null(self): |
| 103 | + # sqlmap stores a DB NULL as a single space (" "); the machine format must emit JSON null, |
| 104 | + # consistent with --report-json. An empty string is a real value and must stay "". |
| 105 | + content = self._dump({ |
| 106 | + "__infos__": {"count": 1, "db": "testdb", "table": "t"}, |
| 107 | + "a": {"length": 1, "values": [" "]}, # DB NULL marker |
| 108 | + "b": {"length": 1, "values": [""]}, # genuine empty string |
| 109 | + "c": {"length": 1, "values": ["x"]}, |
| 110 | + }) |
| 111 | + row = self._rows(content)[0] |
| 112 | + self.assertIsNone(row["a"]) |
| 113 | + self.assertEqual(row["b"], "") |
| 114 | + self.assertEqual(row["c"], "x") |
| 115 | + |
| 116 | + def test_missing_value_is_null(self): |
| 117 | + # a column whose values list is short for this row index must serialize as null, not crash |
| 118 | + content = self._dump({ |
| 119 | + "__infos__": {"count": 2, "db": "testdb", "table": "t"}, |
| 120 | + "id": {"length": 2, "values": ["1", "2"]}, |
| 121 | + "lagging": {"length": 4, "values": ["only-one"]}, # missing index 1 |
| 122 | + }) |
| 123 | + rows = self._rows(content) |
| 124 | + self.assertEqual(rows[0], {"id": "1", "lagging": "only-one"}) |
| 125 | + self.assertEqual(rows[1], {"id": "2", "lagging": None}) |
| 126 | + |
| 127 | + def test_column_order_matches_csv(self): |
| 128 | + # The serialized byte stream must keep the (priority-sorted) column order so output is |
| 129 | + # reproducible - even on Python 2 where a plain dict would not - and that order must be |
| 130 | + # the SAME one CSV uses. Build the input as an OrderedDict so the expectation is fixed, |
| 131 | + # then dump the identical data as both JSONL and CSV and compare the column sequences. |
| 132 | + def table(): |
| 133 | + tv = OrderedDict() |
| 134 | + tv["__infos__"] = {"count": 1, "db": "testdb", "table": "t"} |
| 135 | + tv["zebra"] = {"length": 1, "values": ["1"]} |
| 136 | + tv["alpha"] = {"length": 1, "values": ["2"]} |
| 137 | + tv["middle"] = {"length": 1, "values": ["3"]} |
| 138 | + return tv |
| 139 | + |
| 140 | + jsonl_line = [l for l in self._dump(table()).splitlines() if l.strip()][0] |
| 141 | + jsonl_order = [k for k, _ in json.loads(jsonl_line, object_pairs_hook=lambda p: p)] |
| 142 | + |
| 143 | + conf.dumpFormat = DUMP_FORMAT.CSV |
| 144 | + csv_path = os.path.join(self.tmp, "testdb", "t.csv") |
| 145 | + if os.path.exists(csv_path): |
| 146 | + os.remove(csv_path) |
| 147 | + self.d.dbTableValues(table()) |
| 148 | + with open(csv_path) as f: |
| 149 | + csv_header = f.read().splitlines()[0] |
| 150 | + csv_order = [c.strip() for c in csv_header.split(conf.csvDel)] |
| 151 | + |
| 152 | + self.assertEqual(jsonl_order, csv_order) |
| 153 | + |
| 154 | + def test_unicode_value_not_escaped(self): |
| 155 | + # ensure_ascii=False keeps multibyte data readable; it must round-trip through json.loads |
| 156 | + content = self._dump({ |
| 157 | + "__infos__": {"count": 1, "db": "testdb", "table": "t"}, |
| 158 | + "name": {"length": 6, "values": [u"\u0107evap"]}, |
| 159 | + }) |
| 160 | + self.assertEqual(self._rows(content)[0]["name"], u"\u0107evap") |
| 161 | + |
| 162 | + |
| 163 | +if __name__ == "__main__": |
| 164 | + unittest.main() |
0 commit comments