json import/export (gui + batch) and recent-databases menu

The file extension decides the format everywhere:
- import button and .import accept .json/.jsonl: records land in a
  1-column raw table and a ready-made shredding query tab opens
  (json_extract per key of the first record; DuckDB engine uses native
  read_json_auto instead)
- export_writer emits json when the target ends in .json (array) or
  .jsonl (one object per line), so the export dialog and .once/.output
  batch commands gain json for free
- Database > Open Recent lists the ten most recently opened databases
- new code stays python-3.4 compatible (no f-strings; the two of the
  dpi commit converted back)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: stonebig

HISTORY.rst

@@ -4,6 +4,14 @@ Changelog
 (unreleased) : v1.1.1
 ---------------------
 
+* json : the import button also accepts .json / .jsonl files : records land in a 1-column raw table, and a ready-made 'shredding' query tab opens (DuckDB engine : native read_json_auto instead)
+
+* json : exports write json when the file name ends in .json (array) or .jsonl (one object per line) : same buttons, same .once / .output batch commands, the file extension decides the format
+
+* json : .import file.json [table] works in scripts too (batch mode with -sc)
+
+* menu : Database > Open Recent : the ten most recently opened databases
+
 * high-dpi : the app declares dpi-awareness on Windows : no more blurry rendering on scaled screens (125%/150%/200%)
 
 * high-dpi : icons, Treeview row heights, pane widths and the minimal window size follow the screen scale ; other OSes unchanged (scale never goes below 1)

sqlite_bro/sqlite_bro.py

@@ -18,6 +18,7 @@
 import csv
 import datetime
 import io
+import json
 import re
 import shlex  # Simple lexical analysis
 from os.path import expanduser
@@ -145,8 +146,11 @@ def __init__(self, use_gui=True):
             # bitmap-enlarged by Windows, and the natural (content) size
             # would not fit the screen anyway
             self.tk_win.geometry(
-                f"{self.tk_win.winfo_screenwidth() * 4 // 5}"
-                f"x{self.tk_win.winfo_screenheight() * 4 // 5}"
+                "%sx%s"
+                % (
+                    self.tk_win.winfo_screenwidth() * 4 // 5,
+                    self.tk_win.winfo_screenheight() * 4 // 5,
+                )
             )
 
             # start from the real default font size (9 on Windows), so the
@@ -251,6 +255,8 @@ def create_menu(self):
                 command=lambda: self.new_db(":memory:", engine="duckdb"),
             )
         self.menu.add_command(label="Open Database ...", command=self.open_db)
+        self.menu_recent = Menu(self.menu, postcommand=self.refresh_recent_menu)
+        self.menu.add_cascade(menu=self.menu_recent, label="Open Recent")
         self.menu.add_command(
             label="Open Database ...(legacy auto-commit)",
             command=lambda: self.open_db(""),
@@ -359,6 +365,7 @@ def new_db(self, filename="", engine=None):
                 ):
                     os.remove(filename)
             self.conn = Baresql(self.database_file, engine=engine)
+            self.add_recent_db(filename)
             self.show_duckdb_demo()
             self.actualize_db()
 
@@ -379,9 +386,43 @@ def open_db(self, filename="", isolation_level=None, engine=None):
             self.set_initialdir(filename)
             self.database_file = filename
             self.conn = Baresql(self.database_file, engine=engine)
+            self.add_recent_db(filename)
             self.show_duckdb_demo()
             self.actualize_db()
 
+    def recent_dbs(self):
+        """return the list of recently opened databases"""
+        try:
+            with open(
+                os.path.join(self.home, ".sqlite_bro_recent"), encoding="utf-8"
+            ) as f:
+                return [line.strip() for line in f if line.strip()]
+        except (OSError, IOError):
+            return []
+
+    def add_recent_db(self, filename):
+        """remember the ten most recently opened databases"""
+        if filename in ("", ":memory:"):
+            return
+        filename = os.path.abspath(filename)
+        recents = [filename] + [r for r in self.recent_dbs() if r != filename]
+        try:
+            with open(
+                os.path.join(self.home, ".sqlite_bro_recent"), "w", encoding="utf-8"
+            ) as f:
+                f.write("\n".join(recents[:10]))
+        except (OSError, IOError):
+            pass
+
+    def refresh_recent_menu(self):
+        """(re)feed the 'Open Recent' menu, at click time"""
+        self.menu_recent.delete(0, "end")
+        for recent in self.recent_dbs():
+            if os.path.isfile(recent):
+                self.menu_recent.add_command(
+                    label=recent, command=lambda r=recent: self.open_db(r)
+                )
+
     def show_duckdb_demo(self):
         """open the DuckDB welcome demo tab, once, when DuckDB is first used"""
         if (
@@ -1360,6 +1401,32 @@ def bip(c):
                         csv_file = csv_file.strip('"')
                         if (csv_file + "z")[0] == "~":
                             csv_file = os.path.join(self.home, csv_file[1:])
+                    if (
+                        shell_list[0] == ".import"
+                        and len(shell_list) >= 2
+                        and csv_file.lower().endswith((".json", ".jsonl", ".ndjson"))
+                    ):  # the file extension decides the format
+                        table_name = json_table_name(csv_file)
+                        if len(shell_list) >= 3:
+                            table_name = shell_list[2]
+                        records = read_this_json(csv_file)
+                        self.conn.insert_reader(
+                            ((json.dumps(r, ensure_ascii=False),) for r in records),
+                            table_name,
+                            'CREATE TABLE "%s" (json TEXT)' % table_name,
+                            create_table=False,
+                            replace=False,
+                        )
+                        dot_result = 'file %s imported in "%s"' % (
+                            csv_file,
+                            table_name,
+                        )
+                        if log is not None:  # write to logFile
+                            log.write(
+                                '-- File %s imported in "%s"\n'
+                                % (csv_file, table_name)
+                            )
+                    elif shell_list[0] == ".import" and len(shell_list) >= 2:
                         guess = guess_csv(csv_file)
                         if len(shell_list) >= 3:
                             guess.table_name = shell_list[2]
@@ -1613,10 +1680,19 @@ def import_csvtb(self, csv_file=None):
             csv_file = filedialog.askopenfilename(
                 initialdir=self.initialdir,
                 defaultextension=".db",
-                title="Choose a csv fileto import ",
-                filetypes=[("default", "*.csv"), ("other", "*.txt"), ("all", "*.*")],
+                title="Choose a csv or json file to import ",
+                filetypes=[
+                    ("default", "*.csv"),
+                    ("json", "*.json *.jsonl *.ndjson"),
+                    ("other", "*.txt"),
+                    ("all", "*.*"),
+                ],
             )
-        if csv_file != "":
+        if csv_file != "" and csv_file.lower().endswith(
+            (".json", ".jsonl", ".ndjson")
+        ):  # the file extension decides the format
+            self.import_jsontb(csv_file)
+        elif csv_file != "":
             self.set_initialdir(csv_file)
             # guess all via an object
             guess = guess_csv(csv_file)
@@ -1658,6 +1734,35 @@ def import_csvtb(self, csv_file=None):
                 actions,
             )
 
+    def import_jsontb(self, json_file):
+        """import a .json (array) or .jsonl file into a 1-column raw table,
+        and propose a first 'shredding' query in a new tab"""
+        self.set_initialdir(json_file)
+        table_name = json_table_name(json_file)
+        if self.conn.engine == "duckdb":
+            # DuckDB reads (and types) json natively : do it in visible sql
+            base = table_name[: -len("_raw")]
+            query = (
+                'CREATE OR REPLACE TABLE "%s" AS\n'
+                "SELECT * FROM read_json_auto('%s');\n"
+                'SELECT * FROM "%s" LIMIT 100;'
+            ) % (base, json_file.replace("'", "''"), base)
+            self.n.new_query_tab("import %s" % base, query)
+            self.run_tab()
+        else:
+            records = read_this_json(json_file)
+            self.conn.insert_reader(
+                ((json.dumps(r, ensure_ascii=False),) for r in records),
+                table_name,
+                'CREATE TABLE "%s" (json TEXT)' % table_name,
+                create_table=True,
+                replace=True,
+            )
+            self.n.new_query_tab(
+                "shred %s" % table_name, shred_query(table_name, records)
+            )
+        self.actualize_db()
+
     def paste_csvtb(self):
         """paste clipboard into a table, via the csv import dialog"""
         try:
@@ -2279,6 +2384,38 @@ def read_this_csv(csv_file, encoding, delimiter, quotechar, header, decim):
             yield (row)
 
 
+def read_this_json(json_file):
+    """return the records of a .json (array or object) or .jsonl file"""
+    with open(json_file, encoding="utf-8-sig") as f:
+        if json_file.lower().endswith(".json"):
+            data = json.load(f)
+            return data if isinstance(data, list) else [data]
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def json_table_name(json_file):
+    """a table name from a file name : 'd:/some stuff.json' -> some_stuff_raw"""
+    base = os.path.splitext(os.path.basename(json_file))[0]
+    return re.sub(r"\W", "_", base) + "_raw"
+
+
+def shred_query(table_name, records):
+    """sql proposal shredding a raw json table, from the first record keys"""
+    first = records[0] if records else None
+    if not isinstance(first, dict) or not first:
+        return 'SELECT json FROM "%s";' % table_name
+    cols = ",\n".join(
+        "    json_extract(json, '$.%s') AS \"%s\""
+        % (k if re.match(r"^\w+$", k) else '"' + k + '"', k.replace('"', '""'))
+        for k in first
+    )
+    return (
+        '-- shredding "%s" : keys seen in the first record, edit at will\n'
+        'SELECT\n%s\nFROM "%s";\n'
+        "-- nested data ? see json_each(json, '$.path') and json_tree\n"
+    ) % (table_name, cols, table_name)
+
+
 def copy_csv_close_ok(thetop, entries, actions):
     "copy a query result to the clipboard, then close the dialog (action)"
     copy_csv_ok(thetop, entries, actions)
@@ -2792,12 +2929,41 @@ def write_rows(fout):
                 )  # PyPy as a strange list of list
             writer.writerows(cursor.fetchall())
 
+        def write_json_rows(fout, lines_mode):
+            """one object per row ; a .json array, or .jsonl lines"""
+            cols = [i if isinstance(i, str) else i[0] for i in cursor.description]
+
+            def clean(v):
+                if isinstance(v, (bytes, bytearray)):
+                    return v.decode("utf-8", "replace")
+                if v is None or isinstance(v, (int, float, str)):
+                    return v
+                return str(v)  # datetime, Decimal, uuid, ...
+
+            first = True
+            for row in cursor:
+                record = json.dumps(
+                    dict(zip(cols, [clean(v) for v in row])), ensure_ascii=False
+                )
+                if lines_mode:
+                    fout.write(record + "\n")
+                else:
+                    fout.write(("[\n" if first else ",\n") + record)
+                    first = False
+            if not lines_mode:
+                fout.write("[]" if first else "\n]\n")
+
         if hasattr(csv_file, "write"):  # file-like target (e.g. clipboard buffer)
             write_rows(csv_file)
         else:
             write_mode = "w" if initialize else "a"  # Write or Append
+            target = csv_file.lower()
             with open(csv_file, write_mode, newline="", encoding=encoding) as fout:
-                write_rows(fout)
+                if target.endswith((".json", ".jsonl", ".ndjson")):
+                    # the file extension decides the format : json, not csv
+                    write_json_rows(fout, not target.endswith(".json"))
+                else:
+                    write_rows(fout)
         return nb_columns
 
 

sqlite_bro/tests/test_json.py

@@ -0,0 +1,78 @@
+"""json import / export : gui button routing, .import / .once batch, helpers"""
+import json
+import pathlib
+import tempfile
+
+import pytest
+
+from sqlite_bro import sqlite_bro
+from sqlite_bro.sqlite_bro import read_this_json, json_table_name, shred_query
+from sqlite_bro.tests import test_general  # owner of the single gui App
+
+app = sqlite_bro.App(use_gui=False)
+needs_gui = pytest.mark.skipif(not test_general.app.use_gui, reason="no display")
+
+
+def test_helpers():
+    "file name to table name, and shred query proposal"
+    assert json_table_name("d:/some stuff.json") == "some_stuff_raw"
+    records = [{"id": 1, "the name": "bolt"}]
+    query = shred_query("items_raw", records)
+    assert "json_extract(json, '$.id')" in query
+    assert 'json_extract(json, \'$."the name"\')' in query
+    assert shred_query("t", []) == 'SELECT json FROM "t";'
+
+
+def test_read_this_json():
+    ".json array, single object, and .jsonl lines"
+    with tempfile.TemporaryDirectory(prefix=".tmp") as tmp_dir:
+        fj = str(pathlib.PurePath(tmp_dir, "t.json"))
+        with open(fj, "w", encoding="utf-8") as f:
+            f.write('{"solo": true}')
+        assert read_this_json(fj) == [{"solo": True}]
+        fl = str(pathlib.PurePath(tmp_dir, "t.jsonl"))
+        with open(fl, "w", encoding="utf-8") as f:
+            f.write('{"a": 1}\n\n{"a": 2}\n')
+        assert read_this_json(fl) == [{"a": 1}, {"a": 2}]
+
+
+def test_batch_json_import_export():
+    ".import file.json + .once out.json / out.jsonl, no gui needed"
+    app.new_db(":memory:")
+    with tempfile.TemporaryDirectory(prefix=".tmp") as tmp_dir:
+        src = str(pathlib.PurePath(tmp_dir, "items.json")).replace("\\", "/")
+        with open(src, "w", encoding="utf-8") as f:
+            json.dump([{"id": 1, "name": "bolt"}, {"id": 2, "name": "nut"}], f)
+        out_j = str(pathlib.PurePath(tmp_dir, "out.json")).replace("\\", "/")
+        out_l = str(pathlib.PurePath(tmp_dir, "out.jsonl")).replace("\\", "/")
+        script = (
+            '.import "%s"\n'
+            ".once %s\n"
+            "SELECT json_extract(json, '$.id') AS id FROM items_raw;\n"
+            ".once %s\n"
+            "SELECT json_extract(json, '$.name') AS name FROM items_raw;\n"
+        ) % (src, out_j, out_l)
+        app.n.new_query_tab("json batch", script)
+        app.run_tab()
+        with open(out_j, encoding="utf-8") as f:
+            assert json.load(f) == [{"id": 1}, {"id": 2}]
+        with open(out_l, encoding="utf-8") as f:
+            assert [json.loads(l) for l in f] == [{"name": "bolt"}, {"name": "nut"}]
+
+
+@needs_gui
+def test_gui_json_import_opens_shred_tab():
+    "the csv import button routes .json files and proposes a shred query"
+    gui_app = test_general.app
+    gui_app.new_db(":memory:")
+    with tempfile.TemporaryDirectory(prefix=".tmp") as tmp_dir:
+        src = str(pathlib.PurePath(tmp_dir, "goods.json"))
+        with open(src, "w", encoding="utf-8") as f:
+            json.dump([{"id": 7, "name": "screw"}], f)
+        gui_app.import_csvtb(src)
+        assert gui_app.conn.conn.execute(
+            'SELECT count(*) FROM "goods_raw"'
+        ).fetchall() == [(1,)]
+        tab_id = gui_app.n.notebook.select()
+        shred = gui_app.n.fw_labels[tab_id].get("1.0", "end")
+        assert "json_extract" in shred and "goods_raw" in shred