From f97f906943149acc306334e75a91cd7932471b61 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Fri, 3 Jul 2026 12:25:42 +0200 Subject: [PATCH 1/4] Add get_db_overview for fast, lightweight run listing Add `qcodes.dataset.sqlite.db_overview.get_db_overview`, which fetches a lightweight overview of the runs in a database via a single JOIN query on the `runs` and `experiments` tables, without instantiating a DataSet per run. This makes listing databases with many thousands of runs near-instant. The function and its return type `RunOverviewDict` are exported on the public `qcodes.dataset` namespace. An `extra_columns` argument allows reading ad-hoc metadata columns added via `DataSet.add_metadata`. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/qcodes/dataset/__init__.py | 3 + src/qcodes/dataset/sqlite/db_overview.py | 253 +++++++++++++++++++++++ tests/dataset/test_db_overview.py | 174 ++++++++++++++++ 3 files changed, 430 insertions(+) create mode 100644 src/qcodes/dataset/sqlite/db_overview.py create mode 100644 tests/dataset/test_db_overview.py diff --git a/src/qcodes/dataset/__init__.py b/src/qcodes/dataset/__init__.py index 88e5b43d7f24..0494069ab0b5 100644 --- a/src/qcodes/dataset/__init__.py +++ b/src/qcodes/dataset/__init__.py @@ -56,6 +56,7 @@ initialise_or_create_database_at, initialised_database_at, ) +from .sqlite.db_overview import RunOverviewDict, get_db_overview from .sqlite.settings import SQLiteSettings from .threading import ( SequentialParamsCaller, @@ -79,6 +80,7 @@ "ParamSpec", "ParamSpecTree", "RunDescriber", + "RunOverviewDict", "SQLiteSettings", "SequentialParamsCaller", "ThreadPoolParamsCaller", @@ -95,6 +97,7 @@ "export_datasets_and_create_metadata_db", "extract_runs_into_db", "get_data_export_path", + "get_db_overview", "get_default_experiment_id", "get_guids_by_run_spec", "guids_from_dbs", diff --git a/src/qcodes/dataset/sqlite/db_overview.py b/src/qcodes/dataset/sqlite/db_overview.py new file mode 100644 index 000000000000..977738f3f401 --- /dev/null +++ b/src/qcodes/dataset/sqlite/db_overview.py @@ -0,0 +1,253 @@ +""" +This module provides a fast, lightweight overview of the runs stored in a +QCoDeS database. + +:func:`get_db_overview` issues a single ``JOIN`` query against the ``runs`` and +``experiments`` tables to collect run metadata (experiment/sample names, time +stamps, record counts, guids, ...) without instantiating a ``DataSet`` object +per run. This avoids the expensive ``experiments()`` + ``data_sets()`` +enumeration and makes it possible to list the contents of databases with many +thousands of runs almost instantly. It is primarily intended for tools that +need to display a table of runs (e.g. dataset browsers). + +The queries rely on the stable QCoDeS database schema (the ``runs`` and +``experiments`` tables) which has been unchanged across many QCoDeS versions. +""" + +from __future__ import annotations + +import datetime +import json +import logging +from contextlib import closing, nullcontext +from typing import TYPE_CHECKING + +from typing_extensions import TypedDict + +from qcodes.dataset.sqlite.database import conn_from_dbpath_or_conn +from qcodes.dataset.sqlite.query_helpers import is_column_in_table + +if TYPE_CHECKING: + from collections.abc import Sequence + from pathlib import Path + + from qcodes.dataset.sqlite.connection import AtomicConnection + +log = logging.getLogger(__name__) + + +class RunOverviewDict(TypedDict): + """ + Lightweight overview of a single run. + + Contains only cheap-to-query metadata: no snapshot, no data and no full + ``DataSet`` object. Extra ad-hoc metadata columns requested via the + ``extra_columns`` argument of :func:`get_db_overview` are added to the + dictionary under their column name in addition to the keys documented here. + """ + + #: ``run_id`` of the run. + run_id: int + #: Name of the experiment the run belongs to. + experiment: str + #: Sample name of the experiment the run belongs to. + sample: str + #: Name of the run. + name: str + #: Local date the run was started, formatted as ``YYYY-MM-DD`` (empty + #: string if unknown). + started_date: str + #: Local time the run was started, formatted as ``HH:MM:SS`` (empty string + #: if unknown). + started_time: str + #: Local date the run was completed, formatted as ``YYYY-MM-DD`` (empty + #: string if the run has not completed). + completed_date: str + #: Local time the run was completed, formatted as ``HH:MM:SS`` (empty + #: string if the run has not completed). + completed_time: str + #: Best-effort number of data points in the run, see :func:`get_db_overview`. + records: int + #: guid of the run. + guid: str + + +def _format_timestamp(ts: float | None) -> tuple[str, str]: + """ + Convert a unix timestamp into ``(date, time)`` strings in local time. + + Returns a pair of empty strings if the timestamp is missing or invalid. + """ + if ts is None or ts == 0: + return "", "" + try: + dt = datetime.datetime.fromtimestamp(ts) + except (OSError, ValueError, OverflowError): + return "", "" + return dt.strftime("%Y-%m-%d"), dt.strftime("%H:%M:%S") + + +def _records_from_run_description(run_description_json: str | None) -> int: + """ + Extract a data-point count from the ``shapes`` field of a run description. + + A QCoDeS run description may contain a ``shapes`` mapping from dependent + parameter names to their shape tuples. The count returned here is the sum + over all dependent parameters of the product of their shape dimensions. + Returns ``0`` if the run description is missing, cannot be parsed or does + not contain shape information. + """ + if not run_description_json: + return 0 + try: + desc = json.loads(run_description_json) + except (json.JSONDecodeError, TypeError): + return 0 + shapes = desc.get("shapes") if isinstance(desc, dict) else None + if not shapes: + return 0 + total = 0 + for shape in shapes.values(): + if isinstance(shape, (list, tuple)) and len(shape) > 0: + n = 1 + for dim in shape: + n *= dim + total += n + return total + + +def get_db_overview( + path_to_db: str | Path | None = None, + conn: AtomicConnection | None = None, + start_run_id: int = 0, + extra_columns: Sequence[str] | None = None, +) -> dict[int, RunOverviewDict]: + """ + Get a lightweight overview of the runs in a QCoDeS database. + + This uses a single SQL ``JOIN`` query on the ``runs`` and ``experiments`` + tables to fetch run metadata, avoiding the much more expensive + ``experiments()`` + ``data_sets()`` enumeration that instantiates a + ``DataSet`` object per run. It is therefore well suited for listing the + contents of databases with many runs. + + The reported number of ``records`` is a best-effort estimate of the number + of data points in a run: + + * For completed runs the shape information stored in the run description is + preferred (it is the authoritative final count), falling back to the + number of rows in the results table. + * For runs that are still in progress the number of rows in the results + table is preferred (it grows as data is added), falling back to the run + description shapes. + * If neither is available the ``result_counter`` of the run is used, which + counts the number of insertions rather than data points. + + Only one of ``path_to_db`` and ``conn`` should be supplied. If a + ``path_to_db`` is given the database is opened in read-only mode and the + connection is closed again before returning. + + Args: + path_to_db: Path to the database file. Opened read-only if given. + conn: An existing connection to use instead of ``path_to_db``. It is + left open by this function. + start_run_id: Only return runs whose ``run_id`` is strictly greater + than this value. Use ``0`` (the default) to get all runs, or pass + the last known ``run_id`` to fetch only newly added runs. + extra_columns: Names of additional ``runs``-table columns to include in + each :class:`RunOverviewDict`. Columns that do not exist in the + ``runs`` table of the given database are silently skipped. This is + useful for reading ad-hoc metadata columns added via + ``DataSet.add_metadata``. + + Returns: + A dictionary mapping ``run_id`` to a :class:`RunOverviewDict`. + + """ + overview: dict[int, RunOverviewDict] = {} + + created_conn = conn is None + connection = conn_from_dbpath_or_conn( + conn=conn, path_to_db=path_to_db, read_only=True + ) + manager = closing(connection) if created_conn else nullcontext(connection) + + with manager as c: + valid_extra_columns = [ + col for col in (extra_columns or []) if is_column_in_table(c, "runs", col) + ] + extra_select = "".join(f", r.{col}" for col in valid_extra_columns) + + # ``run_description`` is queried to derive the record count for + # completed runs; the (potentially large) snapshot is deliberately + # excluded. + query = f""" + SELECT r.run_id, e.name, e.sample_name, r.name, + r.run_timestamp, r.completed_timestamp, + r.result_counter, r.guid, r.result_table_name, + r.run_description{extra_select} + FROM runs r + JOIN experiments e ON r.exp_id = e.exp_id + WHERE r.run_id > ? + ORDER BY r.run_id + """ + + try: + rows = c.execute(query, (start_run_id,)).fetchall() + except Exception as e: + log.warning("Could not query database overview: %s", e) + return overview + + # ``result_counter`` in the runs table counts INSERT calls, not data + # points. For the ``array`` paramtype a single INSERT can contain many + # data points, so the real row count of the results table is queried + # separately. + result_tables = {row[8] for row in rows if row[8]} + row_counts: dict[str, int] = {} + for table in result_tables: + try: + count = c.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone() + except Exception: + continue # results table may not exist yet + row_counts[table] = count[0] if count else 0 + + n_fixed = 10 # number of columns selected before ``extra_columns`` + for row in rows: + run_id = row[0] + started_date, started_time = _format_timestamp(row[4]) + completed_date, completed_time = _format_timestamp(row[5]) + result_table = row[8] or "" + is_completed = row[5] is not None and row[5] != 0 + + if is_completed: + records = _records_from_run_description(row[9]) + if records == 0: + records = row_counts.get(result_table, 0) + else: + records = row_counts.get(result_table, 0) + if records == 0: + records = _records_from_run_description(row[9]) + if records == 0: + records = row[6] or 0 + + entry: RunOverviewDict = { + "run_id": run_id, + "experiment": row[1] or "", + "sample": row[2] or "", + "name": row[3] or "", + "started_date": started_date, + "started_time": started_time, + "completed_date": completed_date, + "completed_time": completed_time, + "records": records, + "guid": row[7] or "", + } + if valid_extra_columns: + extra = { + col: row[n_fixed + i] for i, col in enumerate(valid_extra_columns) + } + entry.update(extra) # type: ignore[typeddict-item] + + overview[run_id] = entry + + return overview diff --git a/tests/dataset/test_db_overview.py b/tests/dataset/test_db_overview.py new file mode 100644 index 000000000000..20fa2a07120d --- /dev/null +++ b/tests/dataset/test_db_overview.py @@ -0,0 +1,174 @@ +"""Tests for :mod:`qcodes.dataset.sqlite.db_overview`.""" + +from __future__ import annotations + +import json +import sqlite3 +from typing import TYPE_CHECKING + +import pytest + +from qcodes.dataset import ( + get_db_overview, + initialise_or_create_database_at, + load_or_create_experiment, + new_data_set, +) +from qcodes.dataset.descriptions.dependencies import InterDependencies_ +from qcodes.dataset.sqlite.database import connect +from qcodes.dataset.sqlite.db_overview import ( + _format_timestamp, + _records_from_run_description, +) +from qcodes.parameters import ParamSpecBase + +if TYPE_CHECKING: + from pathlib import Path + + +def _make_db_with_runs( + db_path: str, + n_runs: int = 1, + n_points: int = 10, + experiment_name: str = "test_exp", + sample_name: str = "test_sample", +) -> None: + """Create a database at ``db_path`` with ``n_runs`` simple numeric runs.""" + initialise_or_create_database_at(db_path) + load_or_create_experiment(experiment_name, sample_name=sample_name) + p_x = ParamSpecBase("x", "numeric") + p_y = ParamSpecBase("y", "numeric") + interdeps = InterDependencies_(dependencies={p_y: (p_x,)}) + + for r in range(n_runs): + ds = new_data_set(f"run_{r + 1}") + ds.set_interdependencies(interdeps) + ds.mark_started() + for i in range(n_points): + ds.add_results([{p_x.name: float(i), p_y.name: float(i**2)}]) + ds.mark_completed() + ds.conn.close() + + +def test_records_from_run_description() -> None: + desc = json.dumps({"version": 3, "shapes": {"dep1": [100, 50]}}) + assert _records_from_run_description(desc) == 5000 + + multi = json.dumps({"shapes": {"dep1": [10], "dep2": [5, 4]}}) + assert _records_from_run_description(multi) == 30 + + assert _records_from_run_description(json.dumps({"version": 3})) == 0 + assert _records_from_run_description(json.dumps({"shapes": {}})) == 0 + assert _records_from_run_description(None) == 0 + assert _records_from_run_description("") == 0 + assert _records_from_run_description("not valid json") == 0 + + +def test_format_timestamp() -> None: + assert _format_timestamp(None) == ("", "") + assert _format_timestamp(0) == ("", "") + + date, time = _format_timestamp(1_600_000_000.0) + assert len(date) == len("YYYY-MM-DD") + assert date.count("-") == 2 + assert len(time) == len("HH:MM:SS") + assert time.count(":") == 2 + + +def test_get_db_overview_basic_fields(tmp_path: Path) -> None: + db_path = str(tmp_path / "basic.db") + _make_db_with_runs(db_path, n_runs=2) + + overview = get_db_overview(db_path) + + assert set(overview.keys()) == {1, 2} + for run_id, info in overview.items(): + assert info["run_id"] == run_id + assert info["experiment"] == "test_exp" + assert info["sample"] == "test_sample" + assert info["name"] == f"run_{run_id}" + assert info["guid"] + assert info["started_date"] and info["started_time"] + assert info["completed_date"] and info["completed_time"] + + +def test_get_db_overview_counts_result_rows(tmp_path: Path) -> None: + db_path = str(tmp_path / "counts.db") + _make_db_with_runs(db_path, n_runs=3, n_points=10) + + overview = get_db_overview(db_path) + + conn = sqlite3.connect(db_path) + try: + for run_id, info in overview.items(): + (table_name,) = conn.execute( + "SELECT result_table_name FROM runs WHERE run_id=?", (run_id,) + ).fetchone() + (actual,) = conn.execute(f'SELECT COUNT(*) FROM "{table_name}"').fetchone() + assert info["records"] == actual == 10 + finally: + conn.close() + + +def test_get_db_overview_incremental(tmp_path: Path) -> None: + db_path = str(tmp_path / "incremental.db") + _make_db_with_runs(db_path, n_runs=2) + + assert set(get_db_overview(db_path).keys()) == {1, 2} + assert get_db_overview(db_path, start_run_id=2) == {} + + _make_db_with_runs(db_path, n_runs=1, experiment_name="test_exp2", sample_name="s2") + + incremental = get_db_overview(db_path, start_run_id=2) + assert set(incremental.keys()) == {3} + assert incremental[3]["experiment"] == "test_exp2" + + +def test_get_db_overview_extra_columns(tmp_path: Path) -> None: + db_path = str(tmp_path / "extra.db") + initialise_or_create_database_at(db_path) + load_or_create_experiment("exp", sample_name="sample") + p_x = ParamSpecBase("x", "numeric") + p_y = ParamSpecBase("y", "numeric") + interdeps = InterDependencies_(dependencies={p_y: (p_x,)}) + ds = new_data_set("tagged_run") + ds.set_interdependencies(interdeps) + ds.mark_started() + ds.add_results([{p_x.name: 1.0, p_y.name: 2.0}]) + ds.mark_completed() + ds.add_metadata("my_tag", "hello") + ds.conn.close() + + # An existing ad-hoc metadata column is returned ... + overview = get_db_overview(db_path, extra_columns=["my_tag"]) + assert overview[1]["my_tag"] == "hello" # type: ignore[typeddict-item] + + # ... while a non-existent column is silently skipped. + overview = get_db_overview(db_path, extra_columns=["does_not_exist"]) + assert "does_not_exist" not in overview[1] + + +def test_get_db_overview_accepts_connection(tmp_path: Path) -> None: + db_path = str(tmp_path / "conn.db") + _make_db_with_runs(db_path, n_runs=1) + + conn = connect(db_path) + try: + overview = get_db_overview(conn=conn) + assert set(overview.keys()) == {1} + # the externally supplied connection must remain usable + assert conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] == 1 + finally: + conn.close() + + +def test_get_db_overview_rejects_conn_and_path(tmp_path: Path) -> None: + db_path = str(tmp_path / "both.db") + _make_db_with_runs(db_path, n_runs=1) + + conn = connect(db_path) + try: + with pytest.raises(ValueError): + get_db_overview(path_to_db=db_path, conn=conn) + finally: + conn.close() From 811e10a44c255d743dd76f8feb370198edc6e7cb Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Fri, 3 Jul 2026 12:26:49 +0200 Subject: [PATCH 2/4] Add changelog newsfragment for get_db_overview Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/changes/newsfragments/8266.new | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/changes/newsfragments/8266.new diff --git a/docs/changes/newsfragments/8266.new b/docs/changes/newsfragments/8266.new new file mode 100644 index 000000000000..38442261c27a --- /dev/null +++ b/docs/changes/newsfragments/8266.new @@ -0,0 +1,7 @@ +Added :func:`qcodes.dataset.get_db_overview`, a fast way to list the runs in a +QCoDeS database. It fetches run metadata (experiment/sample names, timestamps, +record counts and guids) via a single ``JOIN`` query on the ``runs`` and +``experiments`` tables, without instantiating a ``DataSet`` object per run, +making it possible to list databases with many thousands of runs almost +instantly. The returned :class:`qcodes.dataset.RunOverviewDict` is also exported +on the public ``qcodes.dataset`` namespace. From 151e4eb0084f82fb1296ce311407393fae0a94ae Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Fri, 3 Jul 2026 14:02:16 +0200 Subject: [PATCH 3/4] Address review: kwarg-only, specific exceptions, drop result_counter fallback - Make conn/start_run_id/extra_columns keyword-only. - Remove the leftover schema-stability note from the module docstring. - Catch sqlite3.Error rather than bare Exception around the queries. - Document why the extra_columns dict update needs a type: ignore. - Drop the result_counter fallback for the record count: result_counter is the run's ordinal within its experiment, not a data-point count, so it produced a misleading number; report 0 (unknown) instead. - Add tests for the in-progress, missing-results-table and query-error branches. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/qcodes/dataset/sqlite/db_overview.py | 50 +++++++++-------- tests/dataset/test_db_overview.py | 69 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 23 deletions(-) diff --git a/src/qcodes/dataset/sqlite/db_overview.py b/src/qcodes/dataset/sqlite/db_overview.py index 977738f3f401..230ebd267705 100644 --- a/src/qcodes/dataset/sqlite/db_overview.py +++ b/src/qcodes/dataset/sqlite/db_overview.py @@ -9,9 +9,6 @@ enumeration and makes it possible to list the contents of databases with many thousands of runs almost instantly. It is primarily intended for tools that need to display a table of runs (e.g. dataset browsers). - -The queries rely on the stable QCoDeS database schema (the ``runs`` and -``experiments`` tables) which has been unchanged across many QCoDeS versions. """ from __future__ import annotations @@ -19,6 +16,7 @@ import datetime import json import logging +import sqlite3 from contextlib import closing, nullcontext from typing import TYPE_CHECKING @@ -118,6 +116,7 @@ def _records_from_run_description(run_description_json: str | None) -> int: def get_db_overview( path_to_db: str | Path | None = None, + *, conn: AtomicConnection | None = None, start_run_id: int = 0, extra_columns: Sequence[str] | None = None, @@ -140,8 +139,7 @@ def get_db_overview( * For runs that are still in progress the number of rows in the results table is preferred (it grows as data is added), falling back to the run description shapes. - * If neither is available the ``result_counter`` of the run is used, which - counts the number of insertions rather than data points. + * If neither is available the count is reported as ``0`` (unknown). Only one of ``path_to_db`` and ``conn`` should be supplied. If a ``path_to_db`` is given the database is opened in read-only mode and the @@ -184,7 +182,7 @@ def get_db_overview( query = f""" SELECT r.run_id, e.name, e.sample_name, r.name, r.run_timestamp, r.completed_timestamp, - r.result_counter, r.guid, r.result_table_name, + r.guid, r.result_table_name, r.run_description{extra_select} FROM runs r JOIN experiments e ON r.exp_id = e.exp_id @@ -194,41 +192,44 @@ def get_db_overview( try: rows = c.execute(query, (start_run_id,)).fetchall() - except Exception as e: + except sqlite3.Error as e: log.warning("Could not query database overview: %s", e) return overview - # ``result_counter`` in the runs table counts INSERT calls, not data - # points. For the ``array`` paramtype a single INSERT can contain many - # data points, so the real row count of the results table is queried - # separately. - result_tables = {row[8] for row in rows if row[8]} + # ``result_counter`` in the runs table is the run's ordinal within its + # experiment, not a data-point count, so it is not usable here. For the + # ``array`` paramtype a single INSERT can also contain many data points. + # The real number of data points is therefore the row count of the + # results table, queried separately. + result_tables = {row[7] for row in rows if row[7]} row_counts: dict[str, int] = {} for table in result_tables: try: - count = c.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone() - except Exception: - continue # results table may not exist yet - row_counts[table] = count[0] if count else 0 + (count,) = c.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone() + except sqlite3.Error: + continue # results table may not exist (yet) + row_counts[table] = count - n_fixed = 10 # number of columns selected before ``extra_columns`` + n_fixed = 9 # number of columns selected before ``extra_columns`` for row in rows: run_id = row[0] started_date, started_time = _format_timestamp(row[4]) completed_date, completed_time = _format_timestamp(row[5]) - result_table = row[8] or "" + result_table = row[7] or "" is_completed = row[5] is not None and row[5] != 0 + # The record count is a best-effort data-point count. For completed + # runs the run-description shapes are the authoritative final count; + # for in-progress runs the live results-table row count is preferred + # as it grows while data is added. ``0`` means "unknown". if is_completed: - records = _records_from_run_description(row[9]) + records = _records_from_run_description(row[8]) if records == 0: records = row_counts.get(result_table, 0) else: records = row_counts.get(result_table, 0) if records == 0: - records = _records_from_run_description(row[9]) - if records == 0: - records = row[6] or 0 + records = _records_from_run_description(row[8]) entry: RunOverviewDict = { "run_id": run_id, @@ -240,12 +241,15 @@ def get_db_overview( "completed_date": completed_date, "completed_time": completed_time, "records": records, - "guid": row[7] or "", + "guid": row[6] or "", } if valid_extra_columns: extra = { col: row[n_fixed + i] for i, col in enumerate(valid_extra_columns) } + # The keys of ``extra`` are only known at runtime (they are the + # user-supplied ``extra_columns``), so they cannot be part of + # the closed ``RunOverviewDict`` definition. entry.update(extra) # type: ignore[typeddict-item] overview[run_id] = entry diff --git a/tests/dataset/test_db_overview.py b/tests/dataset/test_db_overview.py index 20fa2a07120d..3b7db3e20c46 100644 --- a/tests/dataset/test_db_overview.py +++ b/tests/dataset/test_db_overview.py @@ -67,6 +67,8 @@ def test_records_from_run_description() -> None: def test_format_timestamp() -> None: assert _format_timestamp(None) == ("", "") assert _format_timestamp(0) == ("", "") + # a value that cannot be converted to a datetime is handled gracefully + assert _format_timestamp(float("nan")) == ("", "") date, time = _format_timestamp(1_600_000_000.0) assert len(date) == len("YYYY-MM-DD") @@ -172,3 +174,70 @@ def test_get_db_overview_rejects_conn_and_path(tmp_path: Path) -> None: get_db_overview(path_to_db=db_path, conn=conn) finally: conn.close() + + +def test_get_db_overview_in_progress_uses_live_row_count(tmp_path: Path) -> None: + db_path = str(tmp_path / "in_progress.db") + initialise_or_create_database_at(db_path) + load_or_create_experiment("exp", sample_name="sample") + p_x = ParamSpecBase("x", "numeric") + p_y = ParamSpecBase("y", "numeric") + interdeps = InterDependencies_(dependencies={p_y: (p_x,)}) + + # run 1: started, still in progress, with 5 data points + ds_live = new_data_set("live") + ds_live.set_interdependencies(interdeps) + ds_live.mark_started() + for i in range(5): + ds_live.add_results([{p_x.name: float(i), p_y.name: float(i)}]) + # run 2: started, in progress, no data yet + ds_empty = new_data_set("empty") + ds_empty.set_interdependencies(interdeps) + ds_empty.mark_started() + ds_empty.conn.close() # shared connection for both datasets + + overview = get_db_overview(db_path) + + # in-progress runs report the live results-table row count and have no + # completed timestamp + assert overview[1]["records"] == 5 + assert overview[1]["completed_date"] == "" + assert overview[1]["completed_time"] == "" + assert overview[2]["records"] == 0 + + +def test_get_db_overview_missing_results_table_reports_zero( + tmp_path: Path, +) -> None: + db_path = str(tmp_path / "no_table.db") + _make_db_with_runs(db_path, n_runs=1, n_points=5) + + conn = connect(db_path) + try: + (table_name,) = conn.execute( + "SELECT result_table_name FROM runs WHERE run_id=1" + ).fetchone() + conn.execute(f'DROP TABLE "{table_name}"') + conn.commit() + finally: + conn.close() + + # with the results table gone (and no shape info) the count is unknown; + # ``result_counter`` is the run ordinal, not a data-point count, so it must + # not be used as a fallback + overview = get_db_overview(db_path) + assert overview[1]["records"] == 0 + + +def test_get_db_overview_query_error_returns_empty(tmp_path: Path) -> None: + db_path = str(tmp_path / "broken.db") + _make_db_with_runs(db_path, n_runs=1) + + conn = connect(db_path) + try: + # remove a table the overview query depends on so the JOIN fails + conn.execute("DROP TABLE experiments") + conn.commit() + assert get_db_overview(conn=conn) == {} + finally: + conn.close() From a4b78852b1fd300df75d7d56b3b735b6a79ee4dd Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Fri, 3 Jul 2026 15:38:36 +0200 Subject: [PATCH 4/4] Address review: document snapshot/count caveats, isolate tests from config - Document in the docstring and newsfragment that snapshots are not read (they can be large and slow) and that the record count may be less precise than DataSet.number_of_results. - Rewrite the tests to build the database via an explicit connection object (load_or_create_experiment(conn=...)/new_data_set(conn=...)) so the global QCoDeS config is not mutated, and close the connection via request.addfinalizer so it is cleaned up even if a test fails. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/changes/newsfragments/8266.new | 7 +- src/qcodes/dataset/sqlite/db_overview.py | 7 +- tests/dataset/test_db_overview.py | 174 +++++++++++------------ 3 files changed, 91 insertions(+), 97 deletions(-) diff --git a/docs/changes/newsfragments/8266.new b/docs/changes/newsfragments/8266.new index 38442261c27a..e86d8b829934 100644 --- a/docs/changes/newsfragments/8266.new +++ b/docs/changes/newsfragments/8266.new @@ -4,4 +4,9 @@ record counts and guids) via a single ``JOIN`` query on the ``runs`` and ``experiments`` tables, without instantiating a ``DataSet`` object per run, making it possible to list databases with many thousands of runs almost instantly. The returned :class:`qcodes.dataset.RunOverviewDict` is also exported -on the public ``qcodes.dataset`` namespace. +on the public ``qcodes.dataset`` namespace. The function does not return +snapshots as they can be large and slow down building the database overview. +The number of results in each dataset is taken from shape information when +available, and otherwise falls back to a best-effort count of the rows in the +so-called results table that exists for every dataset, which may NOT be as +precise as ``DataSet.number_of_results``. diff --git a/src/qcodes/dataset/sqlite/db_overview.py b/src/qcodes/dataset/sqlite/db_overview.py index 230ebd267705..08150f018224 100644 --- a/src/qcodes/dataset/sqlite/db_overview.py +++ b/src/qcodes/dataset/sqlite/db_overview.py @@ -128,10 +128,13 @@ def get_db_overview( tables to fetch run metadata, avoiding the much more expensive ``experiments()`` + ``data_sets()`` enumeration that instantiates a ``DataSet`` object per run. It is therefore well suited for listing the - contents of databases with many runs. + contents of databases with many runs. The (potentially large) snapshot of + each run is deliberately not read, as it would slow down building the + overview significantly. The reported number of ``records`` is a best-effort estimate of the number - of data points in a run: + of data points in a run and may be less precise than + ``DataSet.number_of_results``: * For completed runs the shape information stored in the run description is preferred (it is the authoritative final count), falling back to the diff --git a/tests/dataset/test_db_overview.py b/tests/dataset/test_db_overview.py index 3b7db3e20c46..5e4fdaf63bf2 100644 --- a/tests/dataset/test_db_overview.py +++ b/tests/dataset/test_db_overview.py @@ -3,18 +3,17 @@ from __future__ import annotations import json -import sqlite3 from typing import TYPE_CHECKING import pytest from qcodes.dataset import ( get_db_overview, - initialise_or_create_database_at, load_or_create_experiment, new_data_set, ) from qcodes.dataset.descriptions.dependencies import InterDependencies_ +from qcodes.dataset.sqlite.connection import path_to_dbfile from qcodes.dataset.sqlite.database import connect from qcodes.dataset.sqlite.db_overview import ( _format_timestamp, @@ -25,29 +24,44 @@ if TYPE_CHECKING: from pathlib import Path + from qcodes.dataset.sqlite.connection import AtomicConnection -def _make_db_with_runs( - db_path: str, + +@pytest.fixture(name="db_conn") +def _db_conn(tmp_path: Path, request: pytest.FixtureRequest) -> AtomicConnection: + """Connection to a fresh temporary database. + + An explicit connection object is used rather than + ``initialise_or_create_database_at`` so that the global QCoDeS config is + left untouched. The connection is closed through a finalizer so that it is + cleaned up even if the test fails. + """ + conn = connect(str(tmp_path / "overview.db")) + request.addfinalizer(conn.close) + return conn + + +def _create_runs( + conn: AtomicConnection, + *, n_runs: int = 1, n_points: int = 10, experiment_name: str = "test_exp", sample_name: str = "test_sample", ) -> None: - """Create a database at ``db_path`` with ``n_runs`` simple numeric runs.""" - initialise_or_create_database_at(db_path) - load_or_create_experiment(experiment_name, sample_name=sample_name) + """Create ``n_runs`` simple numeric runs on ``conn``.""" + exp = load_or_create_experiment(experiment_name, sample_name=sample_name, conn=conn) p_x = ParamSpecBase("x", "numeric") p_y = ParamSpecBase("y", "numeric") interdeps = InterDependencies_(dependencies={p_y: (p_x,)}) for r in range(n_runs): - ds = new_data_set(f"run_{r + 1}") + ds = new_data_set(f"run_{r + 1}", exp_id=exp.exp_id, conn=conn) ds.set_interdependencies(interdeps) ds.mark_started() for i in range(n_points): ds.add_results([{p_x.name: float(i), p_y.name: float(i**2)}]) ds.mark_completed() - ds.conn.close() def test_records_from_run_description() -> None: @@ -77,11 +91,10 @@ def test_format_timestamp() -> None: assert time.count(":") == 2 -def test_get_db_overview_basic_fields(tmp_path: Path) -> None: - db_path = str(tmp_path / "basic.db") - _make_db_with_runs(db_path, n_runs=2) +def test_get_db_overview_basic_fields(db_conn: AtomicConnection) -> None: + _create_runs(db_conn, n_runs=2) - overview = get_db_overview(db_path) + overview = get_db_overview(path_to_dbfile(db_conn)) assert set(overview.keys()) == {1, 2} for run_id, info in overview.items(): @@ -94,109 +107,90 @@ def test_get_db_overview_basic_fields(tmp_path: Path) -> None: assert info["completed_date"] and info["completed_time"] -def test_get_db_overview_counts_result_rows(tmp_path: Path) -> None: - db_path = str(tmp_path / "counts.db") - _make_db_with_runs(db_path, n_runs=3, n_points=10) +def test_get_db_overview_counts_result_rows(db_conn: AtomicConnection) -> None: + _create_runs(db_conn, n_runs=3, n_points=10) - overview = get_db_overview(db_path) + overview = get_db_overview(conn=db_conn) - conn = sqlite3.connect(db_path) - try: - for run_id, info in overview.items(): - (table_name,) = conn.execute( - "SELECT result_table_name FROM runs WHERE run_id=?", (run_id,) - ).fetchone() - (actual,) = conn.execute(f'SELECT COUNT(*) FROM "{table_name}"').fetchone() - assert info["records"] == actual == 10 - finally: - conn.close() + for run_id, info in overview.items(): + (table_name,) = db_conn.execute( + "SELECT result_table_name FROM runs WHERE run_id=?", (run_id,) + ).fetchone() + (actual,) = db_conn.execute(f'SELECT COUNT(*) FROM "{table_name}"').fetchone() + assert info["records"] == actual == 10 -def test_get_db_overview_incremental(tmp_path: Path) -> None: - db_path = str(tmp_path / "incremental.db") - _make_db_with_runs(db_path, n_runs=2) +def test_get_db_overview_incremental(db_conn: AtomicConnection) -> None: + _create_runs(db_conn, n_runs=2) + path = path_to_dbfile(db_conn) - assert set(get_db_overview(db_path).keys()) == {1, 2} - assert get_db_overview(db_path, start_run_id=2) == {} + assert set(get_db_overview(path).keys()) == {1, 2} + assert get_db_overview(path, start_run_id=2) == {} - _make_db_with_runs(db_path, n_runs=1, experiment_name="test_exp2", sample_name="s2") + _create_runs(db_conn, n_runs=1, experiment_name="test_exp2", sample_name="s2") - incremental = get_db_overview(db_path, start_run_id=2) + incremental = get_db_overview(path, start_run_id=2) assert set(incremental.keys()) == {3} assert incremental[3]["experiment"] == "test_exp2" -def test_get_db_overview_extra_columns(tmp_path: Path) -> None: - db_path = str(tmp_path / "extra.db") - initialise_or_create_database_at(db_path) - load_or_create_experiment("exp", sample_name="sample") +def test_get_db_overview_extra_columns(db_conn: AtomicConnection) -> None: + exp = load_or_create_experiment("exp", sample_name="sample", conn=db_conn) p_x = ParamSpecBase("x", "numeric") p_y = ParamSpecBase("y", "numeric") interdeps = InterDependencies_(dependencies={p_y: (p_x,)}) - ds = new_data_set("tagged_run") + ds = new_data_set("tagged_run", exp_id=exp.exp_id, conn=db_conn) ds.set_interdependencies(interdeps) ds.mark_started() ds.add_results([{p_x.name: 1.0, p_y.name: 2.0}]) ds.mark_completed() ds.add_metadata("my_tag", "hello") - ds.conn.close() # An existing ad-hoc metadata column is returned ... - overview = get_db_overview(db_path, extra_columns=["my_tag"]) + overview = get_db_overview(conn=db_conn, extra_columns=["my_tag"]) assert overview[1]["my_tag"] == "hello" # type: ignore[typeddict-item] # ... while a non-existent column is silently skipped. - overview = get_db_overview(db_path, extra_columns=["does_not_exist"]) + overview = get_db_overview(conn=db_conn, extra_columns=["does_not_exist"]) assert "does_not_exist" not in overview[1] -def test_get_db_overview_accepts_connection(tmp_path: Path) -> None: - db_path = str(tmp_path / "conn.db") - _make_db_with_runs(db_path, n_runs=1) +def test_get_db_overview_accepts_connection(db_conn: AtomicConnection) -> None: + _create_runs(db_conn, n_runs=1) - conn = connect(db_path) - try: - overview = get_db_overview(conn=conn) - assert set(overview.keys()) == {1} - # the externally supplied connection must remain usable - assert conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] == 1 - finally: - conn.close() + overview = get_db_overview(conn=db_conn) + assert set(overview.keys()) == {1} + # the externally supplied connection must remain usable + assert db_conn.execute("SELECT COUNT(*) FROM runs").fetchone()[0] == 1 -def test_get_db_overview_rejects_conn_and_path(tmp_path: Path) -> None: - db_path = str(tmp_path / "both.db") - _make_db_with_runs(db_path, n_runs=1) +def test_get_db_overview_rejects_conn_and_path(db_conn: AtomicConnection) -> None: + _create_runs(db_conn, n_runs=1) - conn = connect(db_path) - try: - with pytest.raises(ValueError): - get_db_overview(path_to_db=db_path, conn=conn) - finally: - conn.close() + with pytest.raises(ValueError): + get_db_overview(path_to_db=path_to_dbfile(db_conn), conn=db_conn) -def test_get_db_overview_in_progress_uses_live_row_count(tmp_path: Path) -> None: - db_path = str(tmp_path / "in_progress.db") - initialise_or_create_database_at(db_path) - load_or_create_experiment("exp", sample_name="sample") +def test_get_db_overview_in_progress_uses_live_row_count( + db_conn: AtomicConnection, +) -> None: + exp = load_or_create_experiment("exp", sample_name="sample", conn=db_conn) p_x = ParamSpecBase("x", "numeric") p_y = ParamSpecBase("y", "numeric") interdeps = InterDependencies_(dependencies={p_y: (p_x,)}) # run 1: started, still in progress, with 5 data points - ds_live = new_data_set("live") + ds_live = new_data_set("live", exp_id=exp.exp_id, conn=db_conn) ds_live.set_interdependencies(interdeps) ds_live.mark_started() for i in range(5): ds_live.add_results([{p_x.name: float(i), p_y.name: float(i)}]) # run 2: started, in progress, no data yet - ds_empty = new_data_set("empty") + ds_empty = new_data_set("empty", exp_id=exp.exp_id, conn=db_conn) ds_empty.set_interdependencies(interdeps) ds_empty.mark_started() - ds_empty.conn.close() # shared connection for both datasets - overview = get_db_overview(db_path) + overview = get_db_overview(conn=db_conn) # in-progress runs report the live results-table row count and have no # completed timestamp @@ -207,37 +201,29 @@ def test_get_db_overview_in_progress_uses_live_row_count(tmp_path: Path) -> None def test_get_db_overview_missing_results_table_reports_zero( - tmp_path: Path, + db_conn: AtomicConnection, ) -> None: - db_path = str(tmp_path / "no_table.db") - _make_db_with_runs(db_path, n_runs=1, n_points=5) + _create_runs(db_conn, n_runs=1, n_points=5) - conn = connect(db_path) - try: - (table_name,) = conn.execute( - "SELECT result_table_name FROM runs WHERE run_id=1" - ).fetchone() - conn.execute(f'DROP TABLE "{table_name}"') - conn.commit() - finally: - conn.close() + (table_name,) = db_conn.execute( + "SELECT result_table_name FROM runs WHERE run_id=1" + ).fetchone() + db_conn.execute(f'DROP TABLE "{table_name}"') + db_conn.commit() # with the results table gone (and no shape info) the count is unknown; # ``result_counter`` is the run ordinal, not a data-point count, so it must # not be used as a fallback - overview = get_db_overview(db_path) + overview = get_db_overview(conn=db_conn) assert overview[1]["records"] == 0 -def test_get_db_overview_query_error_returns_empty(tmp_path: Path) -> None: - db_path = str(tmp_path / "broken.db") - _make_db_with_runs(db_path, n_runs=1) +def test_get_db_overview_query_error_returns_empty( + db_conn: AtomicConnection, +) -> None: + _create_runs(db_conn, n_runs=1) - conn = connect(db_path) - try: - # remove a table the overview query depends on so the JOIN fails - conn.execute("DROP TABLE experiments") - conn.commit() - assert get_db_overview(conn=conn) == {} - finally: - conn.close() + # remove a table the overview query depends on so the JOIN fails + db_conn.execute("DROP TABLE experiments") + db_conn.commit() + assert get_db_overview(conn=db_conn) == {}