Fixing Bugs found in 1.1.0 (#6)

Co-authored-by: MacFarland, Midgie <macfarlandmj@ornl.gov>

Author: midgemacf

CHANGELOG.md

@@ -37,6 +37,10 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 *Unreleased* versions radiate potential—-and dread. Once you merge an infernal PR, move its bullet under a new version heading with the actual release date.*
 
 -->
+## [1.1.1] - 2025-07-09
+### Fixed
+- 🩹 Added tabulate dependency to pyproject.toml
+- 🩹 Corrected logic for differentiating between universal probe metadata and vendor specifics.
 
 ## [1.1.0] - 2025-07-02
 ### Added

opensampl/load/table_factory.py

@@ -198,6 +198,7 @@ def write(self, data: dict[str, Any], if_exists: conflict_actions = "update"):
             new_entry = self.model(**data)
             self.session.add(new_entry)
             logger.debug(f"New entry created in {self.name}")
+            self.session.flush()
             return new_entry
 
         if if_exists == "error":

opensampl/load_data.py

@@ -9,7 +9,7 @@
 from sqlalchemy.orm import Session
 
 from opensampl.config.base import BaseConfig
-from opensampl.db.orm import Base, ProbeData, ProbeMetadata
+from opensampl.db.orm import Base, ProbeData
 from opensampl.load.routing import route
 from opensampl.load.table_factory import TableFactory
 from opensampl.metrics import MetricType
@@ -176,22 +176,13 @@ def load_probe_metadata(
         raise TypeError("Session must be a SQLAlchemy session")
 
     try:
-        probe = (
-            session.query(ProbeMetadata)
-            .filter(ProbeMetadata.probe_id == probe_key.probe_id, ProbeMetadata.ip_address == probe_key.ip_address)
-            .first()
-        )
-        logger.debug(f"{probe=}")
-        if not probe:
-            probe_info = {"probe_id": probe_key.probe_id, "ip_address": probe_key.ip_address, "vendor": vendor.name}
-            model = data.pop("model", None)
-            if model:
-                probe_info["model"] = str(model)
+        pm_factory = TableFactory(name="probe_metadata", session=session)
 
-            probe = ProbeMetadata(**probe_info)
+        pm_cols = {col.name for col in pm_factory.inspector.columns}
+        probe_info = {k: data.pop(k) for k in list(data.keys()) if k in pm_cols}
+        probe_info.update({"probe_id": probe_key.probe_id, "ip_address": probe_key.ip_address, "vendor": vendor.name})
+        probe = pm_factory.write(data=probe_info, if_exists="update")
 
-            session.add(probe)
-            session.flush()
         data["probe_uuid"] = probe.uuid
 
         write_to_table(table=vendor.metadata_table, data=data, session=session, if_exists="update")

opensampl/server/grafana/grafana-dashboards/twst_dash.json

@@ -627,7 +627,7 @@
           "format": "table",
           "hide": false,
           "rawQuery": true,
-          "rawSql": "select \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC') AS time,\n  pm.name,\n  AVG(pd.value::FLOAT) * 1e9 AS value\nFROM \n  get_probe_data_by_probe_and_metric(${modem:sqlstring}, ${phase_offset_uuid:sqlstring}) pd\njoin castdb.reference r on pd.reference_uuid = r.uuid\nJOIN \n  castdb.probe_metadata pm ON r.compound_reference_uuid = pm.uuid\nwhere \n  $__timeFilter(pd.time)\n  and r.compound_reference_uuid in (${compound_ref:sqlstring})\nGROUP BY \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC'),\n  pm.name\nORDER BY \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC')",
+          "rawSql": "select \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC') AS time,\n  COALESCE(pm.name, pm.probe_id) as name,\n  AVG(pd.value::FLOAT) * 1e9 AS value\nFROM \n  get_probe_data_by_probe_and_metric(${modem:sqlstring}, ${phase_offset_uuid:sqlstring}) pd\njoin castdb.reference r on pd.reference_uuid = r.uuid\nJOIN \n  castdb.probe_metadata pm ON r.compound_reference_uuid = pm.uuid\nwhere \n  $__timeFilter(pd.time)\n  and r.compound_reference_uuid in (${compound_ref:sqlstring})\nGROUP BY \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC'),\n  pm.uuid\nORDER BY \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC')",
           "refId": "B",
           "sql": {
             "columns": [
@@ -762,7 +762,7 @@
           "format": "table",
           "hide": false,
           "rawQuery": true,
-          "rawSql": "select \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC') AS time,\n  pm.name,\n  (MAX(pd.value::FLOAT) - MIN(pd.value::FLOAT)) * 1e9 AS value\nFROM \n  get_probe_data_by_probe_and_metric(${modem:sqlstring}, ${phase_offset_uuid:sqlstring}) pd\njoin castdb.reference r on pd.reference_uuid = r.uuid\nJOIN \n  castdb.probe_metadata pm ON r.compound_reference_uuid = pm.uuid\nwhere \n  $__timeFilter(pd.time)\n  and r.compound_reference_uuid in (${compound_ref:sqlstring})\nGROUP BY \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC'),\n  pm.name\nORDER BY \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC')",
+          "rawSql": "select \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC') AS time,\n  COALESCE(pm.name, pm.probe_id) as name,\n  (MAX(pd.value::FLOAT) - MIN(pd.value::FLOAT)) * 1e9 AS value\nFROM \n  get_probe_data_by_probe_and_metric(${modem:sqlstring}, ${phase_offset_uuid:sqlstring}) pd\njoin castdb.reference r on pd.reference_uuid = r.uuid\nJOIN \n  castdb.probe_metadata pm ON r.compound_reference_uuid = pm.uuid\nwhere \n  $__timeFilter(pd.time)\n  and r.compound_reference_uuid in (${compound_ref:sqlstring})\nGROUP BY \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC'),\n  pm.uuid\nORDER BY \n  time_bucket(${resolution:sqlstring}, pd.time AT TIME ZONE 'UTC')",
           "refId": "B",
           "sql": {
             "columns": [
@@ -1524,15 +1524,15 @@
           "type": "grafana-postgresql-datasource",
           "uid": "castdb-datasource"
         },
-        "definition": "SELECT uuid AS __value, COALESCE(name, CONCAT(ip_address, ' Interface ', probe_id)) AS __text FROM castdb.probe_metadata where vendor = 'MicrosemiTWST' and probe_id = 'modem'",
+        "definition": "SELECT uuid AS __value, COALESCE(name, CONCAT(ip_address, ' Interface ', probe_id)) AS __text FROM castdb.probe_metadata where vendor = 'MicrochipTWST' and probe_id = 'modem'",
         "description": "The modem from which \"local time\" is derived, also the stream from which we are receiving the data. ",
         "hide": 0,
         "includeAll": false,
         "label": "Receiver",
         "multi": false,
         "name": "modem",
         "options": [],
-        "query": "SELECT uuid AS __value, COALESCE(name, CONCAT(ip_address, ' Interface ', probe_id)) AS __text FROM castdb.probe_metadata where vendor = 'MicrosemiTWST' and probe_id = 'modem'",
+        "query": "SELECT uuid AS __value, COALESCE(name, CONCAT(ip_address, ' Interface ', probe_id)) AS __text FROM castdb.probe_metadata where vendor = 'MicrochipTWST' and probe_id = 'modem'",
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "opensampl"
-version = "1.1.0"
+version = "1.1.1"
 description = "Python tools for adding clock data to a timescale db."
 license = {file = "LICENSE"}
 authors = [
@@ -30,6 +30,7 @@ dependencies = [
     "astor",
     "libcst",
     "jinja2>=3.1.6",
+    "tabulate"
 ]
 
 [project.urls]

tests/test_load_data.py

@@ -286,11 +286,10 @@ def test_real_session_flow(
         count_before = mock_session.query(ProbeMetadata).count()
         assert count_before == 0
 
-        with patch("opensampl.load_data.ProbeMetadata", ProbeMetadata):
-            result = load_probe_metadata(
-                vendor=VENDORS.ADVA, probe_key=sample_probe_key, data=sample_adva_metadata, session=mock_session
-            )
-            assert result is None
+        result = load_probe_metadata(
+            vendor=VENDORS.ADVA, probe_key=sample_probe_key, data=sample_adva_metadata, session=mock_session
+        )
+        assert result is None
 
         metadata = mock_session.query(ProbeMetadata).all()
         assert len(metadata) == 1