RCDB great expectation integration

Author: DraTeots

.github/workflows/python-examples.yml

@@ -34,6 +34,7 @@ jobs:
           cd $GITHUB_WORKSPACE/python
           pip install --editable .
           pip install jsonpickle  # for writing object example
+          pip install pandas      # for the DataFrame examples (04, 05)
 
       #      - name: Download HallD SQLite snapshot
       #        run: |
@@ -53,6 +54,12 @@ jobs:
       - name: "Example: 03_select_values_custom_runs"
         run: python $GITHUB_WORKSPACE/python/examples/03_select_values_custom_runs.py
 
+      - name: "Example: 04_conditions_dataframe_run_period"
+        run: python $GITHUB_WORKSPACE/python/examples/04_conditions_dataframe_run_period.py
+
+      - name: "Example: 05_conditions_dataframe_last_runs"
+        run: python $GITHUB_WORKSPACE/python/examples/05_conditions_dataframe_last_runs.py
+
       - name: "Example: 10_create_conditions_basic"
         run: python $GITHUB_WORKSPACE/python/examples/10_create_conditions_basic.py
 

python/examples/04_conditions_dataframe_run_period.py

@@ -0,0 +1,30 @@
+#   E X A M P L E   4
+# Make a pandas DataFrame for a whole run period:
+#   rows    = runs of the period
+#   columns = all known condition types
+#   cells   = condition values (missing ones become NaN)
+
+import pandas as pd
+
+# import RCDB
+from rcdb.provider import RCDBProvider
+
+# connect to DB
+db = RCDBProvider("mysql://rcdb@hallddb.jlab.org/rcdb2")
+
+# pick a run period (here the latest one by run number)
+run_period = max(db.get_run_periods(), key=lambda rp: rp.run_max)
+print("Run period: {} [{}-{}]".format(run_period.name, run_period.run_min, run_period.run_max))
+
+# all known condition names -> these become the DataFrame columns
+condition_names = sorted(ct.name for ct in db.get_condition_types())
+
+# select those values for every run in the period
+table = db.select_values(condition_names, run_min=run_period.run_min, run_max=run_period.run_max)
+
+# build the DataFrame: conditions as columns, run number as the index
+df = pd.DataFrame(table.rows, columns=table.selected_conditions).set_index("run")
+
+#   P R I N T   O U T
+print(df)
+print("It took {:.2f} sec ".format(table.performance['total']))

python/examples/05_conditions_dataframe_last_runs.py

@@ -0,0 +1,32 @@
+#   E X A M P L E   5
+# Make a pandas DataFrame for a set of runs (here the last 10):
+#   rows    = the selected runs
+#   columns = all known condition types
+#   cells   = condition values (missing ones become NaN)
+
+import pandas as pd
+
+# import RCDB
+from rcdb.provider import RCDBProvider
+from rcdb.model import Run
+
+# connect to DB
+db = RCDBProvider("mysql://rcdb@hallddb.jlab.org/rcdb2")
+
+# the last 10 runs (highest run numbers)
+last_runs = [run.number for run in
+             db.session.query(Run).order_by(Run.number.desc()).limit(10).all()]
+print("Last {} runs: {}".format(len(last_runs), last_runs))
+
+# all known condition names -> these become the DataFrame columns
+condition_names = sorted(ct.name for ct in db.get_condition_types())
+
+# select those values for exactly those runs (pass them via runs=)
+table = db.select_values(condition_names, runs=last_runs)
+
+# build the DataFrame: conditions as columns, run number as the index
+df = pd.DataFrame(table.rows, columns=table.selected_conditions).set_index("run")
+
+#   P R I N T   O U T
+print(df)
+print("It took {:.2f} sec ".format(table.performance['total']))

python/pyproject.toml

@@ -47,7 +47,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-examples = ["jsonpickle"]
+examples = ["jsonpickle", "pandas"]
 dev = ["pyflakes>=3"]