Skip to content

Commit 84991f1

Browse files
cdc-ap66cdc-ap66
authored
reporting_testing_and_documentation (#19)
* adding basic reporting example notebook * reporting functionality tests * adding doctest * more documentation * version bump * fixing to run badge creation on merge * updated to make sure my documentation images got uploaded * Added ability to create custom notebook metadata title/HTML title for report * fixed typos, thanks Ryan!
1 parent 58122e7 commit 84991f1

11 files changed

Lines changed: 386 additions & 10 deletions

File tree

‎.github/workflows/run_qaqc.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ name: qaqc
22

33
on:
44
pull_request:
5+
types: [opened, synchronize, reopened, closed]
56
push:
67
branches: [main]
78

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
*.jpeg
2626
*.gif
2727
*.pdf
28-
*.png
28+
# *.png
2929

3030
# Compressed archives
3131
*.gz

‎cfa/dataops/reporting/catalog.py‎

Lines changed: 58 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33
from pprint import pprint
44
from tempfile import TemporaryDirectory
55
from types import SimpleNamespace
6-
from typing import Any
6+
from typing import Any, Optional
77

8+
import nbformat
89
import papermill as pm
910
from cfa_azure.blob_helpers import write_blob_stream
1011
from nbconvert import HTMLExporter
@@ -49,6 +50,26 @@ def remove_ws_and_nonalpha(s: str) -> str:
4950
current[ns_list[-1]] = rp_i
5051

5152

53+
def retitle_notebook(nb_loc: str, new_title: str) -> None:
54+
"""
55+
Retitle a notebook by replacing the title in the first cell.
56+
57+
Args:
58+
nb_loc (str): The path to the notebook file.
59+
new_title (str): The new title to set for the notebook.
60+
"""
61+
assert os.path.exists(nb_loc), f"Notebook {nb_loc} does not exist."
62+
assert nb_loc.endswith(".ipynb"), "Notebook must be a .ipynb file."
63+
64+
with open(nb_loc, "r", encoding="utf-8") as f:
65+
notebook = nbformat.read(f, as_version=4)
66+
67+
notebook.metadata["title"] = new_title
68+
69+
with open(nb_loc, "w", encoding="utf-8") as f:
70+
nbformat.write(notebook, f)
71+
72+
5273
def nb_to_html(nb_path: str) -> str:
5374
"""Convert the executed notebook to HTML format.
5475
@@ -86,7 +107,7 @@ def print_params(self):
86107
"""Pretty print the parameters of the notebook."""
87108
pprint(self.get_params())
88109

89-
def nb_to_html_str(self, **kwargs) -> str:
110+
def nb_to_html_str(self, nb_title: Optional[str] = None, **kwargs) -> str:
90111
"""Convert the notebook to HTML format and optionally save it to a file.
91112
92113
Args:
@@ -104,11 +125,15 @@ def nb_to_html_str(self, **kwargs) -> str:
104125
parameters=kwargs,
105126
progress_bar=True,
106127
)
128+
if nb_title:
129+
retitle_notebook(out_file_path, nb_title)
107130
html_content = nb_to_html(out_file_path)
108131

109132
return html_content
110133

111-
def nb_to_html_file(self, html_out_path: str, **kwargs) -> None:
134+
def nb_to_html_file(
135+
self, html_out_path: str, nb_title: Optional[str] = None, **kwargs
136+
) -> None:
112137
"""Convert the notebook to HTML format.
113138
114139
Args:
@@ -120,14 +145,19 @@ def nb_to_html_file(self, html_out_path: str, **kwargs) -> None:
120145
assert html_out_path.endswith(
121146
".html"
122147
), "Output path must end with .html"
123-
html_content = self.nb_to_html_str(**kwargs)
148+
html_content = self.nb_to_html_str(nb_title=nb_title, **kwargs)
124149
os.makedirs(os.path.dirname(html_out_path), exist_ok=True)
125-
with open(html_out_path, "w") as f:
150+
with open(html_out_path, "w", encoding="utf-8") as f:
126151
f.write(html_content)
127152
print(f"HTML report saved to {os.path.abspath(html_out_path)}")
128153

129154
def nb_to_html_blob(
130-
self, blob_account: str, blob_container: str, blob_path: str, **kwargs
155+
self,
156+
blob_account: str,
157+
blob_container: str,
158+
blob_path: str,
159+
nb_title: Optional[str] = None,
160+
**kwargs,
131161
) -> None:
132162
"""Convert the notebook to HTML format save to blob storage.
133163
@@ -138,7 +168,7 @@ def nb_to_html_blob(
138168
None, saves the HTML content to a blob storage.
139169
"""
140170
assert blob_path.endswith(".html"), "Output path must end with .html"
141-
html_content = self.nb_to_html_str(**kwargs)
171+
html_content = self.nb_to_html_str(nb_title=nb_title, **kwargs)
142172
write_blob_stream(
143173
account_name=blob_account,
144174
container_name=blob_container,
@@ -179,5 +209,26 @@ def get_report_catalog() -> SimpleNamespace:
179209
180210
Returns:
181211
SimpleNamespace: The report catalog.
212+
213+
example:
214+
215+
>>> reportcat = get_report_catalog()
216+
>>> reportcat.examples.basics_ipynb.print_params()
217+
{'intercept': {'default': '0.5',
218+
'help': 'y-intercept of the line',
219+
'inferred_type_name': 'float',
220+
'name': 'intercept'},
221+
'slope': {'default': '1.2',
222+
'help': 'adding help text can be achieved with in-line comments',
223+
'inferred_type_name': 'float',
224+
'name': 'slope'},
225+
'step_size': {'default': '0.5',
226+
'help': 'step size for generating x values',
227+
'inferred_type_name': 'float',
228+
'name': 'step_size'},
229+
'x_range': {'default': '(-5, 5)',
230+
'help': 'range of x values to consider',
231+
'inferred_type_name': 'tuple',
232+
'name': 'x_range'}}
182233
"""
183234
return report_dict_to_sn(report_ns_map)

‎cfa/dataops/reporting/reports/examples/basics.ipynb‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"id": "4dfce678",
6+
"metadata": {},
7+
"source": [
8+
"## Basic Notebook Example\n",
9+
"\n",
10+
"The objective of this notebook is to demonstrate the basic necessities and features of notebook based report generation.\n",
11+
"\n",
12+
"The first thing to establish is that these notebooks ar parameterizable. This is done by creating a python cell, assigning some default values to variables, and tagging the cell with the tag `parameters`\n",
13+
"\n",
14+
"Example:"
15+
]
16+
},
17+
{
18+
"cell_type": "code",
19+
"execution_count": null,
20+
"id": "cc6ca370",
21+
"metadata": {
22+
"tags": [
23+
"parameters",
24+
"remove_input"
25+
]
26+
},
27+
"outputs": [],
28+
"source": [
29+
"# parameters\n",
30+
"\n",
31+
"slope: float = 1.2 # adding help text can be achieved with in-line comments\n",
32+
"intercept: float = 0.5 # y-intercept of the line\n",
33+
"x_range: tuple = (-5, 5) # range of x values to consider\n",
34+
"step_size: float = 0.5 # step size for generating x values"
35+
]
36+
},
37+
{
38+
"cell_type": "markdown",
39+
"id": "629f32d8",
40+
"metadata": {},
41+
"source": [
42+
"Now you can create cells that are used to import or compute reporting dependencies or for display. There are three other cell tags that will help with the formatting of your report:\n",
43+
"\n",
44+
"- `remove_input`: This cell tag removes the display of any code or markdown input. It is not likely that you will use this for markdown text, but very likely that you will use it on every python cell.\n",
45+
"- `remove_output`: This cell tag removes the output (including any items from memory that you may have wanted to use later).\n",
46+
"- `remove_cell`: This cell tag removes the entire cell input and output from the output report. Handy if you want instructions or examples for a report editor that you don't intend to be displayed in the final report.\n",
47+
"\n",
48+
"Next we will import a few common items that are often used in reporting, and remove the input so the import statements don't display in the report."
49+
]
50+
},
51+
{
52+
"cell_type": "code",
53+
"execution_count": null,
54+
"id": "ddef9fb8",
55+
"metadata": {
56+
"tags": [
57+
"remove_input"
58+
]
59+
},
60+
"outputs": [],
61+
"source": [
62+
"import itables # for pagination and searching tabular data\n",
63+
"import matplotlib.pyplot as plt # what's a report without a plot or two?\n",
64+
"import numpy as np # because we all love numpy\n",
65+
"import pandas as pd # for manipulating and displaying tabular data\n",
66+
"from IPython.display import ( # for displaying formatted text in python cells\n",
67+
" Markdown,\n",
68+
" display,\n",
69+
")"
70+
]
71+
},
72+
{
73+
"cell_type": "code",
74+
"execution_count": null,
75+
"id": "f971af94",
76+
"metadata": {
77+
"tags": [
78+
"remove_input"
79+
]
80+
},
81+
"outputs": [],
82+
"source": [
83+
"display(\n",
84+
" Markdown(\n",
85+
" f\"\"\"\n",
86+
"This is a parameterized markdown example. Here we display all the parameters, defaults and any that were passed by the notebook endpoint in the report catalog.:\n",
87+
"- Slope: {slope}\n",
88+
"- Intercept: {intercept}\n",
89+
"- X Range: {x_range}\n",
90+
"- Step Size: {step_size}\n",
91+
"\n",
92+
"Next we will compute some values and display them in a table.\n",
93+
"\"\"\"\n",
94+
" )\n",
95+
")"
96+
]
97+
},
98+
{
99+
"cell_type": "code",
100+
"execution_count": null,
101+
"id": "d845c8b4",
102+
"metadata": {
103+
"tags": [
104+
"remove_input"
105+
]
106+
},
107+
"outputs": [],
108+
"source": [
109+
"x = np.arange(x_range[0], x_range[1] + step_size, step_size)\n",
110+
"y = slope * x + intercept\n",
111+
"df = pd.DataFrame({\"x\": x, \"y\": y})\n",
112+
"\n",
113+
"# Display the DataFrame with itables for better interactivity\n",
114+
"itables.show(\n",
115+
" df.style.background_gradient(cmap=\"viridis\"),\n",
116+
" allow_html=True,\n",
117+
" lengthMenu=[5, 10, 21],\n",
118+
" pageLength=5,\n",
119+
")"
120+
]
121+
},
122+
{
123+
"cell_type": "markdown",
124+
"id": "0e5984e6",
125+
"metadata": {},
126+
"source": [
127+
"Finally, we can display a plot of the values computed using the input parameters."
128+
]
129+
},
130+
{
131+
"cell_type": "code",
132+
"execution_count": null,
133+
"id": "27c55442",
134+
"metadata": {
135+
"tags": [
136+
"remove_input"
137+
]
138+
},
139+
"outputs": [],
140+
"source": [
141+
"fig = plt.figure(figsize=(10, 6))\n",
142+
"plt.plot(x, y, \"c-.\")\n",
143+
"plt.plot(x, y, \"r.\")\n",
144+
"plt.title(\"Line Plot of y = mx + b\")\n",
145+
"plt.xlabel(\"x values\")\n",
146+
"plt.ylabel(\"y values\")\n",
147+
"plt.grid(True)\n",
148+
"plt.axhline(0, color=\"black\", linewidth=1, linestyle=\"--\")\n",
149+
"plt.axvline(0, color=\"black\", linewidth=1, linestyle=\"--\")\n",
150+
"plt.show()"
151+
]
152+
}
153+
],
154+
"metadata": {
155+
"kernelspec": {
156+
"display_name": "venv",
157+
"language": "python",
158+
"name": "python3"
159+
},
160+
"language_info": {
161+
"codemirror_mode": {
162+
"name": "ipython",
163+
"version": 3
164+
},
165+
"file_extension": ".py",
166+
"mimetype": "text/x-python",
167+
"name": "python",
168+
"nbconvert_exporter": "python",
169+
"pygments_lexer": "ipython3",
170+
"version": "3.10.12"
171+
}
172+
},
173+
"nbformat": 4,
174+
"nbformat_minor": 5
175+
}

‎changelog.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,17 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release})
88

99
---
1010

11+
## [2025.08.18.0a]
12+
13+
### Added
14+
15+
- Documentation for the reporting functionality
16+
- Tests for reporting functionality
17+
- Ability to create custom notebook metadata title/HTML title for report
18+
1119
## [2025.08.15.0a]
1220

13-
## Fixes
21+
### Fixes
1422

1523
- Typo in QAQC github workflow and testing operation
1624

‎docs/api.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,5 @@
33
::: cfa.dataops.catalog.get_data
44

55
::: cfa.dataops.catalog.list_datasets
6+
7+
::: cfa.dataops.reporting.catalog.get_report_catalog

‎docs/assets/annotated_report_example.png‎

218 KB
Loading

‎docs/index.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@
22

33
- [Data User Guide](data_user_guide.md)
44
- [Data Developer Guide](data_developer_guide.md)
5+
- [Report Generation](report_generation.md)
56
- [API reference](api.md)

0 commit comments

Comments
 (0)