Skip to content

Commit 05823bf

Browse files
jpuerto-pscjpuerto-psc
authored
Merge pull request #137 from hubmapconsortium/thomcsmits/rewrite-to-ipynb
Use jupyter notebook as template source directly
2 parents f223fcb + 0e10479 commit 05823bf

36 files changed

Lines changed: 4193 additions & 4030 deletions

File tree

‎BUILD‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
6b1b36d
1+
f3faf77

‎CONTRIBUTING.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,13 @@ We use a Docker container. Please see [the docker documentation](https://hub.doc
1616

1717

1818
## Linting
19-
We use Flake8 and Black for linting Python code. If you have installed the packages in requirements-dev.txt, you can simply run the following:
19+
We use Flake8, Black and isort for linting Python code. If you have installed the packages in requirements-dev.txt, you can simply run the following:
2020
```sh
2121
flake8 path-to-file
2222
```
2323
```sh
2424
black path-to-file
25+
```
26+
```sh
27+
isort path-to-file
2528
```

‎src/user_templates_api/templates/jupyter_lab/render.py‎

Lines changed: 10 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -4,75 +4,28 @@
44

55
from django.template import engines
66

7+
from user_templates_api.templates.jupyter_lab.utils.convert_templates.convert_notebook import (
8+
conversion,
9+
)
10+
711
# from nbformat.v4 import new_code_cell, new_markdown_cell
812
# import user_templates_api.templates.jupyter_lab.utils.utils as jl_utils
913

1014

1115
class JupyterLabRender:
1216
def render(self, data):
13-
# We should append the cells
14-
# TODO: Check for the template format in metadata.json
15-
# and use the appropriate method.
16-
1717
metadata = data["metadata"]
1818
data["uuids"] = data.get("uuids", [])
1919

20-
if metadata["template_format"] == "python":
21-
cells = self.python_generate_template_data(data)
22-
elif metadata["template_format"] == "json":
23-
cells = self.json_generate_template_data(data)
24-
elif metadata["template_format"] == "jinja":
25-
cells = self.jinja_generate_template_data(data)
20+
if metadata["template_format"] != "jinja":
21+
return
22+
23+
cells = self.jinja_generate_template_data(data)
2624

2725
nb = {"cells": cells, "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
2826

2927
return json.dumps(nb)
3028

31-
def python_generate_template_data(self, data):
32-
return []
33-
34-
# TODO: see if this can be used without get_client
35-
# def json_generate_template_data(self, data):
36-
# django_engine = engines["django"]
37-
38-
# uuids = data["uuids"]
39-
# group_token = data["group_token"]
40-
41-
# util_client = get_client(group_token)
42-
43-
# template_json = data.get("template")
44-
# if template_json is None:
45-
# class_file_path = inspect.getfile(self.__class__)
46-
# # Convert the string to a pathlib Path
47-
# class_file_path = Path(class_file_path)
48-
# # Grab the parent path and append template.json
49-
# template_file_path = class_file_path.parent / "template.json"
50-
# # Load that filepath since it should be the json template
51-
# template_json = json.load(open(template_file_path))
52-
53-
# cells = []
54-
# for template_item in template_json:
55-
# cell_type = template_item["cell_type"]
56-
# src = template_item["src"]
57-
# if cell_type == "template_cell":
58-
# if src == "get_metadata_cells":
59-
# cells += jl_utils.get_metadata_cells(uuids, util_client)
60-
# elif src == "get_file_cells":
61-
# cells += jl_utils.get_file_cells(uuids, util_client)
62-
# elif src == "get_anndata_cells":
63-
# cells += jl_utils.get_anndata_cells(uuids, util_client)
64-
# elif cell_type == "code_cell":
65-
# template = django_engine.from_string(src)
66-
# # Need to do something w/ the source to make sure that it has its vars replaced
67-
# # Have to use append here since the nbformat cell object has an add/iadd function that is really a merge
68-
# cells.append(new_code_cell(template.render(data)))
69-
# elif cell_type == "markdown_cell":
70-
# template = django_engine.from_string(src)
71-
# # Need to do something w/ the source to make sure that it has its vars replaced
72-
# cells.append(new_markdown_cell(template.render(data)))
73-
74-
# return cells
75-
7629
def jinja_generate_template_data(self, data):
7730
django_engine = engines["django"]
7831

@@ -81,10 +34,10 @@ def jinja_generate_template_data(self, data):
8134
# Convert the string to a pathlib Path
8235
class_file_path = Path(class_file_path)
8336
# Grab the parent path and append template.txt
84-
template_file_path = class_file_path.parent / "template.txt"
37+
template_file_path = class_file_path.parent / "template.ipynb"
8538
# Load that filepath since it should be the json template
8639
template_file = open(template_file_path)
87-
template = django_engine.from_string(template_file.read())
40+
template = django_engine.from_string(conversion(template_file.read()))
8841
rendered_template = template.render(data).strip()
8942
rendered_template = json.loads(rendered_template) if rendered_template else {}
9043

‎src/user_templates_api/templates/jupyter_lab/templates/api_tutorial/template.ipynb‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"This notebook demonstrates how to work with HuBMAP APIs for datasets:"
8+
]
9+
},
10+
{
11+
"cell_type": "code",
12+
"execution_count": null,
13+
"metadata": {},
14+
"outputs": [],
15+
"source": [
16+
"!pip install requests pandas"
17+
]
18+
},
19+
{
20+
"cell_type": "code",
21+
"execution_count": null,
22+
"metadata": {},
23+
"outputs": [],
24+
"source": [
25+
"from csv import DictReader, excel_tab\n",
26+
"from io import StringIO\n",
27+
"import requests\n",
28+
"import pandas as pd\n",
29+
"\n",
30+
"# These are the UUIDS of the search results when this notebook was created:\n",
31+
"\n",
32+
"uuids = {{ uuids | safe }}"
33+
]
34+
},
35+
{
36+
"cell_type": "markdown",
37+
"metadata": {},
38+
"source": [
39+
"## Metadata"
40+
]
41+
},
42+
{
43+
"cell_type": "code",
44+
"execution_count": null,
45+
"metadata": {},
46+
"outputs": [],
47+
"source": [
48+
"# Fetch the metadata, and read it into a list of dicts:\n",
49+
"\n",
50+
"response = requests.post(\n",
51+
" \"https://portal.hubmapconsortium.org/metadata/v0/datasets.tsv\", json={\"uuids\": uuids}\n",
52+
")\n",
53+
"metadata = list(DictReader(StringIO(response.text), dialect=excel_tab))"
54+
]
55+
},
56+
{
57+
"cell_type": "code",
58+
"execution_count": null,
59+
"metadata": {},
60+
"outputs": [],
61+
"source": [
62+
"# The number of metadata dicts will correspond to the number of UUIDs requested:\n",
63+
"\n",
64+
"len(metadata)"
65+
]
66+
},
67+
{
68+
"cell_type": "code",
69+
"execution_count": null,
70+
"metadata": {},
71+
"outputs": [],
72+
"source": [
73+
"# Load it into a DataFrame and see the field definitions:\n",
74+
"\n",
75+
"pd.DataFrame(metadata[:1]).T.head()"
76+
]
77+
},
78+
{
79+
"cell_type": "code",
80+
"execution_count": null,
81+
"metadata": {},
82+
"outputs": [],
83+
"source": [
84+
"# Or review the data itself:\n",
85+
"\n",
86+
"pd.DataFrame(metadata[1:]).head()"
87+
]
88+
},
89+
{
90+
"cell_type": "markdown",
91+
"metadata": {},
92+
"source": [
93+
"## Files"
94+
]
95+
},
96+
{
97+
"cell_type": "code",
98+
"execution_count": null,
99+
"metadata": {},
100+
"outputs": [],
101+
"source": [
102+
"# The Search API can give us information about the files in processed datasets:\n",
103+
"\n",
104+
"search_api = \"https://search.api.hubmapconsortium.org/v3/portal/search\""
105+
]
106+
},
107+
{
108+
"cell_type": "code",
109+
"execution_count": null,
110+
"metadata": {},
111+
"outputs": [],
112+
"source": [
113+
"# The Search API supports Elasticsearch queries:\n",
114+
"\n",
115+
"import json\n",
116+
"\n",
117+
"hits = json.loads(\n",
118+
" requests.post(\n",
119+
" search_api,\n",
120+
" json={\n",
121+
" \"size\": 10000, # To make sure the list is not truncted, set this high.\n",
122+
" \"query\": {\"ids\": {\"values\": uuids}},\n",
123+
" \"_source\": [\"files\"],\n",
124+
" }, # Documents are large, so only request the fields we need.\n",
125+
" ).text\n",
126+
")[\"hits\"][\"hits\"]"
127+
]
128+
},
129+
{
130+
"cell_type": "code",
131+
"execution_count": null,
132+
"metadata": {},
133+
"outputs": [],
134+
"source": [
135+
"# Only processed datasets have file information.\n",
136+
"\n",
137+
"files = {\n",
138+
" hit[\"_id\"]: {\n",
139+
" file[\"rel_path\"]: file[\"description\"]\n",
140+
" for file in hit[\"_source\"].get(\"files\", [])\n",
141+
" if file\n",
142+
" }\n",
143+
" for hit in hits\n",
144+
"}\n",
145+
"\n",
146+
"pd.DataFrame(files).head()"
147+
]
148+
}
149+
],
150+
"metadata": {
151+
"language_info": {
152+
"name": "python"
153+
}
154+
},
155+
"nbformat": 4,
156+
"nbformat_minor": 2
157+
}

0 commit comments

Comments
 (0)