Skip to content

Commit 9dcf333

Browse files
sschmidt23sschmidt23
committed
switch back to s3df to grab data file
1 parent df393f6 commit 9dcf333

2 files changed

Lines changed: 239 additions & 2 deletions

File tree

‎examples/rail_project_example.ipynb‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"id": "3f31a160-8ce9-4fe6-8805-9df7a97b7adc",
6+
"metadata": {},
7+
"source": [
8+
"## Using `RailProject` \n",
9+
"\n",
10+
"This notebook will show you the basics using the `RailProject` class to manage an analysis project"
11+
]
12+
},
13+
{
14+
"cell_type": "markdown",
15+
"id": "d436cc4a-49c9-4f27-93cf-9a196d251a77",
16+
"metadata": {},
17+
"source": [
18+
"### Setup and teardown scripts to setup a test area"
19+
]
20+
},
21+
{
22+
"cell_type": "code",
23+
"execution_count": null,
24+
"id": "96bee599-4711-4a07-b182-1cbe257ea4d8",
25+
"metadata": {},
26+
"outputs": [],
27+
"source": [
28+
"import os\n",
29+
"from rail.projects import library\n",
30+
"\n",
31+
"check_dir = os.path.basename(os.path.abspath(os.curdir))\n",
32+
"if check_dir == 'examples':\n",
33+
" os.chdir('..')\n",
34+
"\n",
35+
"setup = library.setup_project_area()\n",
36+
"assert setup == 0\n",
37+
"\n",
38+
"# use this to cleanup\n",
39+
"# library.teardown_project_area()"
40+
]
41+
},
42+
{
43+
"cell_type": "markdown",
44+
"id": "454c1464-aa61-4a0c-96b9-f48f5ddb94f6",
45+
"metadata": {},
46+
"source": [
47+
"### Load the test project"
48+
]
49+
},
50+
{
51+
"cell_type": "code",
52+
"execution_count": null,
53+
"id": "7136e23a-43f1-4a36-ba77-03b9a649c449",
54+
"metadata": {},
55+
"outputs": [],
56+
"source": [
57+
"from rail.projects import RailProject\n",
58+
"\n",
59+
"project = RailProject.load_config(\"tests/ci_project.yaml\")"
60+
]
61+
},
62+
{
63+
"cell_type": "markdown",
64+
"id": "57c75f4c-23f4-415b-8148-5b39754b5770",
65+
"metadata": {},
66+
"source": [
67+
"### Inspect the test project"
68+
]
69+
},
70+
{
71+
"cell_type": "code",
72+
"execution_count": null,
73+
"id": "f47f06c0-2718-4cdd-aaa9-156341336907",
74+
"metadata": {},
75+
"outputs": [],
76+
"source": [
77+
"catalog_files_truth = project.get_catalog_files(\"truth\")\n",
78+
"print(catalog_files_truth)"
79+
]
80+
},
81+
{
82+
"cell_type": "markdown",
83+
"id": "d37d65cf-6fbe-4fcd-818f-6eaa6a0aa88d",
84+
"metadata": {},
85+
"source": [
86+
"### Run a data reduction algorithm on the test project data\n",
87+
"\n",
88+
"This will use the \"roman_rubin\" reducer to apply the \"gold\" selection to the \"truth\" catalog to make a \"reduced\" catalog"
89+
]
90+
},
91+
{
92+
"cell_type": "code",
93+
"execution_count": null,
94+
"id": "1e4a8135-8131-473b-a132-b9be2109dbe0",
95+
"metadata": {},
96+
"outputs": [],
97+
"source": [
98+
"project.reduce_data(\n",
99+
" catalog_template=\"truth\",\n",
100+
" output_catalog_template=\"reduced\",\n",
101+
" reducer_class_name=\"roman_rubin\",\n",
102+
" input_selection=\"\",\n",
103+
" selection=\"gold\",\n",
104+
")\n"
105+
]
106+
},
107+
{
108+
"cell_type": "markdown",
109+
"id": "a70ff5d0-2e93-47e9-b5d4-7d791e883d9f",
110+
"metadata": {},
111+
"source": [
112+
"### Subsample the test project\n",
113+
"\n",
114+
"This will use the \"random_subsampler\" to apply the \"train_10\" subsample to the \"reduced\" catalog of the baseline flavor with the gold selection"
115+
]
116+
},
117+
{
118+
"cell_type": "code",
119+
"execution_count": null,
120+
"id": "e6b79133-b695-433c-a47f-91bfa9fffc97",
121+
"metadata": {},
122+
"outputs": [],
123+
"source": [
124+
"project.subsample_data(\n",
125+
" catalog_template=\"reduced\",\n",
126+
" file_template=\"train_file_10\",\n",
127+
" subsampler_class_name=\"random_subsampler\",\n",
128+
" subsample_name=\"train_10\",\n",
129+
" flavor=\"baseline\",\n",
130+
" selection=\"gold\",\n",
131+
")"
132+
]
133+
},
134+
{
135+
"cell_type": "markdown",
136+
"id": "45aede11-ea93-4dab-b7e2-e2d108079ac3",
137+
"metadata": {},
138+
"source": [
139+
"### Build all the pipelines for the \"baseline\" flavor"
140+
]
141+
},
142+
{
143+
"cell_type": "code",
144+
"execution_count": null,
145+
"id": "90ca72b7-bd09-4774-996b-8f4e6ca2bc6f",
146+
"metadata": {},
147+
"outputs": [],
148+
"source": [
149+
"project.build_pipelines(flavor=\"baseline\")"
150+
]
151+
},
152+
{
153+
"cell_type": "markdown",
154+
"id": "37354004-48a7-48eb-af8f-b87154be1c8a",
155+
"metadata": {},
156+
"source": [
157+
"### Generate the commands to run the \"pz\" pipeline in the \"baseline\" flavor for the \"gold\" selection"
158+
]
159+
},
160+
{
161+
"cell_type": "code",
162+
"execution_count": null,
163+
"id": "2dd52439-1f2e-4ca6-91c6-ed2b9ffb859d",
164+
"metadata": {},
165+
"outputs": [],
166+
"source": [
167+
"single_ceci_command = project.make_pipeline_single_input_command(\n",
168+
" pipeline_name=\"pz\",\n",
169+
" flavor=\"basline\",\n",
170+
" selection=\"gold\",\n",
171+
")\n",
172+
"print(single_ceci_command)"
173+
]
174+
},
175+
{
176+
"cell_type": "markdown",
177+
"id": "ff29b713-902e-41a5-94ce-f852eb5e461e",
178+
"metadata": {},
179+
"source": [
180+
"### Generate the commands to run the \"spec_selection\" pipeline in the \"baseline\" flavor for the \"gold\" selection"
181+
]
182+
},
183+
{
184+
"cell_type": "code",
185+
"execution_count": null,
186+
"id": "c6461441-b51e-42f0-bb86-9a5d3b0513c3",
187+
"metadata": {},
188+
"outputs": [],
189+
"source": [
190+
"ceci_catalog_commands = project.make_pipeline_catalog_commands(\n",
191+
" pipeline_name=\"spec_selection\",\n",
192+
" flavor=\"basline\",\n",
193+
" selection=\"gold\",\n",
194+
" spec_selections = list(project.get_spec_selections().keys()),\n",
195+
")\n",
196+
"print(ceci_catalog_commands)\n"
197+
]
198+
},
199+
{
200+
"cell_type": "code",
201+
"execution_count": null,
202+
"id": "2e8f3aee-ae84-4a99-9835-797350b94cbf",
203+
"metadata": {},
204+
"outputs": [],
205+
"source": []
206+
},
207+
{
208+
"cell_type": "code",
209+
"execution_count": null,
210+
"id": "13d8b9e3-7603-45d2-bc71-4b50c326c102",
211+
"metadata": {},
212+
"outputs": [],
213+
"source": []
214+
}
215+
],
216+
"metadata": {
217+
"kernelspec": {
218+
"display_name": "testenv",
219+
"language": "python",
220+
"name": "testenv"
221+
},
222+
"language_info": {
223+
"codemirror_mode": {
224+
"name": "ipython",
225+
"version": 3
226+
},
227+
"file_extension": ".py",
228+
"mimetype": "text/x-python",
229+
"name": "python",
230+
"nbconvert_exporter": "python",
231+
"pygments_lexer": "ipython3",
232+
"version": "3.12.12"
233+
}
234+
},
235+
"nbformat": 4,
236+
"nbformat_minor": 5
237+
}

‎src/rail/projects/library.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,8 @@ def setup_project_area() -> int: # pragma: no cover
236236

237237
if not os.path.exists("tests/ci_test.tgz"):
238238
urllib.request.urlretrieve(
239-
"https://portal.nersc.gov/cfs/lsst/PZ/test_data/ci_test.tgz",
240-
# "http://s3df.slac.stanford.edu/people/echarles/xfer/ci_test.tgz",
239+
# "https://portal.nersc.gov/cfs/lsst/PZ/test_data/ci_test.tgz",
240+
"http://s3df.slac.stanford.edu/people/echarles/xfer/ci_test.tgz",
241241
"tests/ci_test.tgz",
242242
)
243243
if not os.path.exists("tests/ci_test.tgz"):

0 commit comments

Comments
 (0)