Skip to content

Commit 3474c68

Browse files
asteikerasteiker
committed
compare outputs
1 parent 3a45df8 commit 3474c68

2 files changed

Lines changed: 230 additions & 25 deletions

File tree

‎notebooks/.ipynb_checkpoints/NSIDC DAAC ICESat-2 Customize and Access-checkpoint.ipynb‎

Lines changed: 113 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
"import geopandas as gpd\n",
5151
"import matplotlib.pyplot as plt\n",
5252
"import fiona\n",
53+
"import h5py\n",
5354
"# To read KML files with geopandas, we will need to enable KML support in fiona (disabled by default)\n",
5455
"fiona.drvsupport.supported_drivers['LIBKML'] = 'rw'\n",
5556
"from shapely.geometry import Polygon, mapping\n",
@@ -681,12 +682,66 @@
681682
"# Select variable subsetting\n",
682683
"\n",
683684
"if len(variable_vals) > 0:\n",
684-
" v = input('Variable subsetting is available. Would you like to subset a selection of variables? (y/n)')\n",
685-
" if v == 'y':\n",
686-
" print('The', short_name, 'variables to select from include:')\n",
687-
" pprint.pprint(variable_vals)\n",
688-
" coverage = input('If you would like to subset by variable, copy and paste the variables you would like separated by comma. Make sure to omit quotes but include all forward slashes: ')\n",
689-
" else: coverage = ''\n",
685+
" var = input('Variable subsetting is available. Would you like to subset by variables? (y/n)')\n",
686+
" if var == 'y':\n",
687+
" v = input('Would you like to view the list of variables? (y/n)')\n",
688+
" if v == 'y':\n",
689+
" print('The', short_name, 'variables to select from include:')\n",
690+
" pprint.pprint(variable_vals)\n",
691+
" # Enter variables to subset separated by comma. Make sure to omit quotes but include all forward slashes to indicate HDF group hierarchy\n",
692+
" coverage = '/ancillary_data/atlas_sdp_gps_epoch,\\\n",
693+
"/gt1l/land_ice_segments/atl06_quality_summary,\\\n",
694+
"/gt1l/land_ice_segments/delta_time,\\\n",
695+
"/gt1l/land_ice_segments/h_li,\\\n",
696+
"/gt1l/land_ice_segments/h_li_sigma,\\\n",
697+
"/gt1l/land_ice_segments/latitude,\\\n",
698+
"/gt1l/land_ice_segments/longitude,\\\n",
699+
"/gt1l/land_ice_segments/segment_id,\\\n",
700+
"/gt1l/land_ice_segments/sigma_geo_h,\\\n",
701+
"/gt1r/land_ice_segments/atl06_quality_summary,\\\n",
702+
"/gt1r/land_ice_segments/delta_time,\\\n",
703+
"/gt1r/land_ice_segments/h_li,\\\n",
704+
"/gt1r/land_ice_segments/h_li_sigma,\\\n",
705+
"/gt1r/land_ice_segments/latitude,\\\n",
706+
"/gt1r/land_ice_segments/longitude,\\\n",
707+
"/gt1r/land_ice_segments/segment_id,\\\n",
708+
"/gt1r/land_ice_segments/sigma_geo_h,\\\n",
709+
"/gt2l/land_ice_segments/atl06_quality_summary,\\\n",
710+
"/gt2l/land_ice_segments/delta_time,\\\n",
711+
"/gt2l/land_ice_segments/h_li,\\\n",
712+
"/gt2l/land_ice_segments/h_li_sigma,\\\n",
713+
"/gt2l/land_ice_segments/latitude,\\\n",
714+
"/gt2l/land_ice_segments/longitude,\\\n",
715+
"/gt2l/land_ice_segments/segment_id,\\\n",
716+
"/gt2l/land_ice_segments/sigma_geo_h,\\\n",
717+
"/gt2r/land_ice_segments/atl06_quality_summary,\\\n",
718+
"/gt2r/land_ice_segments/delta_time,\\\n",
719+
"/gt2r/land_ice_segments/h_li,\\\n",
720+
"/gt2r/land_ice_segments/h_li_sigma,\\\n",
721+
"/gt2r/land_ice_segments/latitude,\\\n",
722+
"/gt2r/land_ice_segments/longitude,\\\n",
723+
"/gt2r/land_ice_segments/segment_id,\\\n",
724+
"/gt2r/land_ice_segments/sigma_geo_h,\\\n",
725+
"/gt3l/land_ice_segments/atl06_quality_summary,\\\n",
726+
"/gt3l/land_ice_segments/delta_time,\\\n",
727+
"/gt3l/land_ice_segments/h_li,\\\n",
728+
"/gt3l/land_ice_segments/h_li_sigma,\\\n",
729+
"/gt3l/land_ice_segments/latitude,\\\n",
730+
"/gt3l/land_ice_segments/longitude,\\\n",
731+
"/gt3l/land_ice_segments/segment_id,\\\n",
732+
"/gt3l/land_ice_segments/sigma_geo_h,\\\n",
733+
"/gt3r/land_ice_segments/atl06_quality_summary,\\\n",
734+
"/gt3r/land_ice_segments/delta_time,\\\n",
735+
"/gt3r/land_ice_segments/h_li,\\\n",
736+
"/gt3r/land_ice_segments/h_li_sigma,\\\n",
737+
"/gt3r/land_ice_segments/latitude,\\\n",
738+
"/gt3r/land_ice_segments/longitude,\\\n",
739+
"/gt3r/land_ice_segments/segment_id,\\\n",
740+
"/gt3r/land_ice_segments/sigma_geo_h,\\\n",
741+
"/orbit_info/cycle_number,\\\n",
742+
"/orbit_info/rgt,\\\n",
743+
"/orbit_info/orbit_number' \n",
744+
" else: coverage = '' \n",
690745
"\n",
691746
"#If no services selected, set \"agent\" to NO, which will tell the endpoint not to send the request through to the service processor. Instead, it will quickly send back the data \"natively\" without any customization applied.\n",
692747
"if reformat == '' and projection == '' and projection_parameters == '' and coverage == '' and timevar == '' and bbox == '' and bounding_shape == '' and aoi != '3':\n",
@@ -883,7 +938,41 @@
883938
"outputs": [],
884939
"source": [
885940
"#List files\n",
886-
"os.listdir(path)"
941+
"sorted(os.listdir(path))"
942+
]
943+
},
944+
{
945+
"cell_type": "code",
946+
"execution_count": null,
947+
"metadata": {},
948+
"outputs": [],
949+
"source": [
950+
"#Compare input and output data\n",
951+
"#move to Outputs path\n",
952+
"os.chdir(path)\n",
953+
"native_file = 'ATL06_20190222031203_08500210_001_01.h5'\n",
954+
"processed_file = 'processed_ATL06_20190222031203_08500210_001_01.h5'\n",
955+
"native = h5py.File(native_file, 'r')\n",
956+
"processed = h5py.File(processed_file, 'r')\n",
957+
"\n",
958+
"# Compare granule groups\n",
959+
"print(list(native.keys()))\n",
960+
"print(list(processed.keys()))\n",
961+
"\n",
962+
"# Compared gt1l land ice height \n",
963+
"n_hvar = native['/gt1l/land_ice_segments/h_li']\n",
964+
"n_h = n_hvar[:]\n",
965+
"n_latvar = native['/gt1l/land_ice_segments/latitude']\n",
966+
"n_latitude = n_latvar[:]\n",
967+
"n_lonvar = native['/gt1l/land_ice_segments/longitude']\n",
968+
"n_longitude = n_lonvar[:]\n",
969+
"\n",
970+
"p_hvar = processed['/gt1l/land_ice_segments/h_li']\n",
971+
"p_h = p_hvar[:]\n",
972+
"p_latvar = processed['/gt1l/land_ice_segments/latitude']\n",
973+
"p_latitude = p_latvar[:]\n",
974+
"p_lonvar = processed['/gt1l/land_ice_segments/longitude']\n",
975+
"p_longitude = p_lonvar[:]\n"
887976
]
888977
},
889978
{
@@ -893,19 +982,31 @@
893982
"outputs": [],
894983
"source": [
895984
"#remove Outputs folder for testing\n",
896-
"#path = str(os.getcwd() + '/Outputs')\n",
897-
"#shutil.rmtree(path)\n",
985+
"path = '/home/jovyan/data-access/notebooks/Outputs'\n",
986+
"shutil.rmtree(path)\n",
898987
"\n",
899-
"glacier_path = str(os.getcwd() + '/pine_island_glims')\n",
900-
"#print(glacier_path)\n",
988+
"glacier_path = '/home/jovyan/data-access/notebooks/pine_island_glims'\n",
901989
"shutil.rmtree(glacier_path)"
902990
]
903991
},
992+
{
993+
"cell_type": "code",
994+
"execution_count": null,
995+
"metadata": {},
996+
"outputs": [],
997+
"source": [
998+
"#move outputs to s3\n",
999+
"#os.rename('Outputs', 'data-access-subsetted-outputs')\n",
1000+
"\n",
1001+
"# aws s3 sync data-access-outputs/ s3://pangeo-data-upload-oregon/icesat2/data-access-outputs/\n",
1002+
"# aws s3 sync data-access-subsetted-outputs/ s3://pangeo-data-upload-oregon/icesat2/data-access-subsetted-outputs/"
1003+
]
1004+
},
9041005
{
9051006
"cell_type": "markdown",
9061007
"metadata": {},
9071008
"source": [
908-
"### To review, we have explored data availability and volume over a region and time of interest, discovered and selected data customization options, and downloaded data directly to our local machine. You are welcome to request different data sets, areas of interest, and/or customization services by re-running the notebook or starting again at the 'Select a data set of interest' step above. "
1009+
"### To review, we have explored data availability and volume over a region and time of interest, discovered and selected data customization options, and downloaded data directly to our Pangeo environment. You are welcome to request different data sets, areas of interest, and/or customization services by re-running the notebook or starting again at the 'Select a data set of interest' step above. "
9091010
]
9101011
}
9111012
],

‎notebooks/NSIDC DAAC ICESat-2 Customize and Access.ipynb‎

Lines changed: 117 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
},
3333
{
3434
"cell_type": "code",
35-
"execution_count": null,
35+
"execution_count": 2,
3636
"metadata": {},
3737
"outputs": [],
3838
"source": [
@@ -50,6 +50,7 @@
5050
"import geopandas as gpd\n",
5151
"import matplotlib.pyplot as plt\n",
5252
"import fiona\n",
53+
"import h5py\n",
5354
"# To read KML files with geopandas, we will need to enable KML support in fiona (disabled by default)\n",
5455
"fiona.drvsupport.supported_drivers['LIBKML'] = 'rw'\n",
5556
"from shapely.geometry import Polygon, mapping\n",
@@ -681,12 +682,66 @@
681682
"# Select variable subsetting\n",
682683
"\n",
683684
"if len(variable_vals) > 0:\n",
684-
" v = input('Variable subsetting is available. Would you like to subset a selection of variables? (y/n)')\n",
685-
" if v == 'y':\n",
686-
" print('The', short_name, 'variables to select from include:')\n",
687-
" pprint.pprint(variable_vals)\n",
688-
" coverage = input('If you would like to subset by variable, copy and paste the variables you would like separated by comma. Make sure to omit quotes but include all forward slashes: ')\n",
689-
" else: coverage = ''\n",
685+
" var = input('Variable subsetting is available. Would you like to subset by variables? (y/n)')\n",
686+
" if var == 'y':\n",
687+
" v = input('Would you like to view the list of variables? (y/n)')\n",
688+
" if v == 'y':\n",
689+
" print('The', short_name, 'variables to select from include:')\n",
690+
" pprint.pprint(variable_vals)\n",
691+
" # Enter variables to subset separated by comma. Make sure to omit quotes but include all forward slashes to indicate HDF group hierarchy\n",
692+
" coverage = '/ancillary_data/atlas_sdp_gps_epoch,\\\n",
693+
"/gt1l/land_ice_segments/atl06_quality_summary,\\\n",
694+
"/gt1l/land_ice_segments/delta_time,\\\n",
695+
"/gt1l/land_ice_segments/h_li,\\\n",
696+
"/gt1l/land_ice_segments/h_li_sigma,\\\n",
697+
"/gt1l/land_ice_segments/latitude,\\\n",
698+
"/gt1l/land_ice_segments/longitude,\\\n",
699+
"/gt1l/land_ice_segments/segment_id,\\\n",
700+
"/gt1l/land_ice_segments/sigma_geo_h,\\\n",
701+
"/gt1r/land_ice_segments/atl06_quality_summary,\\\n",
702+
"/gt1r/land_ice_segments/delta_time,\\\n",
703+
"/gt1r/land_ice_segments/h_li,\\\n",
704+
"/gt1r/land_ice_segments/h_li_sigma,\\\n",
705+
"/gt1r/land_ice_segments/latitude,\\\n",
706+
"/gt1r/land_ice_segments/longitude,\\\n",
707+
"/gt1r/land_ice_segments/segment_id,\\\n",
708+
"/gt1r/land_ice_segments/sigma_geo_h,\\\n",
709+
"/gt2l/land_ice_segments/atl06_quality_summary,\\\n",
710+
"/gt2l/land_ice_segments/delta_time,\\\n",
711+
"/gt2l/land_ice_segments/h_li,\\\n",
712+
"/gt2l/land_ice_segments/h_li_sigma,\\\n",
713+
"/gt2l/land_ice_segments/latitude,\\\n",
714+
"/gt2l/land_ice_segments/longitude,\\\n",
715+
"/gt2l/land_ice_segments/segment_id,\\\n",
716+
"/gt2l/land_ice_segments/sigma_geo_h,\\\n",
717+
"/gt2r/land_ice_segments/atl06_quality_summary,\\\n",
718+
"/gt2r/land_ice_segments/delta_time,\\\n",
719+
"/gt2r/land_ice_segments/h_li,\\\n",
720+
"/gt2r/land_ice_segments/h_li_sigma,\\\n",
721+
"/gt2r/land_ice_segments/latitude,\\\n",
722+
"/gt2r/land_ice_segments/longitude,\\\n",
723+
"/gt2r/land_ice_segments/segment_id,\\\n",
724+
"/gt2r/land_ice_segments/sigma_geo_h,\\\n",
725+
"/gt3l/land_ice_segments/atl06_quality_summary,\\\n",
726+
"/gt3l/land_ice_segments/delta_time,\\\n",
727+
"/gt3l/land_ice_segments/h_li,\\\n",
728+
"/gt3l/land_ice_segments/h_li_sigma,\\\n",
729+
"/gt3l/land_ice_segments/latitude,\\\n",
730+
"/gt3l/land_ice_segments/longitude,\\\n",
731+
"/gt3l/land_ice_segments/segment_id,\\\n",
732+
"/gt3l/land_ice_segments/sigma_geo_h,\\\n",
733+
"/gt3r/land_ice_segments/atl06_quality_summary,\\\n",
734+
"/gt3r/land_ice_segments/delta_time,\\\n",
735+
"/gt3r/land_ice_segments/h_li,\\\n",
736+
"/gt3r/land_ice_segments/h_li_sigma,\\\n",
737+
"/gt3r/land_ice_segments/latitude,\\\n",
738+
"/gt3r/land_ice_segments/longitude,\\\n",
739+
"/gt3r/land_ice_segments/segment_id,\\\n",
740+
"/gt3r/land_ice_segments/sigma_geo_h,\\\n",
741+
"/orbit_info/cycle_number,\\\n",
742+
"/orbit_info/rgt,\\\n",
743+
"/orbit_info/orbit_number' \n",
744+
" else: coverage = '' \n",
690745
"\n",
691746
"#If no services selected, set \"agent\" to NO, which will tell the endpoint not to send the request through to the service processor. Instead, it will quickly send back the data \"natively\" without any customization applied.\n",
692747
"if reformat == '' and projection == '' and projection_parameters == '' and coverage == '' and timevar == '' and bbox == '' and bounding_shape == '' and aoi != '3':\n",
@@ -883,7 +938,41 @@
883938
"outputs": [],
884939
"source": [
885940
"#List files\n",
886-
"os.listdir(path)"
941+
"sorted(os.listdir(path))"
942+
]
943+
},
944+
{
945+
"cell_type": "code",
946+
"execution_count": null,
947+
"metadata": {},
948+
"outputs": [],
949+
"source": [
950+
"#Compare input and output data\n",
951+
"#move to Outputs path\n",
952+
"os.chdir(path)\n",
953+
"native_file = 'ATL06_20190222031203_08500210_001_01.h5'\n",
954+
"processed_file = 'processed_ATL06_20190222031203_08500210_001_01.h5'\n",
955+
"native = h5py.File(native_file, 'r')\n",
956+
"processed = h5py.File(processed_file, 'r')\n",
957+
"\n",
958+
"# Compare granule groups\n",
959+
"print(list(native.keys()))\n",
960+
"print(list(processed.keys()))\n",
961+
"\n",
962+
"# Compared gt1l land ice height \n",
963+
"n_hvar = native['/gt1l/land_ice_segments/h_li']\n",
964+
"n_h = n_hvar[:]\n",
965+
"n_latvar = native['/gt1l/land_ice_segments/latitude']\n",
966+
"n_latitude = n_latvar[:]\n",
967+
"n_lonvar = native['/gt1l/land_ice_segments/longitude']\n",
968+
"n_longitude = n_lonvar[:]\n",
969+
"\n",
970+
"p_hvar = processed['/gt1l/land_ice_segments/h_li']\n",
971+
"p_h = p_hvar[:]\n",
972+
"p_latvar = processed['/gt1l/land_ice_segments/latitude']\n",
973+
"p_latitude = p_latvar[:]\n",
974+
"p_lonvar = processed['/gt1l/land_ice_segments/longitude']\n",
975+
"p_longitude = p_lonvar[:]\n"
887976
]
888977
},
889978
{
@@ -893,19 +982,34 @@
893982
"outputs": [],
894983
"source": [
895984
"#remove Outputs folder for testing\n",
896-
"#path = str(os.getcwd() + '/Outputs')\n",
897-
"#shutil.rmtree(path)\n",
985+
"path = '/home/jovyan/data-access/notebooks/Outputs'\n",
986+
"shutil.rmtree(path)\n",
898987
"\n",
899-
"glacier_path = str(os.getcwd() + '/pine_island_glims')\n",
900-
"#print(glacier_path)\n",
988+
"glacier_path = '/home/jovyan/data-access/notebooks/pine_island_glims'\n",
901989
"shutil.rmtree(glacier_path)"
902990
]
903991
},
992+
{
993+
"cell_type": "code",
994+
"execution_count": 3,
995+
"metadata": {},
996+
"outputs": [],
997+
"source": [
998+
"#move outputs to s3\n",
999+
"#os.rename('Outputs', 'data-access-subsetted-outputs')\n",
1000+
"\n",
1001+
"# aws s3 sync data-access-outputs/ s3://pangeo-data-upload-oregon/icesat2/data-access-outputs/\n",
1002+
"# aws s3 sync data-access-subsetted-outputs/ s3://pangeo-data-upload-oregon/icesat2/data-access-subsetted-outputs/\n",
1003+
"\n",
1004+
"#shutil.rmtree('/home/jovyan/data-access/notebooks/data-access-outputs')\n",
1005+
"#shutil.rmtree('/home/jovyan/data-access/notebooks/data-access-subsetted-outputs')"
1006+
]
1007+
},
9041008
{
9051009
"cell_type": "markdown",
9061010
"metadata": {},
9071011
"source": [
908-
"### To review, we have explored data availability and volume over a region and time of interest, discovered and selected data customization options, and downloaded data directly to our local machine. You are welcome to request different data sets, areas of interest, and/or customization services by re-running the notebook or starting again at the 'Select a data set of interest' step above. "
1012+
"### To review, we have explored data availability and volume over a region and time of interest, discovered and selected data customization options, and downloaded data directly to our Pangeo environment. You are welcome to request different data sets, areas of interest, and/or customization services by re-running the notebook or starting again at the 'Select a data set of interest' step above. "
9091013
]
9101014
}
9111015
],

0 commit comments

Comments
 (0)