Skip to content

Commit 9e0dd90

Browse files
cfbastarzcfbastarz
committed
Removendo dependência do inctime dos scripts e atualizando outras coisas
1 parent bd5762c commit 9e0dd90

46 files changed

Lines changed: 45591 additions & 41 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎costFile_Oper.db‎

964 KB
Binary file not shown.

‎cron_scripts/anls/catalog_anl.yml‎

Lines changed: 494 additions & 0 deletions
Large diffs are not rendered by default.

‎cron_scripts/anls/catalog_bkg.yml‎

Lines changed: 494 additions & 0 deletions
Large diffs are not rendered by default.

‎cron_scripts/anls/convert_smna_icn_fct_to_zarr_pyBAM.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#!/usr/bin/env python
2+
# coding: utf-8
3+
4+
# In[1]:
5+
6+
import os
7+
import pyBAM as pb
8+
import matplotlib.pyplot as plt
9+
import xarray as xr
10+
import hvplot.xarray
11+
12+
13+
# In[2]:
14+
15+
lpath = '/share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls'
16+
17+
ficn = os.path.join(lpath, 'spec_tmp', '2025101600', 'GFCTCPT20251016002025101600F.dic.TQ0299L064')
18+
ffct = os.path.join(lpath, 'spec_tmp', '2025101600', 'GFCTCPT20251016002025101606F.dir.TQ0299L064')
19+
20+
icn = pb.openBAM(ficn)
21+
fct = pb.openBAM(ffct)
22+
23+
24+
# In[3]:
25+
26+
27+
#%%time
28+
29+
vars3d_spec = [
30+
'DIVERGENCE',
31+
'VORTICITY'
32+
'SPECIFIC HUMIDITY',
33+
'VIRTUAL TEMPERATURE',
34+
]
35+
36+
vars3d_grid = [
37+
'LIQ MIXING RATIO PROGNOSTIC',
38+
'ICE MIXING RATIO PROGNOSTIC',
39+
'CLOUD TOTAL PROGNOSTIC',
40+
]
41+
42+
icn_lst = []
43+
44+
for field in icn.VarNames:
45+
if field in vars3d_spec:
46+
icn_lst.append(icn.getField3D(field))
47+
else:
48+
if field in vars3d_grid:
49+
icn_lst_z = []
50+
for level in range(icn.nlevels):
51+
icn_lst_z.append(icn.getField(field, zlevel=level))
52+
#icn_lst.append(xr.merge(icn_lst_z, compat='override'))
53+
icn_lst.append(xr.concat(icn_lst_z, dim='lev'))
54+
else:
55+
icn_lst.append(icn.getField(field))
56+
57+
icn_dset = xr.merge(icn_lst)
58+
59+
60+
# In[ ]:
61+
62+
63+
icn.close()
64+
65+
66+
# In[4]:
67+
68+
69+
#%%time
70+
71+
icn_dset.to_zarr(os.path.join(lpath, '2025101600','GFCTCPT20251016002025101600F.icn.TQ0299L064.zarr'), mode='w', consolidated=True)
72+
73+
74+
# In[5]:
75+
76+
77+
#%%time
78+
79+
vars3d_spec = [
80+
'DIVERGENCE',
81+
'VORTICITY'
82+
'SPECIFIC HUMIDITY',
83+
'VIRTUAL TEMPERATURE',
84+
]
85+
86+
vars3d_grid = [
87+
'LIQ MIXING RATIO PROGNOSTIC',
88+
'ICE MIXING RATIO PROGNOSTIC',
89+
'CLOUD TOTAL PROGNOSTIC',
90+
'OZONE MIXING RATIO',
91+
]
92+
93+
fct_lst = []
94+
95+
for field in fct.VarNames:
96+
if field in vars3d_spec:
97+
fct_lst.append(fct.getField3D(field))
98+
else:
99+
if field in vars3d_grid:
100+
fct_lst_z = []
101+
for level in range(fct.nlevels):
102+
fct_lst_z.append(fct.getField(field, zlevel=level))
103+
#fct_lst.append(xr.merge(fct_lst_z, compat='override'))
104+
fct_lst.append(xr.concat(fct_lst_z, dim='lev'))
105+
else:
106+
fct_lst.append(fct.getField(field))
107+
108+
fct_dset = xr.merge(fct_lst)
109+
110+
111+
# In[6]:
112+
113+
114+
#%%time
115+
116+
fct_dset.to_zarr(os.path.join(lpath, '2025101600','GFCTCPT20251016002025101606F.fct.TQ0299L064.zarr'), mode='w', consolidated=True)
117+
118+
119+
# In[7]:
120+
121+
122+
fct.close()
123+
124+
125+
# In[ ]:
126+
127+
128+
129+

‎cron_scripts/anls/create_catalog.sh‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#! /bin/bash -x
2+
3+
#inctime=${HOME}/bin/inctime
4+
5+
lpath=/share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls
6+
7+
Names=(anl bkg)
8+
9+
for name in ${Names[@]}
10+
do
11+
12+
datai=2025100900
13+
dataf=2025101600
14+
15+
data=${datai}
16+
17+
echo "sources:" > ${lpath}/catalog_${name}.yml
18+
19+
while [ ${data} -le ${dataf} ]
20+
do
21+
22+
echo ${data}
23+
24+
if [ ${name} == "anl" ]
25+
then
26+
27+
cat << EOF >> ${lpath}/catalog_${name}.yml
28+
29+
'${data}':
30+
args:
31+
consolidated: true
32+
urlpath: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}/GFCTCPT${data}${data}F.icn.TQ0299L064.zarr
33+
description: SMNA Analysis for ${data}
34+
driver: intake_xarray.xzarr.ZarrSource
35+
metadata:
36+
catalog_dir: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}
37+
tags:
38+
- atmosphere
39+
- analysis
40+
- data_assimilation
41+
- smna
42+
- field
43+
url: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}
44+
45+
EOF
46+
47+
chmod 755 ${lpath}/catalog_${name}.yml
48+
49+
else
50+
51+
#datafct=$(${inctime} ${data} +6h %y4%m2%d2%h2)
52+
datafct=$(date -u +%Y%m%d%H -d "${data:0:8} ${data:8:2} +6 hours")
53+
54+
cat << EOF >> ${lpath}/catalog_${name}.yml
55+
56+
'${data}':
57+
args:
58+
consolidated: true
59+
urlpath: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}/GFCTCPT${data}${datafct}F.fct.TQ0299L064.zarr
60+
description: SMNA Analysis for ${data}
61+
driver: intake_xarray.xzarr.ZarrSource
62+
metadata:
63+
catalog_dir: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}
64+
tags:
65+
- atmosphere
66+
- background
67+
- data_assimilation
68+
- smna
69+
- field
70+
url: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}
71+
72+
EOF
73+
74+
chmod 755 ${lpath}/catalog_${name}.yml
75+
76+
fi
77+
78+
#data=$(${inctime} ${data} +6h %y4%m2%d2%h2)
79+
data=$(date -u +%Y%m%d%H -d "${data:0:8} ${data:8:2} +6 hours")
80+
81+
done
82+
83+
done
84+
85+
exit 0

‎cron_scripts/anls/create_catalog.sh-template‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#! /bin/bash -x
22

3-
inctime=${HOME}/bin/inctime
3+
#inctime=${HOME}/bin/inctime
44

55
lpath=/share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls
66

@@ -29,48 +29,54 @@ cat << EOF >> ${lpath}/catalog_${name}.yml
2929
'${data}':
3030
args:
3131
consolidated: true
32-
urlpath: https://s0.cptec.inpe.br/pesquisa/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/${data}/GFCTCPT${data}${data}F.icn.TQ0299L064.zarr
32+
urlpath: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}/GFCTCPT${data}${data}F.icn.TQ0299L064.zarr
3333
description: SMNA Analysis for ${data}
3434
driver: intake_xarray.xzarr.ZarrSource
3535
metadata:
36-
catalog_dir: https://s0.cptec.inpe.br/pesquisa/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/${data}
36+
catalog_dir: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}
3737
tags:
3838
- atmosphere
3939
- analysis
4040
- data_assimilation
4141
- smna
4242
- field
43-
url: https://s0.cptec.inpe.br/pesquisa/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/${data}
43+
url: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}
4444
4545
EOF
4646

47+
chmod 755 ${lpath}/catalog_${name}.yml
48+
4749
else
4850

49-
datafct=$(${inctime} ${data} +6h %y4%m2%d2%h2)
51+
#datafct=$(${inctime} ${data} +6h %y4%m2%d2%h2)
52+
datafct=$(date -u +%Y%m%d%H -d "${data:0:8} ${data:8:2} +6 hours")
5053

5154
cat << EOF >> ${lpath}/catalog_${name}.yml
5255
5356
'${data}':
5457
args:
5558
consolidated: true
56-
urlpath: https://s0.cptec.inpe.br/pesquisa/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/${data}/GFCTCPT${data}${datafct}F.fct.TQ0299L064.zarr
59+
urlpath: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}/GFCTCPT${data}${datafct}F.fct.TQ0299L064.zarr
5760
description: SMNA Analysis for ${data}
5861
driver: intake_xarray.xzarr.ZarrSource
5962
metadata:
60-
catalog_dir: https://s0.cptec.inpe.br/pesquisa/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/${data}
63+
catalog_dir: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}
6164
tags:
6265
- atmosphere
6366
- background
6467
- data_assimilation
6568
- smna
6669
- field
67-
url: https://s0.cptec.inpe.br/pesquisa/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/${data}
70+
url: https://dataserver.cptec.inpe.br/dataserver_dimnt/das/carlos.bastarz/SMNAMonitoringApp/anls/${data}
6871
6972
EOF
73+
74+
chmod 755 ${lpath}/catalog_${name}.yml
7075

7176
fi
72-
73-
data=$(${inctime} ${data} +6h %y4%m2%d2%h2)
77+
78+
#data=$(${inctime} ${data} +6h %y4%m2%d2%h2)
79+
data=$(date -u +%Y%m%d%H -d "${data:0:8} ${data:8:2} +6 hours")
7480

7581
done
7682

‎cron_scripts/anls/run_convert_smna_dataset_to_zarr.sh-template‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#
55
# @cfbastarz (March, 2024)
66

7-
inctime=/home/carlos.bastarz/bin/inctime
7+
#inctime=/home/carlos.bastarz/bin/inctime
88

99
lpath=/share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls
1010
rpath=/lustre_xc50/ioper/models/SMNA-Oper/SMG/datainout/bam/pos/dataout/TQ0299L064
@@ -25,7 +25,8 @@ do
2525

2626
rsync -arv carlos_bastarz@login-xc50.cptec.inpe.br:${rpath}/${data}/GPOSCPT${data}${data}P.icn.TQ0299L064.grb .
2727

28-
data=$(${inctime} ${data} +1d %y4%m2%d2%h2)
28+
#data=$(${inctime} ${data} +1d %y4%m2%d2%h2)
29+
data=$(date -u +%Y%m%d%H -d "${data:0:8} ${data:8:2} +24 hours")
2930

3031
done
3132

‎cron_scripts/anls/run_convert_smna_icn_fct_to_zarr_pyBAM.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#! /bin/bash -x
2+
3+
# Wrapper script to run convert_smna_dataset_to_zarr.py.
4+
#
5+
# @cfbastarz (March, 2024)
6+
7+
source /scripts/das/intel_fortran/intel/oneapi/setvars.sh
8+
9+
export SIGIOBAM=/scripts/das/carlos.bastarz/libsigiobam-1.1-ifort/local
10+
export LD_LIBRARY_PATH=/${SIGIOBAM}/lib:${LD_LIBRARY_PATH}
11+
12+
eval "$(conda shell.bash hook)"
13+
conda activate pyBAM
14+
15+
#inctime=/home/carlos.bastarz/bin/inctime
16+
17+
lpath=/share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls
18+
rpath=/lustre_xc50/ioper/models/SMNA-Oper/SMG/datainout/bam/model/dataout/TQ0299L064/DAS
19+
20+
datai=2025100900
21+
dataf=2025101600
22+
23+
data=${datai}
24+
25+
while [ ${data} -le ${dataf} ]
26+
do
27+
28+
#datafct=$(${inctime} ${data} +6h %y4%m2%d2%h2)
29+
datafct=$(date -u +%Y%m%d%H -d "${data:0:8} ${data:8:2} +6 hours")
30+
31+
# First, get the SMNA analysis from the BAM post-processing in XC50
32+
mkdir -p /share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/spec_tmp/${data}
33+
mkdir -p /share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/${data}
34+
35+
cd /share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/spec_tmp/${data}
36+
37+
rsync -arv carlos_bastarz@login-xc50.cptec.inpe.br:${rpath}/${data}/GFCTCPT${data}${data}F.icn.TQ0299L064 .
38+
rsync -arv carlos_bastarz@login-xc50.cptec.inpe.br:${rpath}/${data}/GFCTCPT${data}${data}F.dic.TQ0299L064 .
39+
rsync -arv carlos_bastarz@login-xc50.cptec.inpe.br:${rpath}/${data}/GFCTCPT${data}${datafct}F.dir.TQ0299L064 .
40+
rsync -arv carlos_bastarz@login-xc50.cptec.inpe.br:${rpath}/${data}/GFCTCPT${data}${datafct}F.fct.TQ0299L064 .
41+
42+
cat ${lpath}/convert_smna_icn_fct_to_zarr_pyBAM.py-template | sed "s,%DATAI%,${data},g" > ${lpath}/convert_smna_icn_fct_to_zarr_pyBAM.py
43+
sed -i "s,%DATAF%,${datafct},g" ${lpath}/convert_smna_icn_fct_to_zarr_pyBAM.py
44+
45+
/share/das/miniconda3/envs/pyBAM/bin/python3 ${lpath}/convert_smna_icn_fct_to_zarr_pyBAM.py
46+
47+
#data=$(${inctime} ${data} +6h %y4%m2%d2%h2)
48+
data=$(date -u +%Y%m%d%H -d "${data:0:8} ${data:8:2} +6 hours")
49+
50+
done
51+
52+
conda deactivate pyBAM
53+
54+
chmod +x /share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/create_catalog.sh
55+
56+
chmod -R 755 /share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/20*
57+
58+
/share/das/dist/carlos.bastarz/SMNAMonitoringApp/anls/create_catalog.sh

0 commit comments

Comments
 (0)