Skip to content

Commit 0213529

Browse files
authored
Merge pull request #392 from ukmda/markmac99
December 2024 updates
2 parents 0381f8a + b8a5adc commit 0213529

12 files changed

Lines changed: 152 additions & 27 deletions

File tree

archive/analysis/getLogData.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@ fi
1616
# create performance metrics
1717
cd $SRC/logs
1818
python -m metrics.timingMetrics $rundate
19-
mv -f $rundate-perfNightly.jpg $DATADIR/
20-
aws s3 cp $DATADIR/$rundate-perfNightly.jpg $WEBSITEBUCKET/reports/batchcharts/ --quiet
19+
aws s3 cp $DATADIR/batchcharts/$rundate-perfNightly.jpg $WEBSITEBUCKET/reports/batchcharts/ --quiet
2120

2221
#lastmtch=$(ls -1tr $SRC/logs/matches-${rundate}-*.log | tail -1)
2322
if [ $rundate != $(date +%Y%m%d) ] ; then

archive/analysis/getRMSSingleData.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ if [ ! -f $mrgfile ] ; then
3636
cat $SRC/analysis/templates/ukmon-single.txt > $mrgfile
3737
fi
3838
# file containing only new data
39-
cat $SRC/analysis/templates/ukmon-single.txt > > $newsngl
39+
cat $SRC/analysis/templates/ukmon-single.txt >> $newsngl
4040

4141
ls -1 $outdir/ukmda_??????_${yr}*.csv | while read i
4242
do

archive/analysis/runDistrib.sh

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,13 @@ stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reser
5050
if [ $stat -eq 80 ]; then
5151
aws ec2 start-instances --instance-ids $SERVERINSTANCEID
5252
fi
53-
log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" runDistrib
53+
5454
while [ "$stat" -ne 16 ]; do
5555
sleep 5
5656
log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" runDistrib
5757
stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text)
5858
done
59+
5960
log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate}" runDistrib
6061
conda activate $HOME/miniconda3/envs/${WMPL_ENV}
6162

@@ -89,6 +90,7 @@ rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py ec2-use
8990
rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py ec2-user@$privip:src/ukmon_pylib/traj
9091

9192
# now run the script
93+
log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib
9294
ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execdist"
9395

9496
log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib
@@ -135,9 +137,12 @@ if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then
135137
execConsolsh=/tmp/$execcons
136138
python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh')"
137139
chmod +x $execConsolsh
140+
141+
log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib
138142
scp -i $SERVERSSHKEY $execConsolsh ec2-user@$privip:data/distrib/$execcons
139143
ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execcons"
140144

145+
log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation" runDistrib
141146
scp -i $SERVERSSHKEY ec2-user@$privip:data/distrib/processed_trajectories.json $DATADIR/distrib
142147

143148
ssh -i $SERVERSSHKEY ec2-user@$privip "rm -f data/distrib/*.json /tmp/processed_trajectories.json"
@@ -151,7 +156,8 @@ if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then
151156

152157
python -c "from traj.consolidateDistTraj import patchTrajDB ; patchTrajDB('$DATADIR/distrib/processed_trajectories.json','/home/ec2-user/ukmon-shared/matches/RMSCorrelate', '/home/ec2-user/data/distrib');"
153158

154-
# push the updated traj db to the S3 bucket
159+
# archive older data then push the updated traj db to the S3 bucket
160+
python -m traj.jsonDbMaintenance $DATADIR/distrib/
155161
aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet
156162

157163
log2cw $NJLOGGRP $NJLOGSTREAM "compressing the procssed data" runDistrib

archive/cronjobs/nightlyJob.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,8 @@ $SRC/utils/loadSingleCsvMDB.sh
142142
# has to be run quite late as not all trajectories have synced to the website earlier
143143
$SRC/analysis/updatePlotsAndDetStatus.sh
144144

145+
aws s3 sync $SRC/share/ s3://ukmda-website/browse --exclude "*" --include "datadictionary.xlsx" --quiet
146+
145147
log2cw $NJLOGGRP $NJLOGSTREAM "finished nightlyJob" nightlyJob
146148

147149
# grab the logs for the website - run this last to capture the above Finished message

archive/deployment/analysis.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,5 @@
3232
- {src: '{{srcdir}}/analysis/templates/match_hdr_full.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no }
3333
- {src: '{{srcdir}}/analysis/templates/UA_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no }
3434
- {src: '{{srcdir}}/analysis/templates/UO_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no }
35+
- {src: '{{srcdir}}/analysis/templates/ukmon-single.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no }
3536
- {src: '{{srcdir}}/analysis/README.md', dest: '{{destdir}}/analysis', mode: '644', backup: no }

archive/terraform/ukmda/ec2.tf

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,46 @@ resource "aws_network_interface" "calcserver_if" {
3535
"billingtag" = "ukmda"
3636
}
3737
}
38+
################################################
39+
# Ubuntu calc server
40+
################################################
41+
42+
resource "aws_instance" "ubuntu_calc_server" {
43+
ami = "ami-0bdf149a42243bde8"
44+
instance_type = "c6g.4xlarge"
45+
iam_instance_profile = aws_iam_instance_profile.calcserverrole.name
46+
key_name = aws_key_pair.marks_key.key_name
47+
tags = {
48+
"Name" = "Calcengine2"
49+
"billingtag" = "ukmda"
50+
}
51+
root_block_device {
52+
tags = {
53+
"Name" = "calcengine2"
54+
"billingtag" = "ukmda"
55+
}
56+
volume_size = 100
57+
}
58+
network_interface {
59+
network_interface_id = aws_network_interface.ubuntu_calcserver_if.id
60+
device_index = 0
61+
}
62+
}
63+
64+
# elastic network interface attached to the calc server
65+
66+
resource "aws_network_interface" "ubuntu_calcserver_if" {
67+
subnet_id = aws_subnet.ec2_subnet.id
68+
description = "Primary network interface"
69+
private_ips = [var.ubuntu_calcserverip]
70+
security_groups = [aws_security_group.ec2_secgrp.id]
71+
ipv6_address_list_enabled = false
72+
tags = {
73+
"Name" = "calcengine2"
74+
"billingtag" = "ukmda"
75+
}
76+
}
77+
3878
################################################
3979
# admin server
4080
################################################

archive/terraform/ukmda/variables.tf

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ variable mgmt_cidr { default = "172.32.36.0/22" }
2525
variable lambda_cidr { default = "172.32.32.0/22" }
2626
variable ec2_cidr { default = "172.32.16.0/20" }
2727
variable calcserverip { default = "172.32.16.136" }
28+
variable ubuntu_calcserverip { default = "172.32.16.137" }
29+
2830

2931
variable archalias { default = "archive.ukmeteors.co.uk"}
3032
variable mainalias { default = "www.ukmeteors.co.uk"}

archive/ukmon_pylib/metrics/costMetrics.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,21 +32,21 @@ def getAllMthly():
3232
lastmth=(datetime.datetime.now() + datetime.timedelta(days=-20)).strftime('%Y%m')
3333
monthlyCostByService(lastmth, '183798037734')
3434
monthlyCostByService(lastmth, '317976261112')
35-
monthlyCostByService(lastmth, '822069317839')
35+
#monthlyCostByService(lastmth, '822069317839')
3636

3737

3838
def getAllPrvMthly():
3939
lastmth=(datetime.datetime.now() + datetime.timedelta(days=-50)).strftime('%Y%m')
4040
monthlyCostByService(lastmth, '183798037734')
4141
monthlyCostByService(lastmth, '317976261112')
42-
monthlyCostByService(lastmth, '822069317839')
42+
#monthlyCostByService(lastmth, '822069317839')
4343

4444

4545
def getAllMthToDate():
4646
lastmth=(datetime.datetime.now()).strftime('%Y%m')
4747
monthlyCostByService(lastmth, '183798037734')
4848
monthlyCostByService(lastmth, '317976261112')
49-
monthlyCostByService(lastmth, '822069317839')
49+
#monthlyCostByService(lastmth, '822069317839')
5050

5151

5252
def getAllCostsAndUsage(ceclient, startdt, enddt, svcs, tagval, acctid):

archive/ukmon_pylib/metrics/timingMetrics.py

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,30 +12,30 @@
1212

1313

1414
def graphOfData(logf, dtstr):
15-
with open(logf,'r') as inf:
16-
lis = inf.readlines()
15+
lis = open(logf,'r').readlines()
1716
dta = [li for li in lis if li[:8]==dtstr]
1817

1918
# note: timings in log are the start times of the process
2019
# so we offset the labels by 1 to align with the end times
2120
times = []
22-
events = ['start']
21+
events = []
2322
elapsed = []
24-
lasttime = 0
23+
lasttime = None
24+
starttime = None
2525
for li in dta:
2626
spls = li.split(',')
27-
elapsed.append(int(spls[2])) # runtime so far
27+
currtime = datetime.datetime.strptime(f'{spls[0]}_{spls[1]}', '%Y%m%d_%H:%M:%S')
28+
if starttime is None:
29+
starttime = currtime
30+
lasttime = currtime
31+
elap = (currtime - starttime).seconds
32+
elapsed.append(elap) # runtime so far
2833
events.append(spls[4].strip()) # event name
29-
times.append(int(spls[2])-lasttime) # duration of event
30-
lasttime = int(spls[2])
34+
times.append((currtime - lasttime).seconds) # duration of event
35+
lasttime = currtime
3136

32-
# add two dummy end-time values
33-
elapsed.append(elapsed[-1])
34-
times.append(0)
35-
3637
fig, ax = plt.subplots()
3738
width = 0.35
38-
3939
ax.set_ylabel('Task')
4040
ax.set_xlabel('Duration (s)')
4141
ax.set_title('Batch Phases: {}'.format(dtstr))
@@ -47,7 +47,9 @@ def graphOfData(logf, dtstr):
4747
plt.grid(axis='x')
4848
plt.gca().invert_yaxis()
4949
logname, _ = os.path.splitext(os.path.basename(logf))
50-
plt.savefig(f'./{dtstr}-{logname}.jpg', dpi=100)
50+
datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data')
51+
os.makedirs(os.path.join(datadir, 'batchcharts'), exist_ok=True)
52+
plt.savefig(os.path.join(datadir, 'batchcharts',f'./{dtstr}-{logname}.jpg'), dpi=100)
5153
plt.close()
5254

5355

@@ -122,7 +124,8 @@ def getLogStats(nightlogf, matchlogf, thisdy):
122124
df = pd.DataFrame(zip(dts,tss,secs,tsks,msgs), columns=['dts','tss','secs','tsk','msgs'])
123125
df = df.sort_values(by=['tss'])
124126
#print(df)
125-
df.to_csv('perfNightly.csv', mode='a', header=False, index=False)
127+
outdir = os.path.split(nightlogf)[0]
128+
df.to_csv(os.path.join(outdir, 'perfNightly.csv'), mode='a', header=False, index=False)
126129

127130

128131
if __name__ == '__main__':
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# small script to archive off old data from the JSON database
2+
import datetime
3+
import os
4+
import sys
5+
from dateutil.relativedelta import relativedelta
6+
7+
from wmpl.Utils.TrajConversions import datetime2JD, jd2Date
8+
from wmpl.Trajectory.CorrelateRMS import DatabaseJSON
9+
10+
# Name of json file with the list of processed directories
11+
JSON_DB_NAME = "processed_trajectories.json"
12+
13+
14+
def archiveOldRecords(db, db_dir, older_than=3):
15+
"""
16+
Archive off old records to keep the database size down
17+
18+
Keyword Arguments:
19+
older_than: [int] number of months to keep, default 3
20+
"""
21+
class DummyMetObs():
22+
def __init__(self, station, obs_id):
23+
self.station_code = station
24+
self.id = obs_id
25+
26+
archdate = datetime.datetime.now(datetime.timezone.utc) - relativedelta(months=older_than)
27+
archdate_jd = datetime2JD(archdate)
28+
29+
arch_db_path = os.path.join(db_dir, 'archive', f'{archdate.strftime("%Y%m")}_{JSON_DB_NAME}')
30+
os.makedirs(os.path.join(db_dir, 'archive'), exist_ok=True)
31+
archdb = DatabaseJSON(arch_db_path)
32+
33+
for traj in [t for t in db.trajectories if t < archdate_jd]:
34+
if traj < archdate_jd:
35+
archdb.addTrajectory(None, db.trajectories[traj], False)
36+
db.removeTrajectory(db.trajectories[traj])
37+
38+
for traj in [t for t in db.failed_trajectories if t < archdate_jd]:
39+
if traj < archdate_jd:
40+
archdb.addTrajectory(None, db.failed_trajectories[traj], True)
41+
db.removeTrajectory(db.failed_trajectories[traj])
42+
43+
for station in db.processed_dirs:
44+
arch_processed = [dirname for dirname in db.processed_dirs[station] if
45+
datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate]
46+
for dirname in arch_processed:
47+
archdb.addProcessedDir(station, dirname)
48+
db.processed_dirs[station].remove(dirname)
49+
50+
for station in db.paired_obs:
51+
arch_processed = [obs_id for obs_id in db.paired_obs[station] if
52+
datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate]
53+
for obs_id in arch_processed:
54+
archdb.addPairedObservation(DummyMetObs(station, obs_id))
55+
db.paired_obs[station].remove(obs_id)
56+
57+
archdb.save()
58+
db.save()
59+
return
60+
61+
62+
if __name__ == '__main__':
63+
db_dir = sys.argv[1]
64+
database_path = os.path.join(db_dir, JSON_DB_NAME)
65+
db = DatabaseJSON(database_path)
66+
soonest = jd2Date(min(min(list(db.trajectories.keys())), min(list(db.failed_trajectories.keys()))), dt_obj=True)
67+
68+
nowdt = datetime.datetime.now()
69+
mthsback = int((nowdt - soonest).days/30)
70+
for i in range(mthsback, 2, -1):
71+
print(f'archiving {i} months back')
72+
archiveOldRecords(db, db_dir, i)

0 commit comments

Comments
 (0)