" >> $logfile
cat $DATADIR/failed/${rundate}_failed.txt >> $logfile
echo "" >> $logfile
+
echo "Chart of Batch Element Runtimes
" >> $logfile
echo "This section shows the runtime of the different jobs in the batch
" >> $logfile
echo "
" >> $logfile
@@ -93,4 +94,7 @@ done
cat $TEMPLATES/footer.html >> $DATADIR/lastlogs/index.html
aws s3 cp $DATADIR/lastlogs/index.html $WEBSITEBUCKET/reports/lastlogs/ --quiet
+# housekeep the fails reports
+find $DATADIR/failed -mtime +90 -exec rm -f {} \;
+
find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \;
diff --git a/archive/lambdas/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py
index 73c393db9..3f4d1f178 100644
--- a/archive/lambdas/matchDataApi/matchDataApi.py
+++ b/archive/lambdas/matchDataApi/matchDataApi.py
@@ -67,7 +67,7 @@ def getStationData(statid, dtstr, period=None):
statfrag = f"and s.stations like '%{statid}%' " if statid is not None else ""
perfrag = periodToSqlFragment(period) if period is not None else ""
with connection.cursor() as cursor:
- sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}"
+ sql = f"SELECT s.orbname from matches s where s.orbname like '{dtstr}%' {statfrag} {perfrag}"
cursor.execute(sql)
result = cursor.fetchall()
finally:
@@ -85,7 +85,7 @@ def getSummaryData(dtstr, period=None):
fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\
'_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\
'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations'
- expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%' {perfrag}"
+ expr = f"SELECT {fieldlist} from matches s where s.orbname like '{dtstr}%' {perfrag}"
result=[]
try:
with connection.cursor() as cursor:
@@ -171,7 +171,7 @@ def lambda_handler(event, context):
res = '{"points": "unavailable"}'
else:
- res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}'
+ res = '{"invalid request type - must be one of \'matches\', \'detail\', \'station\',\'summary\'"}'
print(res)
return {
diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc
index a7f39fa35..b8c939f24 100644
--- a/archive/server_setup/.bashrc
+++ b/archive/server_setup/.bashrc
@@ -55,9 +55,12 @@ fi
unset __conda_setup
# <<< conda initialize <<<
+# make the function keys work properly in python
+export TERM=putty
+
# If this is an xterm set the title to user@host:dir
case "$TERM" in
- xterm*|rxvt*)
+ xterm*|rxvt*|putty*)
PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
;;
*)
diff --git a/archive/server_setup/acctStatusChecks.sh b/archive/server_setup/acctStatusChecks.sh
new file mode 100644
index 000000000..eded44c2c
--- /dev/null
+++ b/archive/server_setup/acctStatusChecks.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+cd $here
+
+sudo ls -1 /var/sftp | egrep -v "test|logup" > moved.txt
+grep batchs /home/ubuntu/prod/data/reports/stationlogins.txt | cut -c 42-61 | sed 's/ //g' | sort | uniq > switched.txt
+grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt | cut -c 42-61 | sed 's/ //g' | sort | uniq > pending.txt
+
+ssh ukmonhelper2 "~/prod/server_setup/get-nbd.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > not-being-done.txt
+ssh ukmonhelper2 "~/prod/server_setup/get-all.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > all-accounts.txt
+
+python ~/src/ukmda-dataprocessing/archive/server_setup/checkSftpAccounts.py
+
+
+echo "Moved: $(wc -l moved.txt | awk '{print $1}')" > statusreport.txt
+echo "Switched: $(wc -l switched.txt | awk '{print $1}')" >> statusreport.txt
+echo "Pending: $(wc -l pending.txt | awk '{print $1}')" >> statusreport.txt
+echo "Total: $(wc -l still-live.txt | awk '{print $1}')" >> statusreport.txt
+echo "" >> statusreport.txt
+echo "Not Live: $(wc -l not_live.txt | awk '{print $1}')" >> statusreport.txt
+echo "Not Upl: $(wc -l not_uploading.txt | awk '{print $1}')" >> statusreport.txt
+echo "" >> statusreport.txt
+echo "Dead: $(wc -l not-being-done.txt | awk '{print $1}')" >> statusreport.txt
+echo "" >> statusreport.txt
+cat statusreport.txt
diff --git a/archive/server_setup/backupSftpAccts.sh b/archive/server_setup/backupSftpAccts.sh
new file mode 100644
index 000000000..df09ff641
--- /dev/null
+++ b/archive/server_setup/backupSftpAccts.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+cd $here
+
+bkpOneUser() {
+ userid=$1
+ srchost=$2
+ mkdir -p ./backup/$userid
+ sudo rsync -av $srchost:/var/sftp/$userid/ ./backup/$userid
+ sudo chown -R $USER:$USER ./backup/$userid
+ cat ./backup/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini
+ mv -f /tmp/$userid.ini ./backup/$userid/ukmon.ini
+}
+
+if [ $# -lt 2 ] ; then
+ echo "Usage: ./backupSftpAccounts.sh oldservername userfile"
+ exit
+fi
+
+echo "Warning: this must only be run on the new server"
+read -p "press ctrl-c to quit or enter to continue"
+
+oldserver=$1
+srcfile=$2
+
+cat $srcfile | while read stn
+do
+ bkpOneUser $stn $oldserver
+done
\ No newline at end of file
diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py
new file mode 100644
index 000000000..de6b7696a
--- /dev/null
+++ b/archive/server_setup/checkSftpAccounts.py
@@ -0,0 +1,75 @@
+import pandas as pd
+import datetime
+
+
+data=open('/home/ubuntu/prod/data/reports/stationlogins.txt').readlines()
+camlist = []
+not_live = []
+still_upl = []
+not_upl = []
+livenames = []
+
+livedate = datetime.datetime.now() - datetime.timedelta(days=10)
+for li in data:
+ if 'Last Upload' in li:
+ continue
+ lastup = li[:19]
+ lastlo = li[21:40]
+ loc = li[42:61]
+ gmnid = li[64:73]
+ via = li[76:].strip()
+ camlist.append([lastup, lastlo, loc, gmnid, via])
+
+ if ">" in lastup and ">" in lastlo:
+ not_live.append([loc, gmnid, lastup, lastlo, via])
+
+ elif ">" not in lastup:
+ lastupdt = datetime.datetime.strptime(lastup, '%Y-%m-%dT%H:%M:%S')
+ if lastupdt >= livedate:
+ still_upl.append([loc, gmnid, lastup, lastlo, via])
+ livenames.append(loc.strip())
+ else:
+ if '>' in lastlo:
+ not_live.append([loc, gmnid, lastup, lastlo, via])
+ else:
+ lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S')
+ if lastlodt >= livedate:
+ not_upl.append([loc, gmnid, lastup, lastlo, via])
+ else:
+ not_live.append([loc, gmnid, lastup, lastlo, via])
+ else: # '>' not in lastlo
+ lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S')
+ if lastlodt >= livedate:
+ not_upl.append([loc, gmnid, lastup, lastlo, via])
+ else:
+ not_live.append([loc, gmnid, lastup, lastlo, via])
+
+with open('still-live.txt','w') as outf:
+ for cam in still_upl:
+ outf.write(','.join(cam) + '\n')
+
+with open('not_uploading.txt','w') as outf:
+ for cam in not_upl:
+ outf.write(','.join(cam) + '\n')
+
+with open('not_live.txt','w') as outf:
+ for cam in not_live:
+ outf.write(','.join(cam) + '\n')
+
+donelist = open('moved.txt', 'r').readlines()
+donelist = [x.strip() for x in donelist]
+with open('todo.txt', 'w') as outf:
+ for nam in livenames:
+ if nam.strip() not in donelist:
+ print('done list is missing', nam)
+ outf.write(f'{nam}\n')
+
+
+moved = open('moved.txt').readlines()
+pending = open('pending.txt').readlines()
+switched = open('switched.txt').readlines()
+
+pendingnotmoved = [x for x in pending if x not in moved]
+
+print('stations marked pending but not moved - check if need done')
+print(pendingnotmoved)
\ No newline at end of file
diff --git a/archive/server_setup/copyTestData.sh b/archive/server_setup/copyTestData.sh
deleted file mode 100644
index e2c041c5a..000000000
--- a/archive/server_setup/copyTestData.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-# prep some test data
-$yr=$(date +%Y)
-
-here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
-source $here/../config.ini >/dev/null 2>&1
-
-rsync -avz ~/prod/data/single/ $DATADIR/single/
-rsync -avz ~/prod/data/matched/*${yr}* $DATADIR/matched/
-rsync -avz ~/prod/data/consolidated/*${yr}* $DATADIR/consolidated/
-rsync -avz ~/prod/data/dailyreports/stats.txt $DATADIR/dailyreports/
-rsync -avz ~/prod/data/dailyreports/${yr}*.txt $DATADIR/dailyreports/
-rsync -avz ~/prod/logs/*${yr}*.txt $SRC/logs/
-rsync -avz ~/prod/data/*${yr}*.jpg $DATADIR/
-rsync -avz ~/prod/data/*.png $DATADIR/
\ No newline at end of file
diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh
index 52d5add0f..823444955 100755
--- a/archive/server_setup/migrateSftpAccts.sh
+++ b/archive/server_setup/migrateSftpAccts.sh
@@ -1,5 +1,7 @@
#!/bin/bash
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+cd $here
addOneUser() {
userid=$1
diff --git a/archive/terraform/ukmda/batchserver.tf b/archive/terraform/ukmda/batchserver.tf
index 7ff5993f8..833338e4c 100644
--- a/archive/terraform/ukmda/batchserver.tf
+++ b/archive/terraform/ukmda/batchserver.tf
@@ -45,3 +45,11 @@ resource "aws_network_interface" "batchserver_if" {
"billingtag" = "ukmda"
}
}
+
+resource "aws_eip" "batchserver" {
+ instance = aws_instance.batchserver.id
+ tags = {
+ billingtag = "ukmda"
+ Name = "ukmonhelper_eip"
+ }
+}
diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py
index 36aeaf985..b6198aa5a 100644
--- a/archive/ukmon_pylib/metrics/camMetrics.py
+++ b/archive/ukmon_pylib/metrics/camMetrics.py
@@ -206,15 +206,15 @@ def backPopulate(stationid):
df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0")
df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime
df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload]
- df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime'])
+ df = df.drop(columns=['manual','rundate', 'upddate','uploadtime'])
df = df.sort_values(by=['lastupload'])
outfile=os.path.join(datadir, 'reports', 'stationlogins.txt')
zerodate = datetime.datetime(1970,1,1,0,0,0)
with open(outfile,'w') as outf:
- outf.write('Last Upload, StationID, Last Login, Via\n')
+ outf.write(f'{"Last Upload":19s} {"Last Connect":19s} {"StationID":20s} {"GMN ID":10s} Via\n')
for _,rw in df.iterrows():
- lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S')
- lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S')
- via = '' if pd.isnull(rw.host) else rw.host
- outf.write(f'{lastup} , {rw.location:20s}, {lastlo}, {via}\n')
+ lastup = '> 1 month ago' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S')
+ lastlo = '> 1 month ago' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S')
+ via = 'ukmonhelper2' if pd.isnull(rw.host) else rw.host
+ outf.write(f'{lastup:19s} {lastlo:19s} {rw.location:20s} {rw.stationid:10s} {via}\n')
diff --git a/archive/ukmon_pylib/metrics/timingMetrics.py b/archive/ukmon_pylib/metrics/timingMetrics.py
index d88273d09..cef8b4b0c 100644
--- a/archive/ukmon_pylib/metrics/timingMetrics.py
+++ b/archive/ukmon_pylib/metrics/timingMetrics.py
@@ -12,6 +12,7 @@
def graphOfData(logf, dtstr):
+
lis = open(logf,'r').readlines()
dta = [li for li in lis if li[:8]==dtstr]
@@ -55,78 +56,35 @@ def graphOfData(logf, dtstr):
def getLogStats(nightlogf, matchlogf, thisdy):
- with open(nightlogf,'r') as inf:
- loglines = inf.readlines()
-
- #logf = os.path.basename(nightlogf)
- #spls = logf.split('-')
- #thisdy= spls[1]
- matchstr = 'RUNTIME'
-
- # <13>Nov 1 11:49:55 nightlyJob: RUNTIME 10554 showerReport ALL 202211
- startFAM = 0
- startRD = 0
- dts = []
- tss = []
- tsks = []
- secs = []
- msgs = []
- for li in loglines:
- if matchstr in li:
- li = ' '.join(li.split()) # remove double-spaces eg when date is ' 6 Jan' as opposed to '16 Jan'
- spls = li.split()
- msg = ''
- for s in range(6, len(spls)):
- msg = msg + ' ' + spls[s]
- if 'start findAllMatches' in msg:
- startFAM = int(spls[5])
- dts.append(thisdy)
- tss.append(spls[2])
- tsks.append(spls[3].replace(':',''))
- secs.append(int(spls[5]))
- msgs.append(msg.strip())
-
- with open(matchlogf,'r') as inf:
- loglines = inf.readlines()
+ # logline example
+ # <13>May 8 06:10:02 nightlyJob: start nightlyJob
+ outdir = os.path.split(nightlogf)[0]
- # scan for findAllMatches data first
- for li in loglines:
- if matchstr in li:
- spls = li.split()
- if 'runDistrib' in spls[3]:
- continue
- msg = ''
- for s in range(6, len(spls)):
- msg = msg + ' ' + spls[s]
- if 'start runDistrib' in msg:
- startRD = int(spls[5])
- dts.append(thisdy)
- tss.append(spls[2])
- tsks.append(spls[3].replace(':',''))
- secs.append(int(spls[5])+startFAM)
- msgs.append(msg.strip())
+ loglines = open(nightlogf,'r').readlines()
+ bsfs = [x for x in loglines if 'start'in x or 'finish' in x or 'end' in x]
+ bsfs = [x for x in bsfs if x[0]=='<']
- # rescan for runDistrib data
- for li in loglines:
- if matchstr in li:
- spls = li.split()
- if 'runDistrib' not in spls[3]:
- continue
- msg = ''
- for s in range(6, len(spls)):
- msg = msg + ' ' + spls[s]
- dts.append(thisdy)
- tss.append(spls[2])
- tsks.append(spls[3].replace(':',''))
- secs.append(int(spls[5]) + startFAM + startRD)
- msgs.append(msg.strip())
+ loglines = open(matchlogf,'r').readlines()
+ msfs = [x for x in loglines if 'start'in x or 'finish' in x or 'end' in x]
+ msfs = [x for x in msfs if x[0]=='<']
- df = pd.DataFrame(zip(dts,tss,secs,tsks,msgs), columns=['dts','tss','secs','tsk','msgs'])
- df = df.sort_values(by=['tss'])
- #print(df)
- outdir = os.path.split(nightlogf)[0]
- df.to_csv(os.path.join(outdir, 'perfNightly.csv'), mode='a', header=False, index=False)
-
+ alldata = msfs + bsfs
+ alldata.sort()
+ starttime = None
+ yr = datetime.datetime.now().year
+ with open(os.path.join(outdir, 'perfNightly.csv'), 'a+') as outf:
+ for rw in alldata:
+ dtpart = rw[4:19]
+ evtdt = datetime.datetime.strptime(f'{yr} {dtpart}', '%Y %b %d %H:%M:%S')
+ if 'start nightlyJob' in rw:
+ starttime = evtdt
+ elapsed_secs = (evtdt - starttime).seconds
+ txtpart = rw[20:]
+ task = txtpart[:txtpart.find(':')]
+ msg = txtpart[txtpart.find(':')+1:].replace(',', ' ')
+ outstr = f'{evtdt.strftime("%Y%m%d,%H:%M:%S")},{elapsed_secs},{task},{msg}'
+ outf.write(outstr)
+
if __name__ == '__main__':
dtstr = sys.argv[1]
diff --git a/archive/ukmon_pylib/reports/findFailedMatches.py b/archive/ukmon_pylib/reports/findFailedMatches.py
index 1b110a1c6..c7c0082d4 100644
--- a/archive/ukmon_pylib/reports/findFailedMatches.py
+++ b/archive/ukmon_pylib/reports/findFailedMatches.py
@@ -16,24 +16,27 @@ def saveData(observation, repfile):
def processOneLog(logfile, repfile):
- with open(logfile, 'r') as inf:
- lis = inf.readlines()
+ lis = open(logfile, 'r').readlines()
obs = False
observation = []
for li in lis:
- if 'Observations:' in li:
+ if 'Observations ' in li:
obs = True
if obs is True:
observation.append(li)
- if 'Shower:' in li:
+ if 'Saving trajectory:' in li or 'Updating database' in li:
obs = False
observation = []
- if '-----------------------' in li:
+ if 'Updating database' in li or 'added to fails' in li:
if obs is True:
+ #print('got fail')
+ observation.append('--------------\n\n')
saveData(observation, repfile)
obs = False
observation = []
+ if "SOLVING RUN DONE" in li:
+ break
return
@@ -48,5 +51,6 @@ def processOneLog(logfile, repfile):
logdir = os.path.join(srcdir, 'logs','distrib')
logs = glob.glob(f'{repdt}*.log', root_dir=logdir)
for logf in logs:
+ #print(logf)
processOneLog(os.path.join(logdir, logf), reportfile)
reportfile.close()
diff --git a/archive/ukmon_pylib/reports/getSolutionStati.py b/archive/ukmon_pylib/reports/getSolutionStati.py
index 1031688ca..e4b440971 100644
--- a/archive/ukmon_pylib/reports/getSolutionStati.py
+++ b/archive/ukmon_pylib/reports/getSolutionStati.py
@@ -5,93 +5,14 @@
import sys
-def getSolutionStati(fname):
- with open(fname, 'r') as inf:
- lis = inf.readlines()
- # find start of interesting data
- totnew = 0
- totchk = 0
- gotnew = 0
- gotchk = 0
- chktraj = False
- newtraj = False
- for li in lis:
- # check for new data for existing solns
- if 'Checking trajectory at ' in li and chktraj is False:
- thistraj = li
- chktraj = True
- totchk += 1
- continue
- if 'Checking trajectory at ' in li and chktraj is True:
- spls = thistraj.strip().split(' ')
- print(f'{spls[3]} {spls[4]} no new data')
- totchk += 1
- thistraj = li
- chktraj = True
- continue
- if "New Trajectory solution failed" in li and chktraj is True:
- spls = thistraj.strip().split(' ')
- print(f'{spls[3]} {spls[4]} no improvement')
- chktraj = False
- continue
- if "Shower:" in li and chktraj is True:
- spls = thistraj.strip().split(' ')
- spl2 = li.strip().split(' ')
- try:
- if spl2[3] == '...':
- shwr = 'Spo'
- else:
- shwr = spl2[3]
- except:
- if spl2[1] == '...':
- shwr = 'Spo'
- else:
- shwr = spl2[1]
-
- print(f'{spls[3]} {spls[4]} updated as {shwr}')
- gotchk += 1
- chktraj = False
- continue
-
- # check for new groups
- if "Observations:" in li:
- thistraj = None
- newtraj = True
- totnew += 1
- continue
- if "Observations:" not in li and newtraj is True and thistraj is None:
- thistraj = li
- continue
- if "-------" in li and newtraj is True:
- spls = thistraj.strip().split(' ')
- print(f'{spls[3]} {spls[4]} not matched')
- newtraj = False
- thistraj = None
- continue
- if "Shower:" in li and newtraj is True:
- spls = thistraj.strip().split(' ')
- spl2 = li.strip().split(' ')
- try:
- if spl2[3] == '...':
- shwr = 'Spo'
- else:
- shwr = spl2[3]
- except:
- if spl2[1] == '...':
- shwr = 'Spo'
- else:
- shwr = spl2[1]
- print(f'{spls[3]} {spls[4]} solved as {shwr}')
- gotnew += 1
- newtraj = False
- thistraj = None
- continue
- if "SOLVING RUN DONE" in li:
- break
-
- return totnew, totchk, gotnew, gotchk
+def parseDistriblog(logname):
+ lis = open(logname,'r').readlines()
+ dta = [x for x in lis if ('added to fails' in x or ('saved' in x and 'to' in x)) and ('CorrelateEngine' in x or 'CorrelateRMS' in x)]
+ offset = 49 if 'match' in logname else 76
+ for dd in dta:
+ print(dd[offset:].strip())
if __name__ == '__main__':
fname = sys.argv[1]
- print(getSolutionStati(fname))
+ parseDistriblog(fname)
diff --git a/archive/ukmon_pylib/reports/reportBadCameras.py b/archive/ukmon_pylib/reports/reportBadCameras.py
index adf4d64ca..54e8070b2 100644
--- a/archive/ukmon_pylib/reports/reportBadCameras.py
+++ b/archive/ukmon_pylib/reports/reportBadCameras.py
@@ -14,6 +14,46 @@
mailfrom = 'ukmonhelper2@ukmeteors.co.uk'
+
+def sendBespokeMessage(targlist, messagefile, subj):
+
+ messagecontent = open(messagefile, 'r').read()
+ targets = open(targlist, 'r').readlines()
+ camowners = loadLocationDetails()
+ camowners['location'] = [ x.lower() + '_' + y.lower() for x,y in zip(camowners.site,camowners.direction)]
+ camowners = camowners[camowners.active==1]
+ camowners.drop(columns=['oldcode', 'site', 'camtype', 'active', 'humanName','direction','created'], inplace=True)
+ #camowners = camowners.groupby('eMail', as_index=False).agg(lambda x: ','.join(x.tolist()))
+ owners = []
+ camids = []
+ gmnids = []
+ for target in targets:
+ locationid = target.strip()
+ if locationid[:1] == '#':
+ continue
+ thisrow = camowners[camowners.location.str.contains(locationid)]
+ if len(thisrow) > 0:
+ thisowner = thisrow.values[0][0]
+ if thisowner in owners:
+ idx = owners.index(thisowner)
+ gmnids[idx] = f'{gmnids[idx]},{thisrow.values[0][1]}'
+ camids[idx] = f'{camids[idx]},{thisrow.values[0][2]}'
+ continue
+ else:
+ owners.append(thisowner)
+ gmnids.append(thisrow.values[0][1])
+ camids.append(thisrow.values[0][2])
+ df = pd.DataFrame({'owner':owners, 'camids':camids, 'gmnids':gmnids})
+ print(df)
+ for _,rw in df.iterrows():
+ mailrecip = rw.owner
+ statid = rw.gmnids
+ locationid = rw.camids
+ print(f'emailing {mailrecip} concerning {statid} {locationid} subject {subj}')
+ sendAnEmail(mailrecip, messagecontent.format(statid, locationid), subj, mailfrom)
+ return
+
+
if __name__ == '__main__':
if len(sys.argv) < 2:
print('usage: python reportBadCameras.py daysback')
diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh
index 385fb0353..d76dc602d 100644
--- a/archive/website/cameraStatusReport.sh
+++ b/archive/website/cameraStatusReport.sh
@@ -36,11 +36,11 @@ else
authlog=/var/log/secure
batchuser=ec2-user
fi
-sudo grep publickey $authlog* | grep -v $batchuser > $DATADIR/reports/lastlogins.txt
+sudo grep publickey $authlog* | grep -v $batchuser | grep -v fatal > $DATADIR/reports/lastlogins.txt
if [ "$platform" == "Ubuntu" ]
then
- ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | grep -v ec2-user" > $DATADIR/reports/lastlogins_old.txt
+ ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | egrep -v 'ec2-user|fatal'" > $DATADIR/reports/lastlogins_old.txt
fi
cat $DATADIR/reports/lastlogins_old.txt >> $DATADIR/reports/lastlogins.txt
echo TO HERE
diff --git a/bumpver.toml b/bumpver.toml
index 17d501fc7..f8fc75bba 100644
--- a/bumpver.toml
+++ b/bumpver.toml
@@ -1,5 +1,5 @@
[bumpver]
-current_version = "2026.05.0"
+current_version = "2026.05.3"
version_pattern = "YYYY.0M.PATCH"
commit_message = "bump version {old_version} -> {new_version}"
commit = true
diff --git a/install_or_update.sh b/install_or_update.sh
index 83713e346..c903adce7 100755
--- a/install_or_update.sh
+++ b/install_or_update.sh
@@ -10,8 +10,8 @@ envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]')
here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+pushd $here/archive
echo "Updating codebase..."
-cd $here/archive
git pull
echo "Updating code..."
@@ -32,6 +32,11 @@ mkdir -p $DATADIR/browse/{annual,monthly,daily,showers}
mkdir -p ~/$envname/logs
mkdir -p ~/.logrotate
mkdir -p ~/.aws
+mkdir -p ~/server_setup
+
+rsync -a server_setup/*.sh ~/server_setup
+rsync -a server_setup/*.py ~/server_setup
+chmod +x ~/server_setup/*.sh
echo "Checking conda environment..."
if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]]
@@ -69,4 +74,5 @@ else
echo skipping config and bashrc
fi
echo ""
+popd
echo "$msg complete"