Latest patch release

README.md

@@ -1,5 +1,5 @@
 # UK Meteor Data Analysis Shared code and libraries
-version: 2026.05.2
+version: 2026.05.3
 
 This repository contains the code behind the UK Meteors data archive and data processing pipeline. 
 

archive/README.md

@@ -1,7 +1,7 @@
 Data Processing and Flows
 ==========================
 
-version: 2026.05.2
+version: 2026.05.3
 
 This diagram shows the overall flow of data from Cameras to websites and out to the public.
 

archive/analysis/getLogData.sh

@@ -62,7 +62,7 @@ echo "the entire night was uncalibrated (indicated by the 'skipping' warning).</
 echo "<p>To interpret these data, see " >> $logfile
 echo "<a href=https://github.com/markmac99/ukmon-pitools/wiki/UKMON-Pi-Toolset-FAQ#Interpreting_the_Uncalibrated_Report>here</a></p>" >> $logfile
 echo "<pre>" >> $logfile
-grep "Skipping" $lastmtch >> $logfile
+grep "Skipping" $lastmtch | egrep -v "directory|pickle" >> $logfile
 echo "</pre>" >> $logfile
 
 echo "<h2 id=fails>Failed Match Report</h2>" >> $logfile
@@ -72,6 +72,7 @@ echo "<a href=https://github.com/markmac99/ukmon-pitools/wiki/Trajectory-Solver-
 echo "<pre>" >> $logfile
 cat $DATADIR/failed/${rundate}_failed.txt >> $logfile
 echo "</pre>" >> $logfile
+
 echo "<h2 id=graph>Chart of Batch Element Runtimes</h2>" >> $logfile
 echo "<p>This section shows the runtime of the different jobs in the batch</p>" >> $logfile
 echo "<p><a href=/reports/batchcharts/$rundate-perfNightly.jpg><img src=/reports/batchcharts/$rundate-perfNightly.jpg width=300\%></a></p>"  >> $logfile
@@ -93,4 +94,7 @@ done
 cat $TEMPLATES/footer.html >> $DATADIR/lastlogs/index.html
 aws s3 cp $DATADIR/lastlogs/index.html  $WEBSITEBUCKET/reports/lastlogs/ --quiet
 
+# housekeep the fails reports
+find $DATADIR/failed -mtime +90 -exec rm -f {} \;
+
 find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \;

archive/server_setup/.bashrc

@@ -55,9 +55,12 @@ fi
 unset __conda_setup
 # <<< conda initialize <<<
 
+# make the function keys work properly in python
+export TERM=putty
+
 # If this is an xterm set the title to user@host:dir
 case "$TERM" in
- xterm*|rxvt*)
+ xterm*|rxvt*|putty*)
     PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
     ;;
  *)

archive/server_setup/acctStatusChecks.sh

@@ -4,22 +4,24 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 
 cd $here
 
-sudo ls -1 /var/sftp > moved.txt
-grep batchs /home/ubuntu/prod/data/reports/stationlogins.txt |awk '{print $3}'| sort > switched.txt
-grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt |awk '{print $3}'| sort > pending.txt
+sudo ls -1 /var/sftp  | egrep -v "test|logup" > moved.txt
+grep batchs /home/ubuntu/prod/data/reports/stationlogins.txt | cut -c 42-61 | sed 's/ //g' | sort | uniq > switched.txt
+grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt | cut -c 42-61 | sed 's/ //g' | sort | uniq > pending.txt
 
 ssh ukmonhelper2 "~/prod/server_setup/get-nbd.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > not-being-done.txt
 ssh ukmonhelper2 "~/prod/server_setup/get-all.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > all-accounts.txt
 
 python ~/src/ukmda-dataprocessing/archive/server_setup/checkSftpAccounts.py
 
-echo "Moved:    $(wc -l moved.txt | awk '{print $1}')"
-echo "Switched: $(wc -l switched.txt | awk '{print $1}')"
-echo "Pending:  $(wc -l pending.txt | awk '{print $1}')"
-echo "Total:    $(wc -l still-live.txt | awk '{print $1}')"
-echo ""
-echo "Not Live: $(wc -l not_live.txt | awk '{print $1}')"
-echo "Not Upl:  $(wc -l not_uploading.txt | awk '{print $1}')"
-echo ""
-echo "Dead:     $(wc -l not-being-done.txt | awk '{print $1}')"
-echo ""
+
+echo "Moved:    $(wc -l moved.txt | awk '{print $1}')" > statusreport.txt
+echo "Switched: $(wc -l switched.txt | awk '{print $1}')" >> statusreport.txt
+echo "Pending:  $(wc -l pending.txt | awk '{print $1}')" >> statusreport.txt
+echo "Total:    $(wc -l still-live.txt | awk '{print $1}')" >> statusreport.txt
+echo "" >> statusreport.txt
+echo "Not Live: $(wc -l not_live.txt | awk '{print $1}')" >> statusreport.txt
+echo "Not Upl:  $(wc -l not_uploading.txt | awk '{print $1}')" >> statusreport.txt
+echo "" >> statusreport.txt
+echo "Dead:     $(wc -l not-being-done.txt | awk '{print $1}')" >> statusreport.txt
+echo "" >> statusreport.txt
+cat statusreport.txt

archive/server_setup/checkSftpAccounts.py

@@ -62,4 +62,14 @@
 	for nam in livenames:
 		if nam.strip() not in donelist:
 			print('done list is missing', nam)
-			outf.write(f'{nam}\n')
+			outf.write(f'{nam}\n')
+
+
+moved = open('moved.txt').readlines()
+pending = open('pending.txt').readlines()
+switched = open('switched.txt').readlines()
+
+pendingnotmoved = [x for x in pending if x not in moved]
+
+print('stations marked pending but not moved - check if need done')
+print(pendingnotmoved)

archive/terraform/ukmda/batchserver.tf

@@ -45,3 +45,11 @@ resource "aws_network_interface" "batchserver_if" {
     "billingtag" = "ukmda"
   }
 }
+
+resource "aws_eip" "batchserver" {
+  instance = aws_instance.batchserver.id
+  tags = {
+    billingtag = "ukmda"
+    Name       = "ukmonhelper_eip"
+  }
+}

archive/ukmon_pylib/metrics/camMetrics.py

@@ -216,5 +216,5 @@ def backPopulate(stationid):
             for _,rw in df.iterrows():
                 lastup = '> 1 month ago' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S')
                 lastlo = '> 1 month ago' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S')
-                via = '' if pd.isnull(rw.host) else rw.host
+                via = 'ukmonhelper2' if pd.isnull(rw.host) else rw.host
                 outf.write(f'{lastup:19s}  {lastlo:19s}  {rw.location:20s}  {rw.stationid:10s}  {via}\n')

archive/ukmon_pylib/metrics/timingMetrics.py

@@ -12,6 +12,7 @@
 
 
 def graphOfData(logf, dtstr):
+
     lis = open(logf,'r').readlines()
     dta = [li for li in lis if li[:8]==dtstr]
 
@@ -55,78 +56,35 @@ def graphOfData(logf, dtstr):
 
 
 def getLogStats(nightlogf, matchlogf, thisdy):
-    with open(nightlogf,'r') as inf:
-        loglines = inf.readlines()
-
-    #logf = os.path.basename(nightlogf)
-    #spls = logf.split('-')
-    #thisdy= spls[1]
-    matchstr = 'RUNTIME'    
-
-    # <13>Nov  1 11:49:55 nightlyJob: RUNTIME 10554 showerReport ALL 202211
-    startFAM = 0
-    startRD = 0
-    dts = []
-    tss = []
-    tsks = []
-    secs = []
-    msgs = []
-    for li in loglines:
-        if matchstr in li:
-            li = ' '.join(li.split()) # remove double-spaces eg when date is ' 6 Jan' as opposed to '16 Jan'
-            spls = li.split()
-            msg = ''
-            for s in range(6, len(spls)):
-                msg = msg + ' ' + spls[s]
-            if 'start findAllMatches' in msg:
-                startFAM = int(spls[5])
-            dts.append(thisdy)
-            tss.append(spls[2])
-            tsks.append(spls[3].replace(':',''))
-            secs.append(int(spls[5]))
-            msgs.append(msg.strip())
-
-    with open(matchlogf,'r') as inf:
-        loglines = inf.readlines()
+    # logline example
+    # <13>May  8 06:10:02 nightlyJob: start nightlyJob
+    outdir = os.path.split(nightlogf)[0]
 
-    # scan for findAllMatches data first
-    for li in loglines:
-        if matchstr in li:
-            spls = li.split()
-            if 'runDistrib' in spls[3]: 
-                continue
-            msg = ''
-            for s in range(6, len(spls)):
-                msg = msg + ' ' + spls[s]
-            if 'start runDistrib' in msg:
-                startRD = int(spls[5])
-            dts.append(thisdy)
-            tss.append(spls[2])
-            tsks.append(spls[3].replace(':',''))
-            secs.append(int(spls[5])+startFAM)
-            msgs.append(msg.strip())
+    loglines = open(nightlogf,'r').readlines()
+    bsfs = [x for x in loglines if 'start'in x or 'finish' in x or 'end' in x]
+    bsfs = [x for x in bsfs if x[0]=='<']
 
-    # rescan for runDistrib data
-    for li in loglines:
-        if matchstr in li:
-            spls = li.split()
-            if 'runDistrib' not in spls[3]: 
-                continue
-            msg = ''
-            for s in range(6, len(spls)):
-                msg = msg + ' ' + spls[s]
-            dts.append(thisdy)
-            tss.append(spls[2])
-            tsks.append(spls[3].replace(':',''))
-            secs.append(int(spls[5]) + startFAM + startRD)
-            msgs.append(msg.strip())
+    loglines = open(matchlogf,'r').readlines()
+    msfs = [x for x in loglines if 'start'in x or 'finish' in x or 'end' in x]
+    msfs = [x for x in msfs if x[0]=='<']
 
-    df = pd.DataFrame(zip(dts,tss,secs,tsks,msgs), columns=['dts','tss','secs','tsk','msgs'])
-    df = df.sort_values(by=['tss'])
-    #print(df)
-    outdir = os.path.split(nightlogf)[0]
-    df.to_csv(os.path.join(outdir, 'perfNightly.csv'), mode='a', header=False, index=False)
-
+    alldata = msfs + bsfs
+    alldata.sort()
+    starttime = None
+    yr = datetime.datetime.now().year
+    with open(os.path.join(outdir, 'perfNightly.csv'), 'a+') as outf:
+        for rw in alldata:
+            dtpart = rw[4:19]
+            evtdt = datetime.datetime.strptime(f'{yr} {dtpart}', '%Y %b %d %H:%M:%S')
+            if 'start nightlyJob' in rw:
+                starttime = evtdt
+            elapsed_secs = (evtdt - starttime).seconds
+            txtpart = rw[20:]
+            task = txtpart[:txtpart.find(':')]
+            msg = txtpart[txtpart.find(':')+1:].replace(',', ' ')
+            outstr = f'{evtdt.strftime("%Y%m%d,%H:%M:%S")},{elapsed_secs},{task},{msg}'
+            outf.write(outstr)        
+
 
 if __name__ == '__main__':
     dtstr = sys.argv[1]

archive/ukmon_pylib/reports/findFailedMatches.py

@@ -16,24 +16,27 @@ def saveData(observation, repfile):
 
 
 def processOneLog(logfile, repfile):
-    with open(logfile, 'r') as inf:
-        lis = inf.readlines()
+    lis = open(logfile, 'r').readlines()
 
     obs = False
     observation = []
     for li in lis:
-        if 'Observations:' in li:
+        if 'Observations ' in li:
             obs = True
         if obs is True:
             observation.append(li)
-        if 'Shower:' in li:
+        if 'Saving trajectory:' in li or 'Updating database' in li:
             obs = False
             observation = []
-        if '-----------------------' in li:
+        if 'Updating database' in li or 'added to fails' in li:
             if obs is True:
+                #print('got fail')
+                observation.append('--------------\n\n')
                 saveData(observation, repfile)
             obs = False
             observation = []
+        if "SOLVING RUN DONE" in li:
+            break
 
     return
 
@@ -48,5 +51,6 @@ def processOneLog(logfile, repfile):
     logdir = os.path.join(srcdir, 'logs','distrib')
     logs = glob.glob(f'{repdt}*.log', root_dir=logdir)
     for logf in logs:
+        #print(logf)
         processOneLog(os.path.join(logdir, logf), reportfile)
     reportfile.close()

archive/ukmon_pylib/reports/getSolutionStati.py

@@ -5,93 +5,14 @@
 import sys
 
 
-def getSolutionStati(fname):
-    with open(fname, 'r') as inf:
-        lis = inf.readlines()
-    # find start of interesting data
-    totnew = 0
-    totchk = 0
-    gotnew = 0
-    gotchk = 0
-    chktraj = False
-    newtraj = False
-    for li in lis:
-        # check for new data for existing solns 
-        if 'Checking trajectory at ' in li and chktraj is False:
-            thistraj = li
-            chktraj = True
-            totchk += 1
-            continue
-        if 'Checking trajectory at ' in li and chktraj is True:
-            spls = thistraj.strip().split(' ')
-            print(f'{spls[3]} {spls[4]} no new data')
-            totchk += 1
-            thistraj = li
-            chktraj = True
-            continue
-        if "New Trajectory solution failed" in li and chktraj is True:
-            spls = thistraj.strip().split(' ')
-            print(f'{spls[3]} {spls[4]} no improvement')
-            chktraj = False
-            continue
-        if "Shower:" in li and chktraj is True:
-            spls = thistraj.strip().split(' ')
-            spl2 = li.strip().split(' ')
-            try:
-                if spl2[3] == '...':
-                    shwr = 'Spo'
-                else:
-                    shwr = spl2[3]
-            except:
-                if spl2[1] == '...':
-                    shwr = 'Spo'
-                else:
-                    shwr = spl2[1]
-
-            print(f'{spls[3]} {spls[4]} updated as {shwr}')
-            gotchk += 1
-            chktraj = False
-            continue
-
-        # check for new groups
-        if "Observations:" in li:
-            thistraj = None
-            newtraj = True
-            totnew += 1
-            continue
-        if "Observations:" not in li and newtraj is True and thistraj is None:
-            thistraj = li
-            continue
-        if "-------" in li and newtraj is True:
-            spls = thistraj.strip().split(' ')
-            print(f'{spls[3]} {spls[4]} not matched')
-            newtraj = False
-            thistraj = None
-            continue
-        if "Shower:" in li and newtraj is True:
-            spls = thistraj.strip().split(' ')
-            spl2 = li.strip().split(' ')
-            try: 
-                if spl2[3] == '...':
-                    shwr = 'Spo'
-                else:
-                    shwr = spl2[3]
-            except:
-                if spl2[1] == '...':
-                    shwr = 'Spo'
-                else:
-                    shwr = spl2[1]
-            print(f'{spls[3]} {spls[4]} solved as {shwr}')
-            gotnew += 1
-            newtraj = False
-            thistraj = None
-            continue
-        if "SOLVING RUN DONE" in li:
-            break
-
-    return totnew, totchk, gotnew, gotchk 
+def parseDistriblog(logname):
+    lis = open(logname,'r').readlines()
+    dta = [x for x in lis if ('added to fails' in x or ('saved' in x and 'to' in x)) and ('CorrelateEngine' in x or 'CorrelateRMS' in x)]
+    offset = 49 if 'match' in logname else 76
+    for dd in dta:
+        print(dd[offset:].strip())
 
 
 if __name__ == '__main__':
     fname = sys.argv[1]
-    print(getSolutionStati(fname))
+    parseDistriblog(fname)