reverse merge master into dev

.github/workflows/codeql-analysis.yml

@@ -39,11 +39,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v4
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v4
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v4

README.md

@@ -1,5 +1,5 @@
 # UK Meteor Data Analysis Shared code and libraries
-version: 2026.05.0
+version: 2026.05.3
 
 This repository contains the code behind the UK Meteors data archive and data processing pipeline. 
 

api_examples/README.md

@@ -0,0 +1,6 @@
+#Available APIs and Examples
+
+We have a number of APIs which can be used to retrieve our data programatically, either using commandline tools like curl and wget, or from programming languages such as Python. 
+
+Data are returned in JSON format which can readily be converted into a python list, array or Pandas DataFrame. Simple examples are shown on our main website (here)[https://ukmeteornetwork.org/our-data-apis/], and a set of Python examples are available here. A Data Dictionary in Excel format can be downloaded from (here)[https://archive.ukmeteors.co.uk/browse/datadictionary.xlsx].
+

archive/README.md

@@ -1,7 +1,7 @@
 Data Processing and Flows
 ==========================
 
-version: 2026.05.0
+version: 2026.05.3
 
 This diagram shows the overall flow of data from Cameras to websites and out to the public.
 

archive/analysis/getLogData.sh

@@ -62,7 +62,7 @@ echo "the entire night was uncalibrated (indicated by the 'skipping' warning).</
 echo "<p>To interpret these data, see " >> $logfile
 echo "<a href=https://github.com/markmac99/ukmon-pitools/wiki/UKMON-Pi-Toolset-FAQ#Interpreting_the_Uncalibrated_Report>here</a></p>" >> $logfile
 echo "<pre>" >> $logfile
-grep "Skipping" $lastmtch >> $logfile
+grep "Skipping" $lastmtch | egrep -v "directory|pickle" >> $logfile
 echo "</pre>" >> $logfile
 
 echo "<h2 id=fails>Failed Match Report</h2>" >> $logfile
@@ -72,6 +72,7 @@ echo "<a href=https://github.com/markmac99/ukmon-pitools/wiki/Trajectory-Solver-
 echo "<pre>" >> $logfile
 cat $DATADIR/failed/${rundate}_failed.txt >> $logfile
 echo "</pre>" >> $logfile
+
 echo "<h2 id=graph>Chart of Batch Element Runtimes</h2>" >> $logfile
 echo "<p>This section shows the runtime of the different jobs in the batch</p>" >> $logfile
 echo "<p><a href=/reports/batchcharts/$rundate-perfNightly.jpg><img src=/reports/batchcharts/$rundate-perfNightly.jpg width=300\%></a></p>"  >> $logfile
@@ -93,4 +94,7 @@ done
 cat $TEMPLATES/footer.html >> $DATADIR/lastlogs/index.html
 aws s3 cp $DATADIR/lastlogs/index.html  $WEBSITEBUCKET/reports/lastlogs/ --quiet
 
+# housekeep the fails reports
+find $DATADIR/failed -mtime +90 -exec rm -f {} \;
+
 find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \;

archive/lambdas/matchDataApi/matchDataApi.py

@@ -67,7 +67,7 @@ def getStationData(statid, dtstr, period=None):
         statfrag = f"and s.stations like '%{statid}%' " if statid is not None else ""
         perfrag = periodToSqlFragment(period) if period is not None else ""
         with connection.cursor() as cursor:
-            sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}"
+            sql = f"SELECT s.orbname from matches s where s.orbname like '{dtstr}%' {statfrag} {perfrag}"
             cursor.execute(sql)
             result = cursor.fetchall()
     finally:
@@ -85,7 +85,7 @@ def getSummaryData(dtstr, period=None):
     fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\
         '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\
         'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations'
-    expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%' {perfrag}"
+    expr = f"SELECT {fieldlist} from matches s where s.orbname like '{dtstr}%' {perfrag}"
     result=[]
     try:
         with connection.cursor() as cursor:
@@ -171,7 +171,7 @@ def lambda_handler(event, context):
                 res = '{"points": "unavailable"}'
 
     else:
-        res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}'
+        res = '{"invalid request type - must be one of \'matches\', \'detail\', \'station\',\'summary\'"}'
 
     print(res)
     return {

archive/server_setup/.bashrc

@@ -55,9 +55,12 @@ fi
 unset __conda_setup
 # <<< conda initialize <<<
 
+# make the function keys work properly in python
+export TERM=putty
+
 # If this is an xterm set the title to user@host:dir
 case "$TERM" in
- xterm*|rxvt*)
+ xterm*|rxvt*|putty*)
     PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
     ;;
  *)

archive/server_setup/acctStatusChecks.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+cd $here
+
+sudo ls -1 /var/sftp  | egrep -v "test|logup" > moved.txt
+grep batchs /home/ubuntu/prod/data/reports/stationlogins.txt | cut -c 42-61 | sed 's/ //g' | sort | uniq > switched.txt
+grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt | cut -c 42-61 | sed 's/ //g' | sort | uniq > pending.txt
+
+ssh ukmonhelper2 "~/prod/server_setup/get-nbd.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > not-being-done.txt
+ssh ukmonhelper2 "~/prod/server_setup/get-all.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > all-accounts.txt
+
+python ~/src/ukmda-dataprocessing/archive/server_setup/checkSftpAccounts.py
+
+
+echo "Moved:    $(wc -l moved.txt | awk '{print $1}')" > statusreport.txt
+echo "Switched: $(wc -l switched.txt | awk '{print $1}')" >> statusreport.txt
+echo "Pending:  $(wc -l pending.txt | awk '{print $1}')" >> statusreport.txt
+echo "Total:    $(wc -l still-live.txt | awk '{print $1}')" >> statusreport.txt
+echo "" >> statusreport.txt
+echo "Not Live: $(wc -l not_live.txt | awk '{print $1}')" >> statusreport.txt
+echo "Not Upl:  $(wc -l not_uploading.txt | awk '{print $1}')" >> statusreport.txt
+echo "" >> statusreport.txt
+echo "Dead:     $(wc -l not-being-done.txt | awk '{print $1}')" >> statusreport.txt
+echo "" >> statusreport.txt
+cat statusreport.txt

archive/server_setup/backupSftpAccts.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+cd $here
+
+bkpOneUser() {
+    userid=$1
+    srchost=$2
+    mkdir -p ./backup/$userid
+    sudo rsync -av $srchost:/var/sftp/$userid/ ./backup/$userid 
+    sudo chown -R $USER:$USER ./backup/$userid
+    cat ./backup/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini
+    mv -f /tmp/$userid.ini ./backup/$userid/ukmon.ini
+}
+
+if [ $# -lt 2 ] ; then 
+    echo "Usage: ./backupSftpAccounts.sh oldservername userfile"
+    exit
+fi 
+
+echo "Warning: this must only be run on the new server"
+read -p "press ctrl-c to quit or enter to continue"
+
+oldserver=$1
+srcfile=$2
+
+cat $srcfile | while read stn
+do 
+    bkpOneUser $stn $oldserver
+done

archive/server_setup/checkSftpAccounts.py

@@ -0,0 +1,75 @@
+import pandas as pd
+import datetime
+
+
+data=open('/home/ubuntu/prod/data/reports/stationlogins.txt').readlines()
+camlist = []
+not_live = []
+still_upl = []
+not_upl = []
+livenames = []
+
+livedate = datetime.datetime.now() - datetime.timedelta(days=10)
+for li in data:
+	if 'Last Upload' in li:
+		continue
+	lastup = li[:19]
+	lastlo = li[21:40]
+	loc = li[42:61]
+	gmnid = li[64:73]
+	via = li[76:].strip()
+	camlist.append([lastup, lastlo, loc, gmnid, via])
+
+	if ">" in lastup and ">" in lastlo:
+		not_live.append([loc, gmnid, lastup, lastlo, via])
+
+	elif ">" not in lastup:
+		lastupdt = datetime.datetime.strptime(lastup, '%Y-%m-%dT%H:%M:%S')
+		if lastupdt >= livedate:
+			still_upl.append([loc, gmnid, lastup, lastlo, via])
+			livenames.append(loc.strip())
+		else:
+			if '>' in lastlo:
+				not_live.append([loc, gmnid, lastup, lastlo, via])
+			else:
+				lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S')
+				if lastlodt >= livedate:
+					not_upl.append([loc, gmnid, lastup, lastlo, via])
+				else:
+					not_live.append([loc, gmnid, lastup, lastlo, via])
+	else: # '>' not in lastlo
+		lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S')
+		if lastlodt >= livedate:
+			not_upl.append([loc, gmnid, lastup, lastlo, via])
+		else:
+			not_live.append([loc, gmnid, lastup, lastlo, via])
+
+with open('still-live.txt','w') as outf:
+	for cam in still_upl:
+		outf.write(','.join(cam) + '\n')
+
+with open('not_uploading.txt','w') as outf:
+	for cam in not_upl:
+		outf.write(','.join(cam) + '\n')
+
+with open('not_live.txt','w') as outf:
+	for cam in not_live:
+		outf.write(','.join(cam) + '\n')
+
+donelist = open('moved.txt', 'r').readlines()
+donelist = [x.strip() for x in donelist]
+with open('todo.txt', 'w') as outf:
+	for nam in livenames:
+		if nam.strip() not in donelist:
+			print('done list is missing', nam)
+			outf.write(f'{nam}\n')
+
+
+moved = open('moved.txt').readlines()
+pending = open('pending.txt').readlines()
+switched = open('switched.txt').readlines()
+
+pendingnotmoved = [x for x in pending if x not in moved]
+
+print('stations marked pending but not moved - check if need done')
+print(pendingnotmoved)

archive/server_setup/migrateSftpAccts.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+cd $here
 
 addOneUser() {
     userid=$1

archive/terraform/ukmda/batchserver.tf

@@ -45,3 +45,11 @@ resource "aws_network_interface" "batchserver_if" {
     "billingtag" = "ukmda"
   }
 }
+
+resource "aws_eip" "batchserver" {
+  instance = aws_instance.batchserver.id
+  tags = {
+    billingtag = "ukmda"
+    Name       = "ukmonhelper_eip"
+  }
+}

archive/ukmon_pylib/metrics/camMetrics.py

@@ -206,15 +206,15 @@ def backPopulate(stationid):
         df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0")
         df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime
         df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload]
-        df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime'])
+        df = df.drop(columns=['manual','rundate', 'upddate','uploadtime'])
         df = df.sort_values(by=['lastupload'])
 
         outfile=os.path.join(datadir, 'reports', 'stationlogins.txt')
         zerodate = datetime.datetime(1970,1,1,0,0,0)
         with open(outfile,'w') as outf:
-            outf.write('Last Upload,          StationID,            Last Login,          Via\n')
+            outf.write(f'{"Last Upload":19s}  {"Last Connect":19s}  {"StationID":20s}  {"GMN ID":10s}  Via\n')
             for _,rw in df.iterrows():
-                lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S')
-                lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S')
-                via = '' if pd.isnull(rw.host) else rw.host
-                outf.write(f'{lastup} , {rw.location:20s}, {lastlo}, {via}\n')
+                lastup = '> 1 month ago' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S')
+                lastlo = '> 1 month ago' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S')
+                via = 'ukmonhelper2' if pd.isnull(rw.host) else rw.host
+                outf.write(f'{lastup:19s}  {lastlo:19s}  {rw.location:20s}  {rw.stationid:10s}  {via}\n')