Minor tweaks

README.md

@@ -1,5 +1,5 @@
 # UK Meteor Data Analysis Shared code and libraries
-version: 2026.04.1
+version: 2026.05.0
 
 This repository contains the code behind the UK Meteors data archive and data processing pipeline. 
 

api_examples/README.md

@@ -0,0 +1,6 @@
+#Available APIs and Examples
+
+We have a number of APIs which can be used to retrieve our data programatically, either using commandline tools like curl and wget, or from programming languages such as Python. 
+
+Data are returned in JSON format which can readily be converted into a python list, array or Pandas DataFrame. Simple examples are shown on our main website (here)[https://ukmeteornetwork.org/our-data-apis/], and a set of Python examples are available here. A Data Dictionary in Excel format can be downloaded from (here)[https://archive.ukmeteors.co.uk/browse/datadictionary.xlsx].
+

archive/README.md

@@ -1,7 +1,7 @@
 Data Processing and Flows
 ==========================
 
-version: 2026.04.1
+version: 2026.05.0
 
 This diagram shows the overall flow of data from Cameras to websites and out to the public.
 

archive/analysis/findAllMatches.sh

@@ -23,12 +23,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 source $here/../config.ini >/dev/null 2>&1
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
 
-# logstream name inherited from parent environment but set it if not
-if [ "$NJLOGSTREAM" == "" ]; then
-    NJLOGSTREAM=$(date +%Y%m%d-%H%M%S)
-    aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared
-fi
-log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" findAllMatches
+logger -s -t findAllMatches "starting"
 
 [ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d)
 
@@ -52,40 +47,39 @@ mkdir -p $SRC/logs/distrib > /dev/null 2>&1
 
 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000')
 enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000')
-log2cw $NJLOGGRP $NJLOGSTREAM "solving for ${startdt} to ${enddt}" findAllMatches
 
-log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches
-$SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND
+logger -s -t findAllMatches "solving for ${startdt} to ${enddt}"
+logger -s -t findAllMatches "start runDistrib"
 
-log2cw $NJLOGGRP $NJLOGSTREAM "clean duplicate/deleted trajs" findAllMatches
+$SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND
 $SRC/utils/cleanupDeletedTrajs.sh
 
-log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches
+logger -s -t findAllMatches "Solving Run Done" 
+
 success=$(grep "Total run time:" $SRC/logs/matchJob.log)
 
 if [ "$success" == "" ]
 then
-    python -c "from meteortools.utils import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')"
+    python -c "from utils.sendAnEmail import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')"
     echo problems with solver
 fi
-log2cw $NJLOGGRP $NJLOGSTREAM "Solving Run Done" findAllMatches
 
-log2cw $NJLOGGRP $NJLOGSTREAM "start rerunFailedLambdas" findAllMatches
 python -m maintenance.rerunFailedLambdas
 
 cd $here
-log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches
+
+logger -s -t findAllMatches "start reportOfLatestMatches" 
+
 matchlog=${SRC}/logs/matchJob.log
-python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate
-python -m metrics.getMatchStats $matchlog
+python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $rundate
+python -m metrics.getMatchStats $matchlog $rundate
 
 # copy stats to S3 so the daily report can run
 if [ "$RUNTIME_ENV" == "PROD" ] ; then 
     aws s3 sync $DATADIR/dailyreports/ $UKMONSHAREDBUCKET/matches/RMSCorrelate/dailyreports/ --quiet
 fi 
 
-log2cw $NJLOGGRP $NJLOGSTREAM "start purgeLogs" findAllMatches
 find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \;
 find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \;
 
-log2cw $NJLOGGRP $NJLOGSTREAM "finished findAllMatches" findAllMatches
+logger -s -t findAllMatches "finished"

archive/analysis/getLogData.sh

@@ -10,7 +10,7 @@ if [ "$1" != "" ] ; then
     logfile=$DATADIR/lastlogs/lastlog-${rundate}.html
 else
     rundate=$(date +%Y%m%d)
-    logfile=$DATADIR/lastlog.html
+    logfile=$DATADIR/lastlogs/lastlog.html
 fi 
 
 # create performance metrics
@@ -78,7 +78,7 @@ echo "<p><a href=/reports/batchcharts/$rundate-perfNightly.jpg><img src=/reports
 
 cat $TEMPLATES/footer.html >> $logfile
 
-if [ ! -d $DATADIR/lastlogs ] ; then mkdir $DATADIR/lastlogs ; fi
+mkdir -p $DATADIR/lastlogs
 if [ "$1" == "" ] ; then 
     aws s3 cp $logfile  $WEBSITEBUCKET/reports/ --quiet
     cp $logfile $DATADIR/lastlogs/lastlog-${rundate}.html
@@ -92,3 +92,5 @@ ls -1r $DATADIR/lastlogs/last*.html  | head -90 | while read i ; do
 done
 cat $TEMPLATES/footer.html >> $DATADIR/lastlogs/index.html
 aws s3 cp $DATADIR/lastlogs/index.html  $WEBSITEBUCKET/reports/lastlogs/ --quiet
+
+find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \;

archive/analysis/reportActiveShowers.sh

@@ -30,7 +30,7 @@ fi
 logger -s -t reportActiveShowers "report on active showers"
 python -m reports.reportActiveShowers -m
 
-python -c "from meteortools.utils import getActiveShowers; getActiveShowers('$rundt', inclMinor=True)" | while read shwr
+python -c "from utils.getActiveShowers import getActiveShowers;getActiveShowers('$rundt', inclMinor=True)" | while read shwr
 do 
     aws s3 sync $DATADIR/reports/${yr}/$shwr $WEBSITEBUCKET/reports/${yr}/${shwr} --quiet 
 done

archive/analysis/runDistrib.sh

@@ -20,129 +20,110 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 # load the configuration
 source $here/../config.ini >/dev/null 2>&1
 
-# logstream name inherited from parent environment but set it if not
-if [ "$NJLOGSTREAM" == "" ]; then
-    NJLOGSTREAM=$(date +%Y%m%d-%H%M%S)
-    aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared
-fi
-log2cw $NJLOGGRP $NJLOGSTREAM "starting runDistrib" runDistrib
-
-# set the profile to the UKMDA account so we can run the server and monitor progress
-export AWS_PROFILE=ukmonshared
+logger -s -t runDistrib "starting runDistrib" 
 
 if [ $# -gt 0 ] ; then
     if [ "$1" != "" ] ; then
-        log2cw $NJLOGGRP $NJLOGSTREAM "selecting range" runDistrib
         MATCHSTART=$1
     fi
     if [ "$2" != "" ] ; then
         MATCHEND=$2
     else
-        log2cw $NJLOGGRP $NJLOGSTREAM "matchend was not supplied, using 2 days" runDistrib
         MATCHEND=$(( $MATCHSTART - 2 ))
     fi
 fi
 begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d')
 rundate=$(date --date="-$MATCHEND days" '+%Y%m%d')
 
-log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" runDistrib
+logger -s -t runDistrib "running phase 1 for dates ${begdate} to ${rundate}"
 
+logger -s -t runDistrib  "start correlation server" 
 aws ec2 start-instances --instance-ids $SERVERINSTANCEID
 stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text)
 while [ "$stat" -ne 16 ]; do
     sleep 5
-    log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" runDistrib
     stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text)
 done
 
-log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate}" runDistrib
-
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
 
-log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" runDistrib
+logger -s -t runDistrib "creating the run script"
 
 execdist=execdistrib.sh
 execMatchingsh=/tmp/$execdist
 python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE
 chmod +x $execMatchingsh
 
-log2cw $NJLOGGRP $NJLOGSTREAM "get server details" runDistrib
-privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text)
-while [ "$privip" == "" ] ; do
-    sleep 5
-    log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" runDistrib
-    privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text)
-done
-
-log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" runDistrib
+logger -s -t runDistrib "deploy the script to the server $CALCSERVERIP and run it"
 
-scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist
+scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist
 while [ $? -ne 0 ] ; do
     # in case the server isn't responding to ssh sessions yet
     sleep 10
-    log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib
-    scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist
+    scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist
 done 
 # push the python code and ECS templates required
-rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$privip:src/ukmon_pylib/traj
-rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$privip:src/ukmon_pylib/traj
-rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$privip:src/ukmon_pylib/traj
-rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$privip:src/ukmon_pylib/traj
-rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj
+rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/
+rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/
+rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/
+rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/
+rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/
+rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/ShowerAssociation.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/
+rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/utils/convertSolLon.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/utils/
 
 # now run the script
-log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib
-ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist"
+logger -s -t runDistrib "start distributed processing"
+ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execdist"
 
-rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates
+rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates
 
-rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/
-ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;"
+rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/
+ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;"
 
-log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib
+logger -s -t runDistrib "job run, stop the server again"
 aws ec2 stop-instances --instance-ids $SERVERINSTANCEID
 
-log2cw $NJLOGGRP $NJLOGSTREAM "monitoring and waiting for completion" runDistrib
+logger -s -t runDistrib "monitoring and waiting for completion"
 
 python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}', '${TESTMODE}'); "
 
 mkdir -p $DATADIR/distrib
 cd $DATADIR/distrib
 
-log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib
+logger -s -t runDistrib "restarting server to consolidate results"
+
 stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text)
 if [ $stat -eq 80 ]; then 
     aws ec2 start-instances --instance-ids $SERVERINSTANCEID
 fi
-log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib
 while [ "$stat" -ne 16 ]; do
     sleep 30
     if [ $stat -eq 80 ]; then 
         aws ec2 start-instances --instance-ids $SERVERINSTANCEID
     fi
-    log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib
     stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text)
 done
-privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text)
 
 execcons=execconsol.sh
 execConsolsh=/tmp/$execcons
 python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', '$TESTMODE')"
 chmod +x $execConsolsh
 
-log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib
-scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$privip:data/distrib/$execcons
-ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execcons"
+logger -s -t runDistrib "running consolidation"
+
+scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execcons
+ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execcons"
 
-log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation, copying databases" runDistrib
-rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib
+logger -s -t runDistrib "finished consolidation, copying databases"
 
-# remote temporary files
-ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickle"  -mtime +7 -exec rm -f {} \;"
+rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib
+ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find /tmp -maxdepth 1 -name "*.pickle"  -mtime +7 -exec rm -f {} \;"
 
-log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib
+logger -s -t runDistrib "stopping calcserver again"
 aws ec2 stop-instances --instance-ids $SERVERINSTANCEID
 
+logger -s -t runDistrib "copying data to batch server and tidying up"
+
 # grab a copy of the indvidual container dbs so we can get a list of new solutions
 rm -Rf $DATADIR/latest/contdbs/
 mkdir -p $DATADIR/latest/contdbs/
@@ -154,8 +135,6 @@ aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATAD
 aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $SRC/logs/distrib/ --exclude "*" --include "correl*.log"  --exclude "logs/" --quiet
 aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "correl*.log"  --exclude "logs/" --recursive  --quiet
 
-log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib
-
 mkdir -p $DATADIR/trajdb
 tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db
 mkdir -p $DATADIR/distrib/containers
@@ -168,7 +147,4 @@ find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \;
 find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \;
 rm -f $DATADIR/distrib/${rundate}.pickle
 
-
-# and then clear the profile again
-unset AWS_PROFILE
-log2cw $NJLOGGRP $NJLOGSTREAM "finished runDistrib" runDistrib
+logger -s -t "finished runDistrib"