Summer2026 update

.gitignore

@@ -617,3 +617,8 @@ fbcollector/config.ini
 usermgmt/windows/stationmaint.ini
 usermgmt/windows/README.md
 fbCollector/README.html
+archive/share/combined_shower_table.parquet
+archive/share/gmn_shower_table_20230518.txt
+archive/share/IMO_Working_Meteor_Shower_List.xml
+**/.aws-sam/**
+archive/lambdas/*/tests/new_output.txt

README.md

@@ -1,23 +1,20 @@
 # UK Meteor Data Analysis Shared code and libraries
-version: 2026.05.3
+version: 2026.05.4
 
 This repository contains the code behind the UK Meteors data archive and data processing pipeline. 
 
 ## ARCHIVE folder
 The code for archive.ukmeteors.co.uk, including the data processing pipeline.
 
 ### Software Deployment
-The software is a mix of Python and Bash shell scripts. All software deployment uses Ansible. 
+The software is a mix of Python and Bash shell scripts. Deployment is via an install script. 
 
 #### Scripts and Python
-The shell scripts and python code are deployed with `deploy-analysis.yml`. Use the "prod" tag to push to production, and the "dev" tag to push to a development environment. Deployments target the batch server in the MM account. 
+The shell scripts and python code are deployed with `install_or_update.sh`. First, clone the repository onto the target server then run `install_or_update.sh` with an argument "PROD"  to create/update a production environment, or "DEV"  for a development environment. . 
 
-```bash
-ansible-playbook deploy-analysis.yml -t prod
-```
 #### Configuration Files
 Parameters are created with Terraform and stored in the AWS Systems Manager Parameter Store. 
-The configuration is then deployed with `deploy-config.yml` which runs a script to create the `config.ini` file. 
+The configuration is then deployed with `utils/makeConfig.sh` which runs a script to create the `config.ini` file. 
 
 #### Website 
 Much of the website is dynamically created by the Python and Bash scripts or by AWS Lambda functions. 
@@ -30,7 +27,3 @@ Done with Terraform - see the Terraform folder readme for details.
 
 ## TESTS folder
 This folder contains tests for the APIs and some of the Python code. Further tests are being developed and will be added to this folder. See the README in the folder for more details. 
-
-## USERMGMT folder
-A python app that is used to add/modify contributors' camera details and grant them permission to upload.
-Not intended for general use, this tool can only be used if you have SSH and AWS keys for the admin role. 

archive/README.md

@@ -1,7 +1,7 @@
 Data Processing and Flows
 ==========================
 
-version: 2026.05.3
+version: 2026.05.4
 
 This diagram shows the overall flow of data from Cameras to websites and out to the public.
 

archive/analysis/consolidateOutput.sh

@@ -27,19 +27,20 @@ fi
 
 cd ${DATADIR}
 # consolidate UFO and RMS original CSVs.
-logger -s -t consolidateOutput "starting"
+logger -s -t $(basename $0 .sh) "starting"
+
 # note - copying from production S3 even in dev so we have the latest data
 aws s3 sync s3://ukmda-shared/consolidated/ ${DATADIR}/consolidated --exclude "temp/*" --quiet
 aws s3 mv s3://ukmda-shared/consolidated/temp/ ${DATADIR}/consolidated/temp --recursive --quiet
 
-logger -s -t consolidateOutput "Consolidating RMS and UFO CSVs"
+logger -s -t $(basename $0 .sh) "Consolidating RMS and UFO CSVs"
 consdir=${DATADIR}/consolidated/temp
 mkdir -p ${DATADIR}/single/rawcsvs 
 ls -1 $consdir/*.csv | while read csvf
 do
     flen=$(wc -l $csvf | awk '{print $1}')
     if [ $flen -lt 2 ] ; then 
-        logger -s -t consolidateOutput "skipping empty file $csvf"
+        logger -s -t $(basename $0 .sh) "skipping empty file $csvf"
         rm $csvf 
     else
         bn=$(basename $csvf)
@@ -62,14 +63,14 @@ do
     fi
 done
 
-logger -s -t consolidateOutput "purging older raw data which is on S3 anyway"
+logger -s -t $(basename $0 .sh) "purging older raw data which is on S3 anyway"
 find ${DATADIR}/single/rawcsvs -mtime +180 -exec rm -f {} \;
 
 
-logger -s -t consolidateOutput "pushing consolidated information back"
+logger -s -t $(basename $0 .sh) "pushing consolidated information back"
 aws s3 sync ${DATADIR}/consolidated ${UKMONSHAREDBUCKET}/consolidated/  --exclude 'UKMON*' --quiet 
 
-logger -s -t consolidateOutput "Getting latest trajectory data"
+logger -s -t $(basename $0 .sh) "Getting latest trajectory data"
 
 # collect the latest trajectory CSV files
 # make sure target folders exist
@@ -82,19 +83,19 @@ aws s3 sync s3://ukmda-shared/matches/RMSCorrelate/trajectories/${yr}/plots/ $DA
 aws s3 mv ${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv/  ${DATADIR}/orbits/${yr}/fullcsv --recursive --exclude "*" --include "*.csv" --quiet
 
 # get the latest matched data generated by WMPL
-logger -s -t consolidateOutput "getting matched detections for $yr"
+logger -s -t $(basename $0 .sh) "getting matched detections for $yr"
 if [ ! -f ${DATADIR}/matched/matches-full-$yr.csv ] ; then 
     cp $SRC/analysis/templates/match_hdr_full.txt ${DATADIR}/matched/matches-full-$yr.csv
 fi
-logger -s -t consolidateOutput "getting new matched detections for today"
+logger -s -t $(basename $0 .sh) "getting new matched detections for today"
 if [ ! -f ${DATADIR}/searchidx/matches-full-$yr-new.csv ] ; then 
     cp $SRC/analysis/templates/match_hdr_full.txt ${DATADIR}/searchidx/matches-full-$yr-new.csv
 fi
 cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/matched/matches-full-$yr.csv
 cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/searchidx/matches-full-$yr-new.csv
 mv ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv ${DATADIR}/orbits/${yr}/fullcsv/processed
 
-logger -s -t consolidateOutput "purging older raw data"
+logger -s -t $(basename $0 .sh) "purging older raw data"
 find ${DATADIR}/orbits/${yr}/fullcsv/processed -mtime +180 -exec rm -f {} \;
 # and last year, because the data is split across dated folders
 lyr=$(date -d 'last year' +%Y)
@@ -119,4 +120,4 @@ aws s3 sync $DATADIR/matched/ $UKMONSHAREDBUCKET/matches/matchedpq/ --quiet --ex
 aws s3 sync $DATADIR/matched/ $WEBSITEBUCKET/browse/parquet/  --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet
 aws s3 sync $DATADIR/single/ $WEBSITEBUCKET/browse/parquet/  --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet
 
-logger -s -t consolidateOutput "finished"
+logger -s -t $(basename $0 .sh) "finished"

archive/analysis/createSearchable.sh

@@ -17,7 +17,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 
 source $here/../config.ini >/dev/null 2>&1
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
-logger -s -t createSearchable "starting"
+logger -s -t $(basename $0 .sh) "starting"
 
 yr=$1
 whichpass=$2
@@ -38,4 +38,4 @@ fi
 
 aws s3 sync  $DATADIR/searchidx/ $WEBSITEBUCKET/search/indexes/ --exclude "*" --include "*allevents.csv" --quiet 
 
-logger -s -t createSearchable "finished"
+logger -s -t $(basename $0 .sh) "finished"

archive/analysis/findAllMatches.sh

@@ -23,7 +23,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 source $here/../config.ini >/dev/null 2>&1
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
 
-logger -s -t findAllMatches "starting"
+logger -s -t $(basename $0 .sh) "starting"
 
 [ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d)
 
@@ -48,27 +48,27 @@ mkdir -p $SRC/logs/distrib > /dev/null 2>&1
 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000')
 enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000')
 
-logger -s -t findAllMatches "solving for ${startdt} to ${enddt}"
-logger -s -t findAllMatches "start runDistrib"
+logger -s -t $(basename $0 .sh) "solving for ${startdt} to ${enddt}"
+logger -s -t $(basename $0 .sh) "start runDistrib"
 
 $SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND
 $SRC/utils/cleanupDeletedTrajs.sh
 
-logger -s -t findAllMatches "Solving Run Done" 
+logger -s -t $(basename $0 .sh) "Solving Run Done" 
 
 success=$(grep "Total run time:" $SRC/logs/matchJob.log)
 
 if [ "$success" == "" ]
 then
-    python -c "from utils.sendAnEmail import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')"
+    python -c "from utils.sendAnEmail import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching')"
     echo problems with solver
 fi
 
 python -m maintenance.rerunFailedLambdas
 
 cd $here
 
-logger -s -t findAllMatches "start reportOfLatestMatches" 
+logger -s -t $(basename $0 .sh) "running reportOfLatestMatches"
 
 matchlog=${SRC}/logs/matchJob.log
 python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $rundate
@@ -82,4 +82,4 @@ fi
 find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \;
 find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \;
 
-logger -s -t findAllMatches "finished"
+logger -s -t $(basename $0 .sh) "finished"

archive/analysis/getBadStations.sh

@@ -7,8 +7,8 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 source $here/../config.ini >/dev/null 2>&1
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
 
-logger -s -t getBadStations "starting"
+logger -s -t $(basename $0 .sh) "starting"
 aws s3 sync $UKMONSHAREDBUCKET/admin  $DATADIR/admin --dryrun --quiet 
 
 python -m reports.reportBadCameras 3
-logger -s -t getBadStations "finished"
+logger -s -t $(basename $0 .sh) "finished"

archive/analysis/getLogData.sh

@@ -5,6 +5,8 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 source $here/../config.ini >/dev/null 2>&1
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
 
+logger -s -t $(basename $0 .sh) "starting"
+
 if [ "$1" != "" ] ; then
     rundate=$1
     logfile=$DATADIR/lastlogs/lastlog-${rundate}.html
@@ -98,3 +100,5 @@ aws s3 cp $DATADIR/lastlogs/index.html  $WEBSITEBUCKET/reports/lastlogs/ --quiet
 find $DATADIR/failed -mtime +90 -exec rm -f {} \;
 
 find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \;
+
+logger -s -t $(basename $0 .sh) "finished"

archive/analysis/getRMSSingleData.sh

@@ -17,7 +17,8 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 source $here/../config.ini >/dev/null 2>&1
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
 
-logger -s -t getRMSSingleData "starting"
+logger -s -t $(basename $0 .sh) "starting"
+
 indir=$UKMONSHAREDBUCKET/matches/single/new/
 outdir=$DATADIR/single/new
 mkdir -p $outdir/processed > /dev/null 2>&1 
@@ -45,7 +46,7 @@ do
     mv $i $outdir/processed
 done 
 
-logger -s -t getRMSSingleData "convert to parquet"
+logger -s -t $(basename $0 .sh) "convert to parquet"
 if [ -f $mrgfile ] ; then 
     python -m converters.toParquet $mrgfile
 fi 
@@ -55,11 +56,11 @@ if [ -f $newsngl ] ; then
 fi 
 
 # push to S3 bucket for future use by AWS tools
-logger -s -t getRMSSingleData "copy to S3 bucket"
+logger -s -t $(basename $0 .sh) "copy to S3 bucket"
 aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --exclude "rawcsvs/*" --exclude "used/*" --quiet
 aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/singlepq/ --exclude "*" --include "*.parquet.snap" --exclude "*new.parquet.snap" --quiet
 
 logger -s -t getRMSSingleData "purge processed data"
 find $outdir/processed -mtime +180 -exec rm -f {} \;
 
-logger -s -t getRMSSingleData "finished"
+logger -s -t $(basename $0 .sh) "finished"

archive/analysis/reportActiveShowers.sh

@@ -17,7 +17,8 @@
 here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 source $here/../config.ini >/dev/null 2>&1
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
-logger -s -t reportActiveShowers "starting"
+
+logger -s -t $(basename $0 .sh) "starting"
 
 if [ $# -eq 0 ]; then
     yr=$(date +%Y)
@@ -27,16 +28,17 @@ else
     rundt=${1}$(date +%m%d)
 fi
 
-logger -s -t reportActiveShowers "report on active showers"
+logger -s -t $(basename $0 .sh) "start reportActiveShowers"
 python -m reports.reportActiveShowers -m
 
 python -c "from utils.getActiveShowers import getActiveShowers;getActiveShowers('$rundt', inclMinor=True)" | while read shwr
 do 
     aws s3 sync $DATADIR/reports/${yr}/$shwr $WEBSITEBUCKET/reports/${yr}/${shwr} --quiet 
 done
-logger -s -t reportActiveShowers "updating annual index"
+logger -s -t $(basename $0 .sh) "updating annual index"
 ${SRC}/website/createReportIndex.sh ${yr}
 
 python -m analysis.summaryAnalysis ${yr}
 aws s3 sync $DATADIR/reports/${yr}/showers $WEBSITEBUCKET/reports/${yr}/showers --quiet
-logger -s -t reportActiveShowers "finished"
+
+logger -s -t $(basename $0 .sh) "finished"

archive/analysis/runDistrib.sh

@@ -20,7 +20,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 # load the configuration
 source $here/../config.ini >/dev/null 2>&1
 
-logger -s -t runDistrib "starting runDistrib" 
+logger -s -t $(basename $0 .sh) "starting"
 
 if [ $# -gt 0 ] ; then
     if [ "$1" != "" ] ; then
@@ -35,9 +35,9 @@ fi
 begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d')
 rundate=$(date --date="-$MATCHEND days" '+%Y%m%d')
 
-logger -s -t runDistrib "running phase 1 for dates ${begdate} to ${rundate}"
+logger -s -t $(basename $0 .sh) "running phase 1 for dates ${begdate} to ${rundate}"
 
-logger -s -t runDistrib  "start correlation server" 
+logger -s -t $(basename $0 .sh)  "start correlation server" 
 aws ec2 start-instances --instance-ids $SERVERINSTANCEID
 stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text)
 while [ "$stat" -ne 16 ]; do
@@ -47,20 +47,20 @@ done
 
 conda activate $HOME/miniconda3/envs/${WMPL_ENV}
 
-logger -s -t runDistrib "creating the run script"
+logger -s -t $(basename $0 .sh) "creating the run script"
 
 execdist=execdistrib.sh
 execMatchingsh=/tmp/$execdist
 python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE
 chmod +x $execMatchingsh
 
-logger -s -t runDistrib "deploy the script to the server $CALCSERVERIP and run it"
+logger -s -t $(basename $0 .sh) "deploy the script to the server $CALCSERVERIP and run it"
 
-scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist
+scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:runtime/scripts/$execdist
 while [ $? -ne 0 ] ; do
     # in case the server isn't responding to ssh sessions yet
     sleep 10
-    scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist
+    scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:runtime/scripts/$execdist
 done 
 # push the python code and ECS templates required
 rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/
@@ -70,17 +70,19 @@ rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVER
 rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/
 rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/ShowerAssociation.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/
 rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/utils/convertSolLon.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/utils/
+rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/maintenance/dataMaintenance.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/maintenance/
+rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/analysis/getUsedUnused.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/analysis/
 
 # now run the script
-logger -s -t runDistrib "start distributed processing"
-ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execdist"
+logger -s -t $(basename $0 .sh) "start distributed processing"
+ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "runtime/scripts/$execdist"
 
 rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates
 
 rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/
 ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;"
 
-logger -s -t runDistrib "job run, stop the server again"
+logger -s -t $(basename $0 .sh) "job run, stop the server again"
 aws ec2 stop-instances --instance-ids $SERVERINSTANCEID
 
 logger -s -t runDistrib "monitoring and waiting for completion"
@@ -90,7 +92,7 @@ python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${ru
 mkdir -p $DATADIR/distrib
 cd $DATADIR/distrib
 
-logger -s -t runDistrib "restarting server to consolidate results"
+logger -s -t $(basename $0 .sh) "restarting server to consolidate results"
 
 stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text)
 if [ $stat -eq 80 ]; then 
@@ -109,20 +111,20 @@ execConsolsh=/tmp/$execcons
 python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', '$TESTMODE')"
 chmod +x $execConsolsh
 
-logger -s -t runDistrib "running consolidation"
+logger -s -t $(basename $0 .sh) "running consolidation"
 
-scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execcons
-ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execcons"
+scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$CALCSERVERIP:runtime/scripts/$execcons
+ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "runtime/scripts/$execcons"
 
-logger -s -t runDistrib "finished consolidation, copying databases"
+logger -s -t $(basename $0 .sh) "finished consolidation, copying databases"
 
 rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib
 ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find /tmp -maxdepth 1 -name "*.pickle"  -mtime +7 -exec rm -f {} \;"
 
-logger -s -t runDistrib "stopping calcserver again"
+logger -s -t $(basename $0 .sh) "stopping calcserver again"
 aws ec2 stop-instances --instance-ids $SERVERINSTANCEID
 
-logger -s -t runDistrib "copying data to batch server and tidying up"
+logger -s -t $(basename $0 .sh) "copying data to batch server and tidying up"
 
 # grab a copy of the indvidual container dbs so we can get a list of new solutions
 rm -Rf $DATADIR/latest/contdbs/
@@ -147,4 +149,4 @@ find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \;
 find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \;
 rm -f $DATADIR/distrib/${rundate}.pickle
 
-logger -s -t "finished runDistrib"
+logger -s -t $(basename $0 .sh) "finished"