Reverse merge of 2026-Apr release into dev

.gitignore

@@ -531,13 +531,13 @@ StandaloneCurator/.installed
 analysis/orbitSolver/testing
 analysis/FormatConverters/testing
 */test_data/*
-archive/samfunctions*/pytz*
+archive/lambdas*/pytz*
 archive/static_content/data/searchdialog.js
 ToDo*
 archive/costs/**/*.csv
 archive/costs/**/*.json
 archive/costs/plots/*.png
-archive/samfunctions*/.aws-sam/build.toml
+archive/lambdas*/.aws-sam/build.toml
 archive/costs/S3-size-empireelements.xlsx
 archive/ukmon_pylib/share/maps/BM*.jpg
 archive/ukmon_pylib/share/maps/BM*.png
@@ -556,23 +556,22 @@ archive/unused/**
 ukmon_pylib/tests/20220217_032206.832_UK/*
 unused/*
 *tfstate.lock.info
-archive/samfunctions*/getExtraFilesV2/pythoncode/WesternMeteorPyLib/*
-archive/samfunctions*/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/*
-archive/containers/trajsolver/WesternMeteorPyLib/*
+archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/**
+archive/containers/trajsolver/WesternMeteorPyLib/**
 archive/containers/trajsolver/awskeys
-archive/containers/trajsolvertest/WesternMeteorPyLib/*
+archive/containers/trajsolvertest/WesternMeteorPyLib/**
 archive/containers/trajsolvertest/awskeys
 #ukmon_pylib/traj/clusdetails.txt
 **/*.pickle
 archive/containers/gui/awskeys
 archive/terraform/*/ukmonreadonly.key
 .cache_ggshield
 replacements.txt
-archive/samfunctions*/apis-mm.txt
-archive/samfunctions*/apis-ee.txt
+archive/lambdas/apis-mm.txt
+archive/lambdas/apis-ee.txt
 usermgmt/windows/createUserKeyAWS.ps1
-archive/samfunctions*/matchPickle/pythoncode/WesternMeteorPyLib/**
-archive/samfunctions*/matchPickle/pythoncode/*.npy
+archive/lambdas/matchPickle/pythoncode/WesternMeteorPyLib/**
+archive/lambdas/matchPickle/pythoncode/*.npy
 .terraform/
 archive/utils/tmp.json
 usermgmt/windows/caminfo/camera-details-new.csv
@@ -589,8 +588,8 @@ ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M002.ecsv
 ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M004.ecsv
 ukmon_pylib/tests/usertools/20230202_014115.kml
 archive/containers*/simpleUI/awskeys
-archive/samfunctions*/camDetails/pythoncode/camDetails.py
-archive/samfunctions*/camDetails/pythoncode/awskeys
+archive/lambdas*/camDetails/pythoncode/camDetails.py
+archive/lambdas*/camDetails/pythoncode/awskeys
 archive/containers/RMS-docker/rms_docker.tar
 archive/containers/RMS-docker/Dockerfile_orig.txt
 
@@ -609,11 +608,11 @@ usermgmt/server/ukmon*.json
 **/awskeys.test
 **/trajsolver_old/*
 archive/tmp/*
-servercopy/files/*
+servercopy/*
 tests/testing/RMS/
 tests/testing/WesternMeteorPyLib/
 archive/ukmon_pylib/tests/testdata.tar.gz
-servercopybkp/*
+servercopydev/*
 fbcollector/config.ini
 usermgmt/windows/stationmaint.ini
 usermgmt/windows/README.md

README.md

@@ -1,5 +1,5 @@
 # UK Meteor Data Analysis Shared code and libraries
-version: 2024.04.2
+version: 2026.04.1
 
 This repository contains the code behind the UK Meteors data archive and data processing pipeline. 
 

archive/README.md

@@ -1,5 +1,8 @@
 Data Processing and Flows
 ==========================
+
+version: 2026.04.1
+
 This diagram shows the overall flow of data from Cameras to websites and out to the public.
 
 ```mermaid

archive/analysis/README.md

@@ -1,17 +1,20 @@
 # analysis
 
-This folder contains the batch scripts that perform various analyses of the data. They're mostly triggere
+This folder contains the batch scripts that perform various analyses of the data. They're mostly triggered
 from the nightly batch but can also be triggered manually as needed. 
 
+## findAllMatches
+The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. 
+
+## runDistrib
+Called by findAllMatches to execute the distributed processing engine.  This file is designed to be run standalone so you can rerun distributed matching.
+
 ## consolidateOutput
 Collects all single station and match data and consolidates it into two files in parquet format. These are used for all detailed analysis.
 
 ## createSearchable
 Creates a single file for the search engine, by consolidating the required information from the match and single station data. 
 
-## findAllMatches
-The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. 
-
 ## getBadStations
 Checks for stations that failed quality tests such as too many detections, not uploaded for a few days etc. 
 
@@ -24,14 +27,11 @@ Creates a UFO-analyser compatible version of the RMS single-station detections.
 ## reportActiveShowers
 Creates a shower report for any active showers, by calling showerReport for each active shower year-to-date. 
 
-## runDistrib
-Called by findAllMatches to execute the distributed processing engine. 
-
 ## showerReport
 Creates a report for one or more showers. 
 
 ## stationReports
 Creates a report of data for one or all stations, for a month or year to date, which is then pushed to the website. 
 
 ## Copyright
-All code Copyright (C) 2018-2023 Mark McIntyre#
+All code Copyright (C) 2018- Mark McIntyre#

archive/analysis/consolidateOutput.sh

@@ -46,12 +46,12 @@ do
         typ=${bn:0:3}
         if [ "$typ" != "M20" ] ; then 
             pref="P"
-            yr=${bn:7:4}
+            snglyr=${bn:7:4}
         else
             pref="M"
-            yr=${bn:1:4}
+            snglyr=${bn:1:4}
         fi 
-        mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv
+        mrgfile=${DATADIR}/consolidated/${pref}_${snglyr}-unified.csv
         if [ ! -f $mrgfile ] ; then
             cat $csvf >> $mrgfile
         else
@@ -62,6 +62,10 @@ do
     fi
 done
 
+logger -s -t consolidateOutput "purging older raw data which is on S3 anyway"
+find ${DATADIR}/single/rawcsvs -mtime +180 -exec rm -f {} \;
+
+
 logger -s -t consolidateOutput "pushing consolidated information back"
 aws s3 sync ${DATADIR}/consolidated ${UKMONSHAREDBUCKET}/consolidated/  --exclude 'UKMON*' --quiet 
 
@@ -90,6 +94,12 @@ cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/matched/matches-full-$y
 cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/searchidx/matches-full-$yr-new.csv
 mv ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv ${DATADIR}/orbits/${yr}/fullcsv/processed
 
+logger -s -t consolidateOutput "purging older raw data"
+find ${DATADIR}/orbits/${yr}/fullcsv/processed -mtime +180 -exec rm -f {} \;
+# and last year, because the data is split across dated folders
+lyr=$(date -d 'last year' +%Y)
+find ${DATADIR}/orbits/${lyr}/fullcsv/processed -mtime +180 -exec rm -f {} \;
+
 python << EOD3
 import pandas as pd 
 df = pd.read_csv('${DATADIR}/matched/matches-full-${yr}.csv', skipinitialspace=True)

archive/analysis/findAllMatches.sh

@@ -29,7 +29,8 @@ if [ "$NJLOGSTREAM" == "" ]; then
     aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared
 fi
 log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" findAllMatches
-rundate=$(cat $DATADIR/rundate.txt)
+
+[ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d)
 
 # read start/end dates from commandline if rerunning for historical date
 if [ $# -gt 0 ] ; then
@@ -47,15 +48,7 @@ if [ $# -gt 0 ] ; then
 fi
 
 # folder for logs
-mkdir -p $SRC/logs > /dev/null 2>&1
-
-log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" findAllMatches
-# this creates the parquet table for Athena
-$SRC/analysis/getRMSSingleData.sh
-
-log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" findAllMatches
-yr=$(date +%Y)
-$SRC/analysis/createSearchable.sh $yr singles
+mkdir -p $SRC/logs/distrib > /dev/null 2>&1
 
 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000')
 enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000')
@@ -64,6 +57,9 @@ log2cw $NJLOGGRP $NJLOGSTREAM "solving for ${startdt} to ${enddt}" findAllMatche
 log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches
 $SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND
 
+log2cw $NJLOGGRP $NJLOGSTREAM "clean duplicate/deleted trajs" findAllMatches
+$SRC/utils/cleanupDeletedTrajs.sh
+
 log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches
 success=$(grep "Total run time:" $SRC/logs/matchJob.log)
 
@@ -79,28 +75,15 @@ python -m maintenance.rerunFailedLambdas
 
 cd $here
 log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches
-python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate processed_trajectories.json
-
-log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches
-dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1)
-trajlist=$(cat $dailyrep | awk -F, '{print $2}')
-
 matchlog=${SRC}/logs/matchJob.log
-vals=$(python -m metrics.getMatchStats $matchlog )
-evts=$(echo $vals | awk '{print $2}')
-trajs=$(echo $vals | awk '{print $6}')
-matches=$(wc -l $dailyrep | awk '{print $1}')
-rtim=$(echo $vals | awk '{print $7}')
-echo $(basename $dailyrep) $evts $trajs $matches $rtim >>  $DATADIR/dailyreports/stats.txt
+python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate
+python -m metrics.getMatchStats $matchlog
 
 # copy stats to S3 so the daily report can run
 if [ "$RUNTIME_ENV" == "PROD" ] ; then 
     aws s3 sync $DATADIR/dailyreports/ $UKMONSHAREDBUCKET/matches/RMSCorrelate/dailyreports/ --quiet
 fi 
 
-log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" findAllMatches
-$SRC/website/updateIndexPages.sh $dailyrep
-
 log2cw $NJLOGGRP $NJLOGSTREAM "start purgeLogs" findAllMatches
 find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \;
 find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \;

archive/analysis/getRMSSingleData.sh

@@ -56,7 +56,10 @@ fi
 
 # push to S3 bucket for future use by AWS tools
 logger -s -t getRMSSingleData "copy to S3 bucket"
-aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --quiet
+aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --exclude "rawcsvs/*" --exclude "used/*" --quiet
 aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/singlepq/ --exclude "*" --include "*.parquet.snap" --exclude "*new.parquet.snap" --quiet
 
+logger -s -t getRMSSingleData "purge processed data"
+find $outdir/processed -mtime +180 -exec rm -f {} \;
+
 logger -s -t getRMSSingleData "finished"