diff --git a/.gitignore b/.gitignore index e897bccc6..d266d9591 100644 --- a/.gitignore +++ b/.gitignore @@ -531,13 +531,13 @@ StandaloneCurator/.installed analysis/orbitSolver/testing analysis/FormatConverters/testing */test_data/* -archive/samfunctions*/pytz* +archive/lambdas*/pytz* archive/static_content/data/searchdialog.js ToDo* archive/costs/**/*.csv archive/costs/**/*.json archive/costs/plots/*.png -archive/samfunctions*/.aws-sam/build.toml +archive/lambdas*/.aws-sam/build.toml archive/costs/S3-size-empireelements.xlsx archive/ukmon_pylib/share/maps/BM*.jpg archive/ukmon_pylib/share/maps/BM*.png @@ -556,11 +556,10 @@ archive/unused/** ukmon_pylib/tests/20220217_032206.832_UK/* unused/* *tfstate.lock.info -archive/samfunctions*/getExtraFilesV2/pythoncode/WesternMeteorPyLib/* -archive/samfunctions*/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/* -archive/containers/trajsolver/WesternMeteorPyLib/* +archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/** +archive/containers/trajsolver/WesternMeteorPyLib/** archive/containers/trajsolver/awskeys -archive/containers/trajsolvertest/WesternMeteorPyLib/* +archive/containers/trajsolvertest/WesternMeteorPyLib/** archive/containers/trajsolvertest/awskeys #ukmon_pylib/traj/clusdetails.txt **/*.pickle @@ -568,11 +567,11 @@ archive/containers/gui/awskeys archive/terraform/*/ukmonreadonly.key .cache_ggshield replacements.txt -archive/samfunctions*/apis-mm.txt -archive/samfunctions*/apis-ee.txt +archive/lambdas/apis-mm.txt +archive/lambdas/apis-ee.txt usermgmt/windows/createUserKeyAWS.ps1 -archive/samfunctions*/matchPickle/pythoncode/WesternMeteorPyLib/** -archive/samfunctions*/matchPickle/pythoncode/*.npy +archive/lambdas/matchPickle/pythoncode/WesternMeteorPyLib/** +archive/lambdas/matchPickle/pythoncode/*.npy .terraform/ archive/utils/tmp.json usermgmt/windows/caminfo/camera-details-new.csv @@ -589,8 +588,8 @@ ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M002.ecsv ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M004.ecsv ukmon_pylib/tests/usertools/20230202_014115.kml archive/containers*/simpleUI/awskeys -archive/samfunctions*/camDetails/pythoncode/camDetails.py -archive/samfunctions*/camDetails/pythoncode/awskeys +archive/lambdas*/camDetails/pythoncode/camDetails.py +archive/lambdas*/camDetails/pythoncode/awskeys archive/containers/RMS-docker/rms_docker.tar archive/containers/RMS-docker/Dockerfile_orig.txt @@ -609,11 +608,11 @@ usermgmt/server/ukmon*.json **/awskeys.test **/trajsolver_old/* archive/tmp/* -servercopy/files/* +servercopy/* tests/testing/RMS/ tests/testing/WesternMeteorPyLib/ archive/ukmon_pylib/tests/testdata.tar.gz -servercopybkp/* +servercopydev/* fbcollector/config.ini usermgmt/windows/stationmaint.ini usermgmt/windows/README.md diff --git a/README.md b/README.md index 226593df5..9f059c11c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2024.04.2 +version: 2026.05.0 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index 9fefc6338..c37b478af 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,5 +1,8 @@ Data Processing and Flows ========================== + +version: 2026.05.0 + This diagram shows the overall flow of data from Cameras to websites and out to the public. ```mermaid diff --git a/archive/analysis/README.md b/archive/analysis/README.md index a260be3d5..a20d70e2b 100644 --- a/archive/analysis/README.md +++ b/archive/analysis/README.md @@ -1,17 +1,20 @@ # analysis -This folder contains the batch scripts that perform various analyses of the data. They're mostly triggere +This folder contains the batch scripts that perform various analyses of the data. They're mostly triggered from the nightly batch but can also be triggered manually as needed. +## findAllMatches +The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. + +## runDistrib +Called by findAllMatches to execute the distributed processing engine. This file is designed to be run standalone so you can rerun distributed matching. + ## consolidateOutput Collects all single station and match data and consolidates it into two files in parquet format. These are used for all detailed analysis. ## createSearchable Creates a single file for the search engine, by consolidating the required information from the match and single station data. -## findAllMatches -The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. - ## getBadStations Checks for stations that failed quality tests such as too many detections, not uploaded for a few days etc. @@ -24,9 +27,6 @@ Creates a UFO-analyser compatible version of the RMS single-station detections. ## reportActiveShowers Creates a shower report for any active showers, by calling showerReport for each active shower year-to-date. -## runDistrib -Called by findAllMatches to execute the distributed processing engine. - ## showerReport Creates a report for one or more showers. @@ -34,4 +34,4 @@ Creates a report for one or more showers. Creates a report of data for one or all stations, for a month or year to date, which is then pushed to the website. ## Copyright -All code Copyright (C) 2018-2023 Mark McIntyre# \ No newline at end of file +All code Copyright (C) 2018- Mark McIntyre# \ No newline at end of file diff --git a/archive/analysis/consolidateOutput.sh b/archive/analysis/consolidateOutput.sh index 34743434c..51aa0418f 100644 --- a/archive/analysis/consolidateOutput.sh +++ b/archive/analysis/consolidateOutput.sh @@ -46,12 +46,12 @@ do typ=${bn:0:3} if [ "$typ" != "M20" ] ; then pref="P" - yr=${bn:7:4} + snglyr=${bn:7:4} else pref="M" - yr=${bn:1:4} + snglyr=${bn:1:4} fi - mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv + mrgfile=${DATADIR}/consolidated/${pref}_${snglyr}-unified.csv if [ ! -f $mrgfile ] ; then cat $csvf >> $mrgfile else @@ -62,6 +62,10 @@ do fi done +logger -s -t consolidateOutput "purging older raw data which is on S3 anyway" +find ${DATADIR}/single/rawcsvs -mtime +180 -exec rm -f {} \; + + logger -s -t consolidateOutput "pushing consolidated information back" aws s3 sync ${DATADIR}/consolidated ${UKMONSHAREDBUCKET}/consolidated/ --exclude 'UKMON*' --quiet @@ -90,6 +94,12 @@ cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/matched/matches-full-$y cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/searchidx/matches-full-$yr-new.csv mv ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv ${DATADIR}/orbits/${yr}/fullcsv/processed +logger -s -t consolidateOutput "purging older raw data" +find ${DATADIR}/orbits/${yr}/fullcsv/processed -mtime +180 -exec rm -f {} \; +# and last year, because the data is split across dated folders +lyr=$(date -d 'last year' +%Y) +find ${DATADIR}/orbits/${lyr}/fullcsv/processed -mtime +180 -exec rm -f {} \; + python << EOD3 import pandas as pd df = pd.read_csv('${DATADIR}/matched/matches-full-${yr}.csv', skipinitialspace=True) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index db2b35cd6..59872560d 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -23,13 +23,9 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" findAllMatches -rundate=$(cat $DATADIR/rundate.txt) +logger -s -t findAllMatches "starting" + +[ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d) # read start/end dates from commandline if rerunning for historical date if [ $# -gt 0 ] ; then @@ -47,62 +43,43 @@ if [ $# -gt 0 ] ; then fi # folder for logs -mkdir -p $SRC/logs > /dev/null 2>&1 - -log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" findAllMatches -# this creates the parquet table for Athena -$SRC/analysis/getRMSSingleData.sh - -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" findAllMatches -yr=$(date +%Y) -$SRC/analysis/createSearchable.sh $yr singles +mkdir -p $SRC/logs/distrib > /dev/null 2>&1 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000') -log2cw $NJLOGGRP $NJLOGSTREAM "solving for ${startdt} to ${enddt}" findAllMatches -log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches +logger -s -t findAllMatches "solving for ${startdt} to ${enddt}" +logger -s -t findAllMatches "start runDistrib" + $SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND +$SRC/utils/cleanupDeletedTrajs.sh + +logger -s -t findAllMatches "Solving Run Done" -log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches success=$(grep "Total run time:" $SRC/logs/matchJob.log) if [ "$success" == "" ] then - python -c "from meteortools.utils import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')" + python -c "from utils.sendAnEmail import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')" echo problems with solver fi -log2cw $NJLOGGRP $NJLOGSTREAM "Solving Run Done" findAllMatches -log2cw $NJLOGGRP $NJLOGSTREAM "start rerunFailedLambdas" findAllMatches python -m maintenance.rerunFailedLambdas cd $here -log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches -python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate processed_trajectories.json -log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches -dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) -trajlist=$(cat $dailyrep | awk -F, '{print $2}') +logger -s -t findAllMatches "start reportOfLatestMatches" matchlog=${SRC}/logs/matchJob.log -vals=$(python -m metrics.getMatchStats $matchlog ) -evts=$(echo $vals | awk '{print $2}') -trajs=$(echo $vals | awk '{print $6}') -matches=$(wc -l $dailyrep | awk '{print $1}') -rtim=$(echo $vals | awk '{print $7}') -echo $(basename $dailyrep) $evts $trajs $matches $rtim >> $DATADIR/dailyreports/stats.txt +python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $rundate +python -m metrics.getMatchStats $matchlog $rundate # copy stats to S3 so the daily report can run if [ "$RUNTIME_ENV" == "PROD" ] ; then aws s3 sync $DATADIR/dailyreports/ $UKMONSHAREDBUCKET/matches/RMSCorrelate/dailyreports/ --quiet fi -log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" findAllMatches -$SRC/website/updateIndexPages.sh $dailyrep - -log2cw $NJLOGGRP $NJLOGSTREAM "start purgeLogs" findAllMatches find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \; find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \; -log2cw $NJLOGGRP $NJLOGSTREAM "finished findAllMatches" findAllMatches +logger -s -t findAllMatches "finished" diff --git a/archive/analysis/getLogData.sh b/archive/analysis/getLogData.sh index 1f46b6962..67342a511 100644 --- a/archive/analysis/getLogData.sh +++ b/archive/analysis/getLogData.sh @@ -10,7 +10,7 @@ if [ "$1" != "" ] ; then logfile=$DATADIR/lastlogs/lastlog-${rundate}.html else rundate=$(date +%Y%m%d) - logfile=$DATADIR/lastlog.html + logfile=$DATADIR/lastlogs/lastlog.html fi # create performance metrics @@ -78,7 +78,7 @@ echo "

> $logfile -if [ ! -d $DATADIR/lastlogs ] ; then mkdir $DATADIR/lastlogs ; fi +mkdir -p $DATADIR/lastlogs if [ "$1" == "" ] ; then aws s3 cp $logfile $WEBSITEBUCKET/reports/ --quiet cp $logfile $DATADIR/lastlogs/lastlog-${rundate}.html @@ -92,3 +92,5 @@ ls -1r $DATADIR/lastlogs/last*.html | head -90 | while read i ; do done cat $TEMPLATES/footer.html >> $DATADIR/lastlogs/index.html aws s3 cp $DATADIR/lastlogs/index.html $WEBSITEBUCKET/reports/lastlogs/ --quiet + +find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \; diff --git a/archive/analysis/getRMSSingleData.sh b/archive/analysis/getRMSSingleData.sh index 4879e4b36..0174a815c 100644 --- a/archive/analysis/getRMSSingleData.sh +++ b/archive/analysis/getRMSSingleData.sh @@ -56,7 +56,10 @@ fi # push to S3 bucket for future use by AWS tools logger -s -t getRMSSingleData "copy to S3 bucket" -aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --quiet +aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --exclude "rawcsvs/*" --exclude "used/*" --quiet aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/singlepq/ --exclude "*" --include "*.parquet.snap" --exclude "*new.parquet.snap" --quiet +logger -s -t getRMSSingleData "purge processed data" +find $outdir/processed -mtime +180 -exec rm -f {} \; + logger -s -t getRMSSingleData "finished" \ No newline at end of file diff --git a/archive/analysis/onlyConsolDistrib.sh b/archive/analysis/onlyConsolDistrib.sh deleted file mode 100644 index b37254253..000000000 --- a/archive/analysis/onlyConsolDistrib.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# just consolidate solutions of distributed candidates -# -# Used if consolidating a bunch of candidates that were solved outside the normal process -# -# consumes: the trajdb and solution json files -# creates: an updated trajdb and hopefully additional orbits -# - -""" -To use this function, first create the candidate pickle files, then copy them to the candidates -folder on the calcserver and submit them to the distributed processing engine with the following: - -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} -export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib -export AWS_PROFILE=ukmonshared -cd /home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates -time python -m traj.distributeCandidates 20230113 /home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates $UKMONSHAREDBUCKET/matches/distrib - -when this completes, logoff the calcserver and shut it down again -On the ukmonhelper run this script to wait for the distrib processing to finish and then -merge in the data - - -""" - - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -logger -s -t onlyConsolDistrib "starting onlyConsolDistrib" - -# load the configuration -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -# set the profile to the EE account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared - - -if [ "$1" == "" ] ; then - rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -else - rundate=$1 -fi -logger -s -t onlyConsolDistrib "consolidating for $rundate" - -python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}'); " - -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) -while [ "$privip" == "" ] ; do - sleep 5 - echo "getting ipaddress" - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) -done - -if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then - aws s3 sync $UKMONSHAREDBUCKET/matches/distrib/ $DATADIR/distrib/ --exclude "*" --include "*.json" --quiet - cp -f $DATADIR/distrib/processed_trajectories.json $DATADIR/distrib/prev_processed_trajectories.json - - numtoconsol=$(ls -1 $DATADIR/distrib/${rundate}*.json | wc -l) - if [ $numtoconsol -gt 5 ] ; then - logger -s -t onlyConsolDistrib "restarting calcserver to consolidate results" - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text --profile ukmonshared) - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - fi - logger -s -t onlyConsolDistrib "waiting for the server to be ready" - while [ "$stat" -ne 16 ]; do - sleep 30 - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - fi - echo "checking - status is ${stat}" - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text --profile ukmonshared) - done - - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - while [ $? -ne 0 ] ; do - # in case the server isn't responding to ssh sessions yet - sleep 10 - echo "server not responding yet, retrying" - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - done - scp -i $SERVERSSHKEY $DATADIR/distrib/${rundate}*.json ec2-user@$privip:data/distrib - - echo "#!/bin/bash" > /tmp/execConsol.sh - echo "export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib" >> /tmp/execConsol.sh - echo "python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json ${rundate}" >> /tmp/execConsol.sh - chmod +x /tmp/execConsol.sh - scp -i $SERVERSSHKEY /tmp/execConsol.sh ec2-user@$privip:data/distrib - ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/execConsol.sh" - - scp -i $SERVERSSHKEY ec2-user@$privip:data/distrib/processed_trajectories.json $DATADIR/distrib - - ssh -i $SERVERSSHKEY ec2-user@$privip "rm -f data/distrib/*.json" - - logger -s -t runDistrib "stopping calcserver again" - aws ec2 stop-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - - python -c "from traj.consolidateDistTraj import patchTrajDB ; patchTrajDB('$DATADIR/distrib/processed_trajectories.json','/home/ec2-user/ukmon-shared/matches/RMSCorrelate', '/home/ec2-user/data/distrib');" - else - python -m traj.consolidateDistTraj $DATADIR/distrib $DATADIR/distrib/processed_trajectories.json $rundate - fi - # push the updated traj db to the S3 bucket - aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - - logger -s -t onlyConsolDistrib "compressing the procssed data" - gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz - aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet - tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib/done/ --quiet - rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 rm $UKMONSHAREDBUCKET/matches/distrib/ --exclude "*" --include "${rundate}*.json" --exclude "test/*" --recursive -else - echo "trajectory database is size zero... not proceeding with copy" -fi -python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate processed_trajectories.json -dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) -$SRC/website/updateIndexPages.sh $dailyrep -logger -s -t onlyConsolDistrib "finished" diff --git a/archive/analysis/reportActiveShowers.sh b/archive/analysis/reportActiveShowers.sh index a42c90d3e..e26e04638 100644 --- a/archive/analysis/reportActiveShowers.sh +++ b/archive/analysis/reportActiveShowers.sh @@ -30,7 +30,7 @@ fi logger -s -t reportActiveShowers "report on active showers" python -m reports.reportActiveShowers -m -python -c "from meteortools.utils import getActiveShowers; getActiveShowers('$rundt', inclMinor=True)" | while read shwr +python -c "from utils.getActiveShowers import getActiveShowers;getActiveShowers('$rundt', inclMinor=True)" | while read shwr do aws s3 sync $DATADIR/reports/${yr}/$shwr $WEBSITEBUCKET/reports/${yr}/${shwr} --quiet done diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 468f02987..1cf23ab35 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -20,156 +20,131 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "starting runDistrib" runDistrib - -# set the profile to the EE account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared +logger -s -t runDistrib "starting runDistrib" if [ $# -gt 0 ] ; then if [ "$1" != "" ] ; then - log2cw $NJLOGGRP $NJLOGSTREAM "selecting range" runDistrib MATCHSTART=$1 fi if [ "$2" != "" ] ; then MATCHEND=$2 else - log2cw $NJLOGGRP $NJLOGSTREAM "matchend was not supplied, using 2 days" runDistrib MATCHEND=$(( $MATCHSTART - 2 )) fi fi begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" runDistrib -stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) -if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID -fi +logger -s -t runDistrib "running phase 1 for dates ${begdate} to ${rundate}" +logger -s -t runDistrib "start correlation server" +aws ec2 start-instances --instance-ids $SERVERINSTANCEID +stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) while [ "$stat" -ne 16 ]; do sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" runDistrib stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate}" runDistrib conda activate $HOME/miniconda3/envs/${WMPL_ENV} -log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" runDistrib +logger -s -t runDistrib "creating the run script" + execdist=execdistrib.sh execMatchingsh=/tmp/$execdist -python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh +python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE chmod +x $execMatchingsh -log2cw $NJLOGGRP $NJLOGSTREAM "get server details" runDistrib -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -while [ "$privip" == "" ] ; do - sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" runDistrib - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -done - -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" runDistrib +logger -s -t runDistrib "deploy the script to the server $CALCSERVERIP and run it" -scp -i $SERVERSSHKEY $execMatchingsh ec2-user@$privip:data/distrib/$execdist +scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $execMatchingsh ec2-user@$privip:data/distrib/$execdist + scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist done -# push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner.json ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py ec2-user@$privip:src/ukmon_pylib/traj +# push the python code and ECS templates required +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/ShowerAssociation.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/utils/convertSolLon.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/utils/ # now run the script -log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib -ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execdist" +logger -s -t runDistrib "start distributed processing" +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execdist" + +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates -log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" + +logger -s -t runDistrib "job run, stop the server again" aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -log2cw $NJLOGGRP $NJLOGSTREAM "monitoring and waiting for completion" runDistrib +logger -s -t runDistrib "monitoring and waiting for completion" -python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}'); " +python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}', '${TESTMODE}'); " -log2cw $NJLOGGRP $NJLOGSTREAM "merging in the new json files" runDistrib mkdir -p $DATADIR/distrib cd $DATADIR/distrib -# make sure the database isn't corrupt before overwriting it !! -if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then - aws s3 sync $UKMONSHAREDBUCKET/matches/distrib/ $DATADIR/distrib/ --exclude "*" --include "*.json" --quiet - cp -f $DATADIR/distrib/processed_trajectories.json $DATADIR/distrib/prev_processed_trajectories.json +logger -s -t runDistrib "restarting server to consolidate results" - log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +if [ $stat -eq 80 ]; then + aws ec2 start-instances --instance-ids $SERVERINSTANCEID +fi +while [ "$stat" -ne 16 ]; do + sleep 30 if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi - log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib - while [ "$stat" -ne 16 ]; do - sleep 30 - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID - fi - log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) - done - - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - while [ $? -ne 0 ] ; do - # in case the server isn't responding to ssh sessions yet - sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - done - scp -i $SERVERSSHKEY $DATADIR/distrib/${rundate}*.json ec2-user@$privip:data/distrib - - execcons=execconsol.sh - execConsolsh=/tmp/$execcons - python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh')" - chmod +x $execConsolsh - - log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib - scp -i $SERVERSSHKEY $execConsolsh ec2-user@$privip:data/distrib/$execcons - ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execcons" - - log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation" runDistrib - scp -i $SERVERSSHKEY ec2-user@$privip:data/distrib/processed_trajectories.json $DATADIR/distrib - - ssh -i $SERVERSSHKEY ec2-user@$privip "rm -f data/distrib/*.json /tmp/processed_trajectories.json" - # remote temporary files - ssh -i $SERVERSSHKEY ec2-user@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" - # prune trajdb folder on calcserver - ssh -i $SERVERSSHKEY ec2-user@$privip "find ~/ukmon-shared/matches/RMSCorrelate/trajdb -maxdepth 1 -name "*.json*" -mtime +30 -exec rm -f {} \;" - - log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib - aws ec2 stop-instances --instance-ids $SERVERINSTANCEID - - python -c "from traj.consolidateDistTraj import patchTrajDB ; patchTrajDB('$DATADIR/distrib/processed_trajectories.json','/home/ec2-user/ukmon-shared/matches/RMSCorrelate', '/home/ec2-user/data/distrib');" - - # archive older data then push the updated traj db to the S3 bucket - python -m traj.jsonDbMaintenance $DATADIR/distrib/ - aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - - log2cw $NJLOGGRP $NJLOGSTREAM "compressing the procssed data" runDistrib - gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz - aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet - tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib/done/ --quiet - rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 rm $UKMONSHAREDBUCKET/matches/distrib/ --exclude "*" --include "${rundate}*.json" --exclude "test/*" --recursive -else - log2cw $NJLOGGRP $NJLOGSTREAM "trajectory database is size zero... not proceeding with copy" runDistrib -fi -# and then clear the profile again -unset AWS_PROFILE -log2cw $NJLOGGRP $NJLOGSTREAM "finished runDistrib" runDistrib + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +done + +execcons=execconsol.sh +execConsolsh=/tmp/$execcons +python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', '$TESTMODE')" +chmod +x $execConsolsh + +logger -s -t runDistrib "running consolidation" + +scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execcons +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execcons" + +logger -s -t runDistrib "finished consolidation, copying databases" + +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" + +logger -s -t runDistrib "stopping calcserver again" +aws ec2 stop-instances --instance-ids $SERVERINSTANCEID + +logger -s -t runDistrib "copying data to batch server and tidying up" + +# grab a copy of the indvidual container dbs so we can get a list of new solutions +rm -Rf $DATADIR/latest/contdbs/ +mkdir -p $DATADIR/latest/contdbs/ +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/contdbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --exclude "test/*" --quiet +aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "*${rundate}*.db" --exclude "test/*" --exclude "dbs/*" --recursive --quiet +aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATADIR/distrib --quiet + +# grab a copy of the indvidual container logs - duplicated in monitorProgress, but never mind +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $SRC/logs/distrib/ --exclude "*" --include "correl*.log" --exclude "logs/" --quiet +aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "correl*.log" --exclude "logs/" --recursive --quiet + +mkdir -p $DATADIR/trajdb +tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db +mkdir -p $DATADIR/distrib/containers +tar czvf $DATADIR/distrib/containers/contdbs_${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle +aws s3 cp $DATADIR/distrib/containers/contdbs_${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet + +tar czvf $DATADIR/distrib/containers/contlogs_${rundate}.tgz $SRC/logs/distrib/correlator_${rundate}*.log $SRC/logs/distrib/${rundate}_*.log + +find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \; +find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \; +rm -f $DATADIR/distrib/${rundate}.pickle + +logger -s -t "finished runDistrib" diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index 948846cb8..1ef92a489 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -17,86 +17,67 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "starting updatePlotsAndDetStatus" updatePlotsAndDetStatus - -# set the profile to the EE account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared +logger -s -t updatePlotsAndDetStatus "starting updatePlotsAndDetStatus" if [ $# -gt 0 ] ; then if [ "$1" != "" ] ; then - log2cw $NJLOGGRP $NJLOGSTREAM "selecting range" updatePlotsAndDetStatus MATCHSTART=$1 fi if [ "$2" != "" ] ; then MATCHEND=$2 else - log2cw $NJLOGGRP $NJLOGSTREAM "matchend was not supplied, using 2 days" updatePlotsAndDetStatus MATCHEND=$(( $MATCHSTART - 2 )) fi fi begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "updating plots etc for dates ${begdate} to ${rundate}" +logger -s -t updatePlotsAndDetStatus "start correlation server" + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" updatePlotsAndDetStatus while [ "$stat" -ne 16 ]; do sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" updatePlotsAndDetStatus stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -log2cw $NJLOGGRP $NJLOGSTREAM "updating plots etc for dates ${begdate} to ${rundate}" updatePlotsAndDetStatus + conda activate $HOME/miniconda3/envs/${WMPL_ENV} -log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "creating the run script" execrerun=execreplot.sh execrerunsh=/tmp/$execrerun -python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh')" +python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh', '$TESTMODE')" chmod +x $execrerunsh -log2cw $NJLOGGRP $NJLOGSTREAM "get server details" updatePlotsAndDetStatus -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -while [ "$privip" == "" ] ; do - sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" updatePlotsAndDetStatus - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -done - -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "deploy the script to the server $CALCSERVERIP and run it" -scp -i $SERVERSSHKEY $execrerunsh ec2-user@$privip:data/distrib/$execrerun +scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" updatePlotsAndDetStatus - scp -i $SERVERSSHKEY $execrerunsh ec2-user@$privip:data/distrib/$execrerun + scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun done # push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py ec2-user@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj # now run the script -ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execrerun" +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execrerun" + +logger -s -t updatePlotsAndDetStatus "job run, stop the server again" -log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" updatePlotsAndDetStatus aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -log2cw $NJLOGGRP $NJLOGSTREAM "get a list of uncalibrated data" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "get a list of uncalibrated data" + aws s3 sync $UKMONSHAREDBUCKET/matches/consumed/ $DATADIR/single/used/ --exclude "*" --include "*.txt" --quiet rundate=$(cat $DATADIR/rundate.txt) -python -c "from analysis.gatherDetectionData import getUncalibratedImageList;getUncalibratedImageList('$rundate');" +python -c "from utils.getUncalImages import getUncalibratedImageList;getUncalibratedImageList('$rundate');" -# and then clear the profile again -unset AWS_PROFILE # refresh the website index pages just in case any new data dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) $SRC/website/updateIndexPages.sh $dailyrep -log2cw $NJLOGGRP $NJLOGSTREAM "finished updatePlotsAndDetStatus" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "finished updatePlotsAndDetStatus" diff --git a/archive/containers/trajsolver/Dockerfile b/archive/containers/trajsolver/Dockerfile index ff8b6ff5b..47f4ac3fa 100644 --- a/archive/containers/trajsolver/Dockerfile +++ b/archive/containers/trajsolver/Dockerfile @@ -11,6 +11,8 @@ RUN conda install -y matplotlib RUN pip install boto3 RUN apt-get update && apt-get -y upgrade RUN conda install -y -c conda-forge numba +RUN conda install -y -c conda-forge libsqlite sqlite +RUN conda install -y -c conda-forge paramiko COPY WesternMeteorPyLib/ ./WesternMeteorPyLib ENV PYTHONPATH=./WesternMeteorPyLib diff --git a/archive/containers/trajsolver/build.ps1 b/archive/containers/trajsolver/build.ps1 index 4a4dac048..c5bb2a3f5 100644 --- a/archive/containers/trajsolver/build.ps1 +++ b/archive/containers/trajsolver/build.ps1 @@ -15,7 +15,7 @@ $repo = "calcengine/${imagename}" write-output "building $imagename in $accid" $yn=read-host -prompt "update WMPL?" -if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } +if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } if ($env -eq "test") { move-item awskeys.test awskeys} diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py index 2d42835f7..e9fdd9a9a 100644 --- a/archive/containers/trajsolver/trajsolver.py +++ b/archive/containers/trajsolver/trajsolver.py @@ -6,6 +6,8 @@ import boto3 import datetime import tempfile +import glob +import logging from wmpl.Trajectory.CorrelateRMS import RMSDataHandle from wmpl.Utils.Math import generateDatetimeBins @@ -15,8 +17,10 @@ from wmpl.Trajectory.CorrelateRMS import TrajectoryReduced, DatabaseJSON # noqa: F401 from wmpl.Trajectory.CorrelateRMS import MeteorObsRMS, PlateparDummy, MeteorPointRMS # noqa: F401 +log = logging.getLogger('traj_correlator') # must be the same name as the WMPL logger -def runCorrelator(dir_path, time_beg, time_end): + +def runCorrelator(dir_path, time_beg, time_end, max_stns=9999): # Init trajectory constraints maxtoffset = 10.0 maxstationdist = 600.0 @@ -25,7 +29,7 @@ def runCorrelator(dir_path, time_beg, time_end): saveplots = True velpart = 0.40 uncerttime = False - distribute = 2 + distribute = 3 trajectory_constraints = TrajectoryConstraints() trajectory_constraints.max_toffset = maxtoffset @@ -34,6 +38,7 @@ def runCorrelator(dir_path, time_beg, time_end): trajectory_constraints.run_mc = not disablemc trajectory_constraints.save_plots = saveplots trajectory_constraints.geometric_uncert = not uncerttime + trajectory_constraints.max_stations = max_stns # Clock for measuring script time t1 = datetime.datetime.now(datetime.timezone.utc) @@ -42,8 +47,8 @@ def runCorrelator(dir_path, time_beg, time_end): event_time_range = None # Extract time range - dt_beg = datetime.datetime.strptime(time_beg, "%Y%m%d-%H%M%S") - dt_end = datetime.datetime.strptime(time_end, "%Y%m%d-%H%M%S") + dt_beg = datetime.datetime.strptime(time_beg, "%Y%m%d-%H%M%S").replace(tzinfo=datetime.timezone.utc) + dt_end = datetime.datetime.strptime(time_end, "%Y%m%d-%H%M%S").replace(tzinfo=datetime.timezone.utc) print("Custom time range:") print(" BEG: {:s}".format(str(dt_beg))) @@ -52,17 +57,7 @@ def runCorrelator(dir_path, time_beg, time_end): event_time_range = [dt_beg, dt_end] # Init the data handle - dh = RMSDataHandle(dir_path, event_time_range) - - # If there is nothing to process, stop, unless we're in distributed - # processing mode 2 - if not dh.processing_list and distribute !=2: - print() - print("Nothing to process!") - print("Probably everything is already processed.") - print("Exiting...") - sys.exit() - + dh = RMSDataHandle(dir_path, dt_range=event_time_range, mcmode=distribute) ### GENERATE MONTHLY TIME BINS ### @@ -108,25 +103,26 @@ def runCorrelator(dir_path, time_beg, time_end): print("-----------------------------") print() - # Load data of unprocessed observations - dh.unpaired_observations = dh.loadUnpairedObservations(dh.processing_list, dt_range=(bin_beg, bin_end)) - # Run the trajectory correlator - tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, distribute=distribute, enableOSM=True) - tc.run(event_time_range=event_time_range) + tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, enableOSM=True) + tc.run(event_time_range=event_time_range, mcmode=3) + dh.closeObservationsDatabase() + dh.closeTrajectoryDatabase() + print("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) + log.info("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) return # read the source bucket + folder and target buckets + folders from the environment def getSourceAndTargets(): - srcpth = os.getenv('SRCPATH', default='s3://ukmda-shared/matches/distrib') + srcpth = os.getenv('SRCPATH', default='s3://ukmda-shared/test/matches/distrib') srcpth = srcpth[5:] srcbucket = srcpth[:srcpth.find('/')] srcpth = srcpth[srcpth.find('/')+1:] - outpth = os.getenv('OUTPATH', default='s3://ukmda-shared/matches/distrib') + outpth = os.getenv('OUTPATH', default='s3://ukmda-shared/test/matches/RMSCorrelate') outpth = outpth[5:] outbucket = outpth[:outpth.find('/')] outpth = outpth[outpth.find('/')+1:] @@ -136,7 +132,9 @@ def getSourceAndTargets(): webbucket = webpth[:webpth.find('/')] webpth = webpth[webpth.find('/')+1:] - return srcbucket, srcpth, outbucket, outpth, webbucket, webpth + max_stns = int(os.getenv('MAX_STNS', default='9999')) + + return srcbucket, srcpth, outbucket, outpth, webbucket, webpth, max_stns # extra args for setting the MIME type when uploading to S3. @@ -162,6 +160,8 @@ def getExtraArgs(fname): ctyp = 'application/json' elif file_ext=='.zip': ctyp = 'application/zip' + elif file_ext=='.db': + ctyp = 'application/vnd.sqlite3' extraargs = {'ContentType': ctyp} return extraargs @@ -233,15 +233,37 @@ def getS3Client(): # starting point for the process def startup(srcfldr, startdt, enddt, isTest=False): + + localfldr = tempfile.mkdtemp() + + # Init the logger + log.setLevel(logging.DEBUG) + + # Init the log formatter + log_formatter = logging.Formatter( + fmt='%(asctime)s-%(levelname)-5s-%(module)-15s:%(lineno)-5d- %(message)s', + datefmt='%Y/%m/%d %H:%M:%S') + + # Init the file handler + datasetname = os.path.split(srcfldr)[-1] + log_file = os.path.join(localfldr, f'{datasetname}.log') + file_handler = logging.handlers.TimedRotatingFileHandler(log_file, when="midnight", backupCount=7) + file_handler.setFormatter(log_formatter) + log.addHandler(file_handler) + + # Init the console handler (i.e. print to console) + console_handler = logging.StreamHandler() + console_handler.setFormatter(log_formatter) + log.addHandler(console_handler) + print(f'processing {srcfldr}') print(f"Starting at {datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}") - localfldr = tempfile.mkdtemp() canddir = os.path.join(localfldr,'candidates') os.makedirs(canddir, exist_ok = True) s3 = getS3Client() - srcbucket, srcpth, outbucket, outpth, webbucket, webpth = getSourceAndTargets() + srcbucket, srcpth, outbucket, outpth, webbucket, webpth, max_stns = getSourceAndTargets() if isTest is True: outpth = os.path.join(outpth, 'test') @@ -249,7 +271,7 @@ def startup(srcfldr, startdt, enddt, isTest=False): print(f'fetching data from {srcbucket}/{srckey} saving to {outbucket} and {webbucket}') objlist = s3.meta.client.list_objects_v2(Bucket=srcbucket,Prefix=srckey) - print(objlist) + #print(objlist) if objlist['KeyCount'] > 0: keys = objlist['Contents'] for k in keys: @@ -257,24 +279,45 @@ def startup(srcfldr, startdt, enddt, isTest=False): if '.pickle' in fname: _, locfname = os.path.split(fname) targfile = os.path.join(canddir, locfname) - print(f'downloading {locfname}') + print(f'downloading {locfname} to {targfile}') s3.meta.client.download_file(srcbucket, fname, targfile) - runCorrelator(localfldr, startdt, enddt) + runCorrelator(localfldr, startdt, enddt, max_stns = max_stns) + + # flush and close the logs + file_handler.flush() + console_handler.flush() + logging.shutdown() print('uploading data to website') trajfldr = os.path.join(localfldr,'trajectories') - # reacquire tokens just in case the 1hour time limit on chained roles is exceeded + # reacquire tokens just in case the 1 hour time limit on chained roles is exceeded s3 = getS3Client() pushToWebsite(s3, trajfldr, webbucket, webpth, outbucket, outpth) - fname = f'{srcfldr}.json' - jsonfile = os.path.join(localfldr, 'processed_trajectories.json') - targkey = f'{srcpth}/{fname}' - print(f'uploading {jsonfile} to {srcbucket}/{srcpth}') - s3.meta.client.upload_file(jsonfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + + # databases get pushed back to source location not target + if srcfldr[:5] == 'test/': + srcfldr = srcfldr[5:] + for dbname in ['observations', 'trajectories', 'candidates']: + + fname = f'{dbname}_{srcfldr}.db' + localfile = os.path.join(localfldr, f'{dbname}.db') + if os.path.isfile(localfile): + targkey = f'{srcpth}/{fname}' + print(f'uploading {localfile} to {srcbucket}/{targkey}') + s3.meta.client.upload_file(localfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + + lognames = glob.glob(os.path.join(localfldr, '*.log')) + if len(lognames)> 0: + fname = f'correlator_{srcfldr}.log' + targkey = f'{srcpth}/{fname}' + print(f'uploading {lognames[0]} to {srcbucket}/{targkey}') + s3.meta.client.upload_file(lognames[0], srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + else: print('no files found') + print(f"Finished at {datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}") return diff --git a/archive/containers/trajsolver/update_wmpl.sh b/archive/containers/trajsolver/update_wmpl.sh index 5fa92c4b2..1be9f5dc5 100644 --- a/archive/containers/trajsolver/update_wmpl.sh +++ b/archive/containers/trajsolver/update_wmpl.sh @@ -8,9 +8,9 @@ else fi cd /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive/containers/$targ/ -ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout forcontainer && git pull" +ssh ukmonhelper2 "cd ~/src/wmpldev && git stash && git checkout distrib_processing && git pull" rsync -avz ukmonhelper2:src/wmpldev/* ./WesternMeteorPyLib/ --exclude "build/" --exclude "*.egg*" --exclude "dist/" --exclude ".git/" --exclude "__pycache__/" -ssh ukmonhelper2 "cd src/wmpldev && git checkout - && git stash apply" +ssh ukmonhelper2 "cd ~/src/wmpldev && git checkout - && git stash apply" # remove modules and code that don't work in a container because theres no GUI rm -Rf ./WesternMeteorPyLib/wmpl/MetSim diff --git a/archive/cronjobs/captureBright.sh b/archive/cronjobs/captureBright.sh index 06ce1561f..b06ed51d2 100644 --- a/archive/cronjobs/captureBright.sh +++ b/archive/cronjobs/captureBright.sh @@ -6,15 +6,9 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} cd $DATADIR/brightness rundt=$(date -d "yesterday" +%Y%m%d) -python -m analysis.compareBrightnessData $rundt +python -m utils.compareBrightnessData $rundt -mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD -use ukmon; -select count(*) from ukmon.brightness; -load data local infile '${DATADIR}/brightness/CaptureNight_${rundt}.csv' -into table brightness fields terminated by ','; -select count(*) from ukmon.brightness; -EOD +$SRC/utils/loadBrightnessCsvMDB.sh find ${DATADIR}/brightness -name "CaptureNight*" -mtime +30 -exec rm -f {} \; - +find ${DATADIR}/brightness -name "matcheddata*" -mtime +30 -exec rm -f {} \; \ No newline at end of file diff --git a/archive/cronjobs/gatherMonthlyVideos.sh b/archive/cronjobs/gatherMonthlyVideos.sh index 52f670486..e8c9c1464 100644 --- a/archive/cronjobs/gatherMonthlyVideos.sh +++ b/archive/cronjobs/gatherMonthlyVideos.sh @@ -70,7 +70,7 @@ echo "mkdir \$1 ; cd \$1" >> $outdir/getVideos.ps1 echo "foreach(\$line in Get-Content ../best_\$1.txt) { wget https://archive.ukmeteors.co.uk/\$line }" >> $outdir/getVideos.ps1 echo "cd .." >> $outdir/getVideos.ps1 -aws s3 sync $outdir $UKMONSHAREDBUCKET/videos/ --profile ukmonshared --quiet --exclude "*" --include "getVideos*" --include "README*" +aws s3 sync $outdir $UKMONSHAREDBUCKET/videos/ --quiet --exclude "*" --include "getVideos*" --include "README*" -aws s3 sync $outdir $WEBSITEBUCKET/data/bestvideos/ --profile ukmonshared --quiet --include "*.txt" +aws s3 sync $outdir $WEBSITEBUCKET/data/bestvideos/ --quiet --include "*.txt" diff --git a/archive/cronjobs/getImoWSfile.sh b/archive/cronjobs/getImoWSfile.sh index 021b6bad6..d7cb03996 100644 --- a/archive/cronjobs/getImoWSfile.sh +++ b/archive/cronjobs/getImoWSfile.sh @@ -19,7 +19,7 @@ git checkout wmpl/share/streamfulldata.csv #git checkout wmpl/share/ShowerLookUpTable.txt git checkout wmpl/share/gmn_shower_table_20230518.txt -python -c "from meteortools.utils.getShowerDates import numpifyShowerData; numpifyShowerData()" +python -c "from utils.getActiveShowers import numpifyShowerData; numpifyShowerData()" logger -s -t getImoWSfile "finished" diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index 7f184a5fd..5d63a02aa 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -7,12 +7,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) -export NJLOGSTREAM -aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared - -# log2cw is a function defined in config.ini and logs to cloudwatch, syslog and console -log2cw $NJLOGGRP $NJLOGSTREAM "start nightlyJob" nightlyJob +logger -s -t nightlyJob "start nightlyJob" # dates to process for rundate=$(date +%Y%m%d) @@ -25,117 +20,118 @@ mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} -# update the JSON file containing camera location details. This is used by the website -log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera location/dir/fov database" nightlyJob -python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();" -aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --profile ukmonshared --quiet -aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --profile ukmonshared --quiet - -# create the CSV file of camera info and the html versions for search functions on the website -log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera details files for searching" nightlyJob -python -c "from reports.CameraDetails import createCDCsv; createCDCsv('consolidated');" -aws s3 cp $DATADIR/consolidated/camera-details.csv $UKMONSHAREDBUCKET/consolidated/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/statopts.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/activestatopts.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/activestatlocs.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet +#################################################################################### +# START OF DATA PROCESSING. FIRST WE UPDATE THE SEARCH PAGE WITH SINGLE-STATION +# AND CAMERA DETAILS SO IT CAN BE SEARCHED WITHOUT WAITING FOR THE MATCHING ENGINE +#################################################################################### + +# update the JSON and html files containing camera and location details, used by the website +$SRC/website/updateCameraDets.sh + +# consolidate the single-station data - we do this here so the search index can be updated earlier +# and we can search for single-station events without waiting for the main batch +$SRC/analysis/getRMSSingleData.sh +if [ "$(date +%m%d)" == "0101" ] ; then + # catch any data uploaded on 01/01 that is for 31/12 the previous year + $SRC/analysis/getRMSSingleData.sh $(date -d 'last year' +%Y) +fi + +# now update the search index with single-station data +$SRC/analysis/createSearchable.sh $yr singles + +#################################################################################### +# NOW THE RUN THE MATCHING ENGINE VIA findAllmatches.sh +# Take care rerunning it as it will re-create the daily report +#################################################################################### # set up logging for the match process -log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" nightlyJob matchlog=matchJob.log -if [ -f $SRC/logs/$matchlog ] ; then - suff=$(stat matchJob.log -c %X) - mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff + +# save the existing log, in case the process is being rerun on the same day +if [ "$(find $SRC/logs -name $matchlog -mmin +1380 -ls)" != "" ] ; then + dt=$(stat $SRC/logs/$matchlog -c %y) + suff=$(date --date ${dt:0:10} +%Y%m%d) + mv -f $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff fi + +logger -s -t nightlyJob "start findAllMatches" # Run the match process - run this only once as it scoops up all unprocessed data ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 -# from here down, we're creating reports -# consolidate the output of the match process for further analysos -log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob + +#################################################################################### +# FROM HERE DOWN WE'RE CONSOLIDATING DATA AND CREATING REPORTS +# and everything can be rerun safely provided the data are present +#################################################################################### + +# update the website daily, monthly and annual index pages where needed +$SRC/website/updateIndexPages.sh + +# consolidate the output of the match process for further analysis $SRC/analysis/consolidateOutput.sh ${yr} +if [ "$(date +%m%d)" == "0101" ] ; then + # catch any data uploaded on 01/01 that is for 31/12 the previous year + $SRC/analysis/consolidateOutput.sh $(date -d 'last year' +%Y) +fi -# create the search indexes used on the website -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 2" nightlyJob +# update the search indexes used on the website $SRC/analysis/createSearchable.sh $yr matches # add daily report to the website -log2cw $NJLOGGRP $NJLOGSTREAM "start publishDailyReport" nightlyJob $SRC/website/publishDailyReport.sh # create monthly and per-shower CSV extracts of the data -log2cw $NJLOGGRP $NJLOGSTREAM "start createMthlyExtracts" nightlyJob ${SRC}/website/createMthlyExtracts.sh ${mth} - -log2cw $NJLOGGRP $NJLOGSTREAM "start createShwrExtracts" nightlyJob ${SRC}/website/createShwrExtracts.sh ${rundate} # create the fireballs page -log2cw $NJLOGGRP $NJLOGSTREAM "start createFireballPage" nightlyJob #requires search index to have been updated first ${SRC}/website/createFireballPage.sh ${yr} -3.99 # create a report of activity for the current month and whole year -log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $mth" nightlyJob $SRC/analysis/showerReport.sh ALL ${mth} force - -log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $yr" nightlyJob $SRC/analysis/showerReport.sh ALL ${yr} force # if we ran on the 1st of the month we need to catch any late-arrivals for last month if [ $(date +%d) -eq 1 ] ; then lastmth=$(date -d '-1 month' +%Y%m) - log2cw $NJLOGGRP $NJLOGSTREAM "start showerMthlyExtracts ALL $lastmth" nightlyJob ${SRC}/website/createMthlyExtracts.sh ${lastmth} - log2cw $NJLOGGRP $NJLOGSTREAM "start createShwrExtracts ALL $lastmth" nightlyJob ${SRC}/website/createShwrExtracts.sh ${lastmth}28 - log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $lastmth" nightlyJob $SRC/analysis/showerReport.sh ALL ${lastmth} force fi # create a per-shower report for any currently active showers -log2cw $NJLOGGRP $NJLOGSTREAM "start reportActiveShowers" nightlyJob ${SRC}/analysis/reportActiveShowers.sh ${yr} # create the website front page -log2cw $NJLOGGRP $NJLOGSTREAM "start createSummaryTable" nightlyJob ${SRC}/website/createSummaryTable.sh # create the camera status reports -log2cw $NJLOGGRP $NJLOGSTREAM "start cameraStatusReport" nightlyJob ${SRC}/website/cameraStatusReport.sh $rundate -log2cw $NJLOGGRP $NJLOGSTREAM "start createExchangeFiles" nightlyJob +# create the files for exchange with other networks - i think this is unused python -c "from reports.createExchangeFiles import createAll;createAll();" -aws s3 sync $DATADIR/browse/daily/ $WEBSITEBUCKET/browse/daily/ --region eu-west-2 --quiet +aws s3 sync $DATADIR/browse/daily/ $WEBSITEBUCKET/browse/daily/ --quiet cd $DATADIR # do this manually when on PC required as it requires too much memory for the batch server; closes #61 # python $PYLIB/maintenance/plotStationsOnMap.py False -aws s3 cp $DATADIR/stations.png $WEBSITEBUCKET/ --region eu-west-2 --quiet +aws s3 cp $DATADIR/stations.png $WEBSITEBUCKET/ --quiet rm -f $SRC/data/.nightly_running # various reports for management - bad stations, costs, next batch start time. -log2cw $NJLOGGRP $NJLOGSTREAM "start getBadStations" nightlyJob $SRC/analysis/getBadStations.sh - -log2cw $NJLOGGRP $NJLOGSTREAM "start costReport" nightlyJob $SRC/website/costReport.sh - -# set time of next run python $PYLIB/maintenance/getNextBatchStart.py 150 # create station reports. This takes a while hence done after everything else -log2cw $NJLOGGRP $NJLOGSTREAM "start stationReports" nightlyJob $SRC/analysis/stationReports.sh # clear down space where possible -log2cw $NJLOGGRP $NJLOGSTREAM "start clearSpace" nightlyJob $SRC/utils/clearSpace.sh # load the MariaDB with the latest data. The mariadb database isn't used much -log2cw $NJLOGGRP $NJLOGSTREAM "update MariaDB tables" nightlyJob $SRC/utils/loadMatchCsvMDB.sh $SRC/utils/loadSingleCsvMDB.sh @@ -143,12 +139,10 @@ $SRC/utils/loadSingleCsvMDB.sh # has to be run quite late as not all trajectories have synced to the website earlier $SRC/analysis/updatePlotsAndDetStatus.sh +# push the API data dictionary to the website for end-user use aws s3 sync $SRC/share/ s3://ukmda-website/browse --exclude "*" --include "datadictionary.xlsx" --quiet -log2cw $NJLOGGRP $NJLOGSTREAM "finished nightlyJob" nightlyJob +logger -s -t nightlyJog "finished nightlyJob" # grab the logs for the website - run this last to capture the above Finished message $SRC/analysis/getLogData.sh - - - diff --git a/archive/database/Setup.md b/archive/database/Setup.md index 1aef81b10..328cab345 100644 --- a/archive/database/Setup.md +++ b/archive/database/Setup.md @@ -1,42 +1,16 @@ # Installing MariaDB -## Create /etc/yum.repos.d/MariaDB.repo - -[mariadb] -name = MariaDB -baseurl = http://yum.mariadb.org/10.3/centos7-amd64 -gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB -gpgcheck=1 - -## Install packages - -yum update -yum install mariadb-server mariadb-client mariadb-devel mariadb-shared mariadb-common - -## Create Databases and Tables -``` -mysql -u root -h localhost -set root password with -update mysql.user set password= PASSWORD('newpassword') where user = 'root'; -flush privileges; -exit; -``` -execute contents of create_dbs_and_users.sql -execute contents of create_tables.sql - -## migrating to new server -make sure you've installed mariadb on the new server and created databases and users -make sure you can ssh from old to new -then on the old server run -``` -mysqldump -u batch -p'passwd' ukmon | ssh ukmonhelper2 mysql -u batch -p'passwd' ukmon -``` +## Installing and Migrating +For installation and migration instructions see `migratingBatchServer.md` in the `server_setup` folder ## Connecting from Python +These packages are installed as part of the ukmda tooling. +``` bash pip install mysql-connector-python pip install pymysql - +``` +``` python import pymysql.cursors connection = pymysql.connect(host='localhost', @@ -46,12 +20,13 @@ pip install pymysql cursorclass=pymysql.cursors.DictCursor) try: with connection.cursor() as cursor: - sql = "INSERT INTO books VALUES ({},'{}', {}, {})".format(sys.argv[1],'foo',34,56) + #sql = "INSERT INTO books VALUES ({},'{}', {}, {})".format(sys.argv[1],'foo',34,56) cursor.execute(sql) connection.commit() - sql = "SELECT * from `books` " + sql = "SELECT * from matches limit 10 " cursor.execute(sql) result = cursor.fetchall() print(result) finally: connection.close() +``` \ No newline at end of file diff --git a/archive/database/ddl/create_tables.sql b/archive/database/ddl/create_tables.sql index c51027569..ba036622e 100644 --- a/archive/database/ddl/create_tables.sql +++ b/archive/database/ddl/create_tables.sql @@ -28,8 +28,9 @@ create table singles( AngVel float, Shwr varchar(6), filname varchar(64), - Dtstamp double(18,6)), - status varchar(3); + Dtstamp double(18,6), + status varchar(3) + ); alter ignore table ukmon.singles add unique uniq_Dtstamp (Dtstamp); diff --git a/archive/deployment/analysis.yml b/archive/deployment/analysis.yml deleted file mode 100644 index 8e217f41d..000000000 --- a/archive/deployment/analysis.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/analysis exists - file: path={{destdir}}/analysis state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/analysis/updatePlotsAndDetStatus.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/consolidateOutput.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/createSearchable.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/findAllMatches.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getBadStations.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getLogData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getRMSSingleData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/onlyConsolDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/reportActiveShowers.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/runDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/showerReport.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/stationReports.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/templates/extracsv.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/match_hdr_full.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/UA_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/UO_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/ukmon-single.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/README.md', dest: '{{destdir}}/analysis', mode: '644', backup: no } diff --git a/archive/deployment/config.yml b/archive/deployment/config.yml deleted file mode 100644 index b9823e58a..000000000 --- a/archive/deployment/config.yml +++ /dev/null @@ -1,29 +0,0 @@ -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: dev-vars.yml - tags: dev - - name: import prod variables - include_vars: prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/utils exists - file: path={{destdir}}/utils state=directory - tags: [dev,prod] - - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/utils/makeConfig.sh', dest: '{{destdir}}/utils/', mode: '754', backup: yes } - - {src: '{{srcdir}}/utils/stopstart-calcengine.sh', dest: '{{destdir}}/utils/', mode: '754', backup: yes } - - {src: '{{srcdir}}/server_setup/.bashrc', dest: '/home/ec2-user', mode: '644', backup: yes } - - {src: '{{srcdir}}/server_setup/.bash_aliases', dest: '/home/ec2-user', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/.condaon', dest: '/home/ec2-user', mode: '644', backup: no } - - - name: create Config file - shell: '{{destdir}}/utils/makeConfig.sh {{env}}' - tags: [dev,prod] diff --git a/archive/deployment/cronjobs.yml b/archive/deployment/cronjobs.yml deleted file mode 100644 index 6a80c1b5e..000000000 --- a/archive/deployment/cronjobs.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/Dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/cronjobs exists - file: path={{destdir}}/cronjobs state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/cronjobs/nightlyJob.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/captureBright.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/getImoWSfile.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/mergeNewOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/deleteDuplicateOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/gatherMonthlyVideos.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/README.md', dest: '{{destdir}}/cronjobs', mode: '644', backup: no } diff --git a/archive/deployment/database.yml b/archive/deployment/database.yml deleted file mode 100644 index 1b75b1f5f..000000000 --- a/archive/deployment/database.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/database exists - file: path={{destdir}}/database state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - - {src: '{{srcdir}}/database/ddl/create_brightness_table.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/ddl/create_dbs_and_users.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/ddl/create_tables.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/dml/load_data.sql', dest: '{{destdir}}/database/dml', mode: '644', backup: no } - - {src: '{{srcdir}}/database/Setup.md', dest: '{{destdir}}/database', mode: '644', backup: no } - - {src: '{{srcdir}}/database/testDB.py', dest: '{{destdir}}/database', mode: '644', backup: no } - - {src: '{{srcdir}}/database/README.md', dest: '{{destdir}}/database', mode: '644', backup: no } diff --git a/archive/deployment/deploy-usermgmt.yml b/archive/deployment/deploy-usermgmt.yml deleted file mode 100644 index 8429780d3..000000000 --- a/archive/deployment/deploy-usermgmt.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive/usermgmt" - destdir: /home/ec2-user/keymgmt - tasks: - - name: Ensures {{destdir}} exists - file: path={{destdir}} state=directory - - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - with_items: - - {src: '{{srcdir}}/addSftpUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/updateAwsKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/addAdminUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/addFBApiKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } diff --git a/archive/deployment/dev-vars.yml b/archive/deployment/dev-vars.yml deleted file mode 100644 index fb60fbb1d..000000000 --- a/archive/deployment/dev-vars.yml +++ /dev/null @@ -1,6 +0,0 @@ -destdir: /home/ec2-user/dev -env: DEV -websitebucket: mjmm-ukmonarchive.co.uk -#api_id: 0zbnc358p0 -#api_svc: test -s3profile: default diff --git a/archive/deployment/full_deployment.yml b/archive/deployment/full_deployment.yml deleted file mode 100644 index b682b18d1..000000000 --- a/archive/deployment/full_deployment.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- import_playbook: cronjobs.yml -- import_playbook: analysis.yml -- import_playbook: website.yml -- import_playbook: database.yml -- import_playbook: pylib.yml -- import_playbook: server_setup.yml -- import_playbook: shwrinfo.yml -- import_playbook: utils.yml diff --git a/archive/deployment/prod-vars.yml b/archive/deployment/prod-vars.yml deleted file mode 100644 index d26be9990..000000000 --- a/archive/deployment/prod-vars.yml +++ /dev/null @@ -1,6 +0,0 @@ -destdir: /home/ec2-user/prod -env: PROD -websitebucket: ukmda-website -#api_id: 40luvfh1od -#api_svc: Prod -s3profile: ukmda_admin \ No newline at end of file diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml deleted file mode 100644 index 46a4afc71..000000000 --- a/archive/deployment/pylib.yml +++ /dev/null @@ -1,110 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/ukmon_pylib exists - file: path={{destdir}}/ukmon_pylib state=directory - tags: [dev,prod] - - name: clean local filesystem - ansible.builtin.command: 'pyclean {{srcdir}}/ukmon_pylib -v --debris all' - delegate_to: localhost - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - - {src: '{{srcdir}}/ukmon_pylib/analysis/compareBrightnessData.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/gatherDetectionData.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/meteorFlux.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/showerAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/stationAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/summaryAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/__init__.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/README.md', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/converters/gmnTxtToPandas.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/toParquet.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/__init__.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/README.md', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/maintenance/compressOldArchiveData.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/cleanDdbTables.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/dataMaintenance.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/deDuplicate.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/getNextBatchStart.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/getUserAndKeyInfo.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/hcVaultActions.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/manageTraj.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/plotStationsOnMap.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/purgeOldImages.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/purgeOldOrbits.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/recreateOrbitPages.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/rerunFailedLambdas.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/sortToDateDirs.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/README.md', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/metrics/camMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/costMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/getMatchStats.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/statsToMqtt.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/timingMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/volumeMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/README.md', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/reports/CameraDetails.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/cameraStatusReport.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createAnnualBarChart.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createExchangeFiles.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createSearchableFormat.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createSummaryTable.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/dailyReport.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/extractors.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/filterBylocDirBri.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findBestMp4s.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findFailedMatches.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findFireballs.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/getLivestreamData.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/getSolutionStati.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/makeCoverageMap.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportActiveShowers.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportBadCameras.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportOfLatestMatches.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/trackThumbnails.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/__init__.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/README.md', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/requirements.txt', dest: '{{destdir}}/ukmon_pylib', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/README.md', dest: '{{destdir}}/ukmon_pylib', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BM.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMNG_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMUK_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMWE_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BM_medres.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/images.json', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/population.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/README.md', dest: '{{destdir}}/ukmon_pylib/share', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mm.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/consolidateDistTraj.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/createDistribMatchingSh.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/distributeCandidates.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/gatherMissedPlatepars.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/manualBulkReruns.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/pickleAnalyser.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/plotOSMGroundTrack.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/ShowerAssociation.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/README.md', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } diff --git a/archive/deployment/server_setup.yml b/archive/deployment/server_setup.yml deleted file mode 100644 index f621dc278..000000000 --- a/archive/deployment/server_setup.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/server_setup exists - file: path={{destdir}}/server_setup state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/server_setup/copyTestData.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-cartopy.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-geos.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-proj4.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-sqlite3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install_opencv.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/migratingBatchServer.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/newserver.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/remount-s3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/ssh-setup.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/yum-installs.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/README.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } diff --git a/archive/deployment/shwrinfo.yml b/archive/deployment/shwrinfo.yml deleted file mode 100644 index 17a6110ee..000000000 --- a/archive/deployment/shwrinfo.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/data/shwrinfo exists - file: path={{destdir}}/data/shwrinfo state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/shwrinfo/AUR.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/CAP.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/COM.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/costnotes.html', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/DAD.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/ETA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/GEM.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/HYD.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/KCG.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/LEO.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/LYR.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/NOO.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/NTA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/ORI.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/PER.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/QUA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/SDA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/STA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/template.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/URS.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/README.md', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } diff --git a/archive/deployment/utils.yml b/archive/deployment/utils.yml deleted file mode 100644 index f122d7cb5..000000000 --- a/archive/deployment/utils.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/utils exists - file: path={{destdir}}/utils state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/utils/calcserver/clearSpace.sh', dest: '{{destdir}}/utils/calcserver', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/cftpd_templ.json', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/checkAndRollKeys.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/clearCaches.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/clearSpace.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/createTestDataSet.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/deleteOrbit.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/getCostMetrics.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/liveevent.template', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/loadMatchCsvMDB.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/loadSingleCsvMDB.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/makeConfig.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/monthlyCleardown.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/rerunFTPtoUKMONlambda.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/rerunGetExtraFiles.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/ses-msg-templ.json', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/statsToMqtt.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/stopstart-calcengine.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateDb.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateFireballFlag.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateFireballImage.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/userAudit.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/README.md', dest: '{{destdir}}/utils', mode: '644', backup: no } diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml deleted file mode 100644 index 1ee42364e..000000000 --- a/archive/deployment/website.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/website exists - file: path={{destdir}}/website state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/website/cameraStatusReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/costReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createFireballPage.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createMthlyExtracts.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createOrbitIndex.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createReportIndex.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createShwrExtracts.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createSummaryTable.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/publishDailyReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/pushStatic.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/README.md', dest: '{{destdir}}/website', mode: '644', backup: no } - - {src: '{{srcdir}}/website/updateIndexPages.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - - {src: '{{srcdir}}/website/templates/coverage-maps.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/fbreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/footer.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/frontpage.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/header.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/reportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/searchdialog.js', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/shwrcsvindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/statreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } diff --git a/archive/deployment/website_static.yml b/archive/deployment/website_static.yml deleted file mode 100644 index cbe146679..000000000 --- a/archive/deployment/website_static.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/static_content exists - file: path={{destdir}}/static_content state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/static_content/browse/', dest: '{{destdir}}/static_content/browse', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/css/', dest: '{{destdir}}/static_content/css', mode: '754', backup: no, directory_mode: yes } - #- {src: '{{srcdir}}/static_content/data/', dest: '{{destdir}}/static_content/data', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/docs/', dest: '{{destdir}}/static_content/docs', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/fonts/', dest: '{{destdir}}/static_content/fonts', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/img/', dest: '{{destdir}}/static_content/img', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/js/', dest: '{{destdir}}/static_content/js', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/latest/', dest: '{{destdir}}/static_content/latest', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/live/', dest: '{{destdir}}/static_content/live', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/reports/', dest: '{{destdir}}/static_content/reports', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/search/', dest: '{{destdir}}/static_content/search', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/templates/', dest: '{{destdir}}/static_content/templates', mode: '754', backup: no, directory_mode: yes } - - - {src: '{{srcdir}}/static_content/favicon.ico', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/fundraising.html', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/privacy_data.html', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/README.md', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/robots.txt', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - - #- name: Update Static Content on the S3 website bucket - # shell: '{{destdir}}/website/pushStatic.sh {{env}}' - # tags: [dev,prod] diff --git a/archive/samfunctions/README.md b/archive/lambdas/README.md similarity index 100% rename from archive/samfunctions/README.md rename to archive/lambdas/README.md diff --git a/archive/samfunctions/camDetails/camDetails.py b/archive/lambdas/camDetails/camDetails.py similarity index 100% rename from archive/samfunctions/camDetails/camDetails.py rename to archive/lambdas/camDetails/camDetails.py diff --git a/archive/samfunctions/camDetails/samconfig.toml b/archive/lambdas/camDetails/samconfig.toml similarity index 100% rename from archive/samfunctions/camDetails/samconfig.toml rename to archive/lambdas/camDetails/samconfig.toml diff --git a/archive/samfunctions/camDetails/template.yml b/archive/lambdas/camDetails/template.yml similarity index 100% rename from archive/samfunctions/camDetails/template.yml rename to archive/lambdas/camDetails/template.yml diff --git a/archive/samfunctions/fetchECSV/Math.py b/archive/lambdas/fetchECSV/Math.py similarity index 100% rename from archive/samfunctions/fetchECSV/Math.py rename to archive/lambdas/fetchECSV/Math.py diff --git a/archive/samfunctions/fetchECSV/MeteorObservation.py b/archive/lambdas/fetchECSV/MeteorObservation.py similarity index 100% rename from archive/samfunctions/fetchECSV/MeteorObservation.py rename to archive/lambdas/fetchECSV/MeteorObservation.py diff --git a/archive/samfunctions/fetchECSV/fetchECSV.py b/archive/lambdas/fetchECSV/fetchECSV.py similarity index 100% rename from archive/samfunctions/fetchECSV/fetchECSV.py rename to archive/lambdas/fetchECSV/fetchECSV.py diff --git a/archive/samfunctions/fetchECSV/ftpDetectInfo.py b/archive/lambdas/fetchECSV/ftpDetectInfo.py similarity index 100% rename from archive/samfunctions/fetchECSV/ftpDetectInfo.py rename to archive/lambdas/fetchECSV/ftpDetectInfo.py diff --git a/archive/samfunctions/fetchECSV/requirements.txt b/archive/lambdas/fetchECSV/requirements.txt similarity index 100% rename from archive/samfunctions/fetchECSV/requirements.txt rename to archive/lambdas/fetchECSV/requirements.txt diff --git a/archive/samfunctions/fetchECSV/samconfig.toml b/archive/lambdas/fetchECSV/samconfig.toml similarity index 100% rename from archive/samfunctions/fetchECSV/samconfig.toml rename to archive/lambdas/fetchECSV/samconfig.toml diff --git a/archive/samfunctions/fetchECSV/template.yml b/archive/lambdas/fetchECSV/template.yml similarity index 100% rename from archive/samfunctions/fetchECSV/template.yml rename to archive/lambdas/fetchECSV/template.yml diff --git a/archive/samfunctions/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt b/archive/lambdas/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt similarity index 100% rename from archive/samfunctions/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt rename to archive/lambdas/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt diff --git a/archive/samfunctions/fetchECSV/tests/platepars_all_recalibrated.json b/archive/lambdas/fetchECSV/tests/platepars_all_recalibrated.json similarity index 100% rename from archive/samfunctions/fetchECSV/tests/platepars_all_recalibrated.json rename to archive/lambdas/fetchECSV/tests/platepars_all_recalibrated.json diff --git a/archive/samfunctions/fireballApi/getFireballFiles.py b/archive/lambdas/fireballApi/getFireballFiles.py similarity index 100% rename from archive/samfunctions/fireballApi/getFireballFiles.py rename to archive/lambdas/fireballApi/getFireballFiles.py diff --git a/archive/samfunctions/fireballApi/requirements.txt b/archive/lambdas/fireballApi/requirements.txt similarity index 100% rename from archive/samfunctions/fireballApi/requirements.txt rename to archive/lambdas/fireballApi/requirements.txt diff --git a/archive/samfunctions/fireballApi/samconfig.toml b/archive/lambdas/fireballApi/samconfig.toml similarity index 100% rename from archive/samfunctions/fireballApi/samconfig.toml rename to archive/lambdas/fireballApi/samconfig.toml diff --git a/archive/samfunctions/fireballApi/template.yml b/archive/lambdas/fireballApi/template.yml similarity index 100% rename from archive/samfunctions/fireballApi/template.yml rename to archive/lambdas/fireballApi/template.yml diff --git a/archive/samfunctions/ftpToUkmon/.gitignore b/archive/lambdas/ftpToUkmon/.gitignore similarity index 100% rename from archive/samfunctions/ftpToUkmon/.gitignore rename to archive/lambdas/ftpToUkmon/.gitignore diff --git a/archive/samfunctions/ftpToUkmon/README.md b/archive/lambdas/ftpToUkmon/README.md similarity index 100% rename from archive/samfunctions/ftpToUkmon/README.md rename to archive/lambdas/ftpToUkmon/README.md diff --git a/archive/samfunctions/ftpToUkmon/__init__.py b/archive/lambdas/ftpToUkmon/__init__.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/__init__.py rename to archive/lambdas/ftpToUkmon/__init__.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/Dockerfile b/archive/lambdas/ftpToUkmon/pythoncode/Dockerfile similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/Dockerfile rename to archive/lambdas/ftpToUkmon/pythoncode/Dockerfile diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/ShowerAssociation.py b/archive/lambdas/ftpToUkmon/pythoncode/ShowerAssociation.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/ShowerAssociation.py rename to archive/lambdas/ftpToUkmon/pythoncode/ShowerAssociation.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/Showers.py b/archive/lambdas/ftpToUkmon/pythoncode/Showers.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/Showers.py rename to archive/lambdas/ftpToUkmon/pythoncode/Showers.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/__init__.py b/archive/lambdas/ftpToUkmon/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/__init__.py rename to archive/lambdas/ftpToUkmon/pythoncode/__init__.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/established_showers.csv b/archive/lambdas/ftpToUkmon/pythoncode/established_showers.csv similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/established_showers.csv rename to archive/lambdas/ftpToUkmon/pythoncode/established_showers.csv diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/ftpToUkmon1.py b/archive/lambdas/ftpToUkmon/pythoncode/ftpToUkmon1.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/ftpToUkmon1.py rename to archive/lambdas/ftpToUkmon/pythoncode/ftpToUkmon1.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/requirements.txt b/archive/lambdas/ftpToUkmon/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/requirements.txt rename to archive/lambdas/ftpToUkmon/pythoncode/requirements.txt diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/supportFuncs.py b/archive/lambdas/ftpToUkmon/pythoncode/supportFuncs.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/supportFuncs.py rename to archive/lambdas/ftpToUkmon/pythoncode/supportFuncs.py diff --git a/archive/samfunctions/ftpToUkmon/samconfig.toml b/archive/lambdas/ftpToUkmon/samconfig.toml similarity index 100% rename from archive/samfunctions/ftpToUkmon/samconfig.toml rename to archive/lambdas/ftpToUkmon/samconfig.toml diff --git a/archive/samfunctions/ftpToUkmon/template.yaml b/archive/lambdas/ftpToUkmon/template.yaml similarity index 100% rename from archive/samfunctions/ftpToUkmon/template.yaml rename to archive/lambdas/ftpToUkmon/template.yaml diff --git a/archive/samfunctions/ftpToUkmon/tests/expected_results.txt b/archive/lambdas/ftpToUkmon/tests/expected_results.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/expected_results.txt rename to archive/lambdas/ftpToUkmon/tests/expected_results.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/localTest.ps1 b/archive/lambdas/ftpToUkmon/tests/localTest.ps1 similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/localTest.ps1 rename to archive/lambdas/ftpToUkmon/tests/localTest.ps1 diff --git a/archive/samfunctions/ftpToUkmon/tests/new_data.txt b/archive/lambdas/ftpToUkmon/tests/new_data.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/new_data.txt rename to archive/lambdas/ftpToUkmon/tests/new_data.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/remoteResult.txt b/archive/lambdas/ftpToUkmon/tests/remoteResult.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/remoteResult.txt rename to archive/lambdas/ftpToUkmon/tests/remoteResult.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/remoteTest.ps1 b/archive/lambdas/ftpToUkmon/tests/remoteTest.ps1 similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/remoteTest.ps1 rename to archive/lambdas/ftpToUkmon/tests/remoteTest.ps1 diff --git a/archive/samfunctions/ftpToUkmon/tests/testEvent.json b/archive/lambdas/ftpToUkmon/tests/testEvent.json similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/testEvent.json rename to archive/lambdas/ftpToUkmon/tests/testEvent.json diff --git a/archive/samfunctions/getApiDetails.ps1 b/archive/lambdas/getApiDetails.ps1 similarity index 100% rename from archive/samfunctions/getApiDetails.ps1 rename to archive/lambdas/getApiDetails.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/.gitignore b/archive/lambdas/getExtraFilesV2/.gitignore similarity index 100% rename from archive/samfunctions/getExtraFilesV2/.gitignore rename to archive/lambdas/getExtraFilesV2/.gitignore diff --git a/archive/samfunctions/getExtraFilesV2/README.md b/archive/lambdas/getExtraFilesV2/README.md similarity index 100% rename from archive/samfunctions/getExtraFilesV2/README.md rename to archive/lambdas/getExtraFilesV2/README.md diff --git a/archive/samfunctions/getExtraFilesV2/__init__.py b/archive/lambdas/getExtraFilesV2/__init__.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/__init__.py rename to archive/lambdas/getExtraFilesV2/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile similarity index 87% rename from archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile rename to archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile index 26f9034fa..03a07a133 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile +++ b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile @@ -6,10 +6,11 @@ COPY requirements.txt ./ RUN python -m pip install -r requirements.txt -t . RUN python -m pip install numba watchdog basemap-data-hires -t . RUN python -m pip install cartopy -t . -#RUN yum update -y +RUN python -m pip install paramiko -t . + ENV LD_LIBRARY_PATH=./ COPY WesternMeteorPyLib/ ./WesternMeteorPyLib -#COPY *.npy ./WesternMeteorPyLib/wmpl/share/ + COPY *.py ./ ENV PYTHONPATH=./WesternMeteorPyLib ENV PROJ_LIB=./ diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/__init__.py b/archive/lambdas/getExtraFilesV2/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/__init__.py rename to archive/lambdas/getExtraFilesV2/pythoncode/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py b/archive/lambdas/getExtraFilesV2/pythoncode/createOrbitPageIndex.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py rename to archive/lambdas/getExtraFilesV2/pythoncode/createOrbitPageIndex.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py similarity index 85% rename from archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py rename to archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py index 8c71bd7b3..83fb956cc 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py +++ b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py @@ -36,10 +36,15 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): yr = orbname[:4] ym = orbname[:6] ymd= orbname[:8] + istest = False if int(yr) > 2020: webpth = f'reports/{yr}/orbits/{ym}/{ymd}/{orbname}/' else: webpth = f'reports/{yr}/orbits/{ym}/{orbname}/' + + if 'test' in key: + webpth = f'dummy/{yr}/orbits/{ym}/{ymd}/{orbname}/' + istest = True outdir = os.path.join(tmpdir, orbname) try: @@ -162,7 +167,7 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): pass print('pushing files back') - pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3) + pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3, istest=istest) print('updating the index page') createOrbitPageIndex(outdir, websitebucket, s3) @@ -241,7 +246,7 @@ def findOtherFiles(evtdt, archbucket, websitebucket, outdir, fldr, s3): return -def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): +def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3, istest=False): flist = os.listdir(outdir) _, pth =os.path.split(outdir) @@ -253,26 +258,32 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): else: webpth = f'reports/{yr}/orbits/{ym}/{pth}/' + if istest: + webpth = f'dummy/{yr}/orbits/{ym}/{ymd}/{pth}/' + + for f in flist: print(f) locfname = os.path.join(outdir, f) + extraargs = getExtraArgs(locfname) # some files need to be pushed to the website, some to the archive bucket if '3dtrack' in f or '2dtrack' in f: key = os.path.join(webpth, f) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, websitebucket, key, ExtraArgs=extraargs) elif '.lst' in f: - key = os.path.join(f'matches/RMSCorrelate/trajectories/{yr}/{ym}/{ymd}/{pth}', f) - extraargs = getExtraArgs(locfname) + if istest: + key = os.path.join(f'matches/distrib/test/trajectories/{yr}/{ym}/{ymd}/{pth}', f) + else: + key = os.path.join(f'matches/RMSCorrelate/trajectories/{yr}/{ym}/{ymd}/{pth}', f) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'summary' in f: key = os.path.join(fldr, f) - # print(locfname, key) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'orbit_full.csv' in f: - key = os.path.join(f'matches/{yr}/fullcsv', f) - extraargs = getExtraArgs(locfname) + if istest: + key = os.path.join(f'matches/distrib/test/{yr}/fullcsv', f) + else: + key = os.path.join(f'matches/{yr}/fullcsv', f) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) return @@ -345,38 +356,8 @@ def getExtraArgs(fname): def lambda_handler(event, context): - # NB: IN THE CONSOLE, ADD TRUST RELATIONSHIP - # "AWS": "arn:aws:iam::317976261112:root" - # on the target role (replace by this lambda's ARN). - - # Note: if the lambda is connected to a VPC you must - # create a private subnet for the lambda - # create a NAT gateway on a public subnet - # create a routing table on the VPC to send 0.0.0.0/0 to the NAT gw - # attach the new subnet to the NAT gw - # - - sts_client = boto3.client('sts') - response = sts_client.get_caller_identity()['Account'] - if response == '317976261112': - assumed_role_object=sts_client.assume_role( - RoleArn="arn:aws:iam::183798037734:role/service-role/S3FullAccess", - RoleSessionName="GetExtraFilesV2") - - credentials=assumed_role_object['Credentials'] - - # Use the temporary credentials that AssumeRole returns to connections - s3 = boto3.resource('s3', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) - ddb = boto3.resource('dynamodb', region_name='eu-west-2', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) - else: - s3 = boto3.resource('s3') - ddb = boto3.resource('dynamodb', region_name='eu-west-2') + s3 = boto3.resource('s3') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') websitebucket = os.getenv('WEBSITEBUCKET') if websitebucket[:3] == 's3:': diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/pickleAnalysis.py b/archive/lambdas/getExtraFilesV2/pythoncode/pickleAnalysis.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/pickleAnalysis.py rename to archive/lambdas/getExtraFilesV2/pythoncode/pickleAnalysis.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/requirements.txt b/archive/lambdas/getExtraFilesV2/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/requirements.txt rename to archive/lambdas/getExtraFilesV2/pythoncode/requirements.txt diff --git a/archive/samfunctions/getExtraFilesV2/samconfig.toml b/archive/lambdas/getExtraFilesV2/samconfig.toml similarity index 100% rename from archive/samfunctions/getExtraFilesV2/samconfig.toml rename to archive/lambdas/getExtraFilesV2/samconfig.toml diff --git a/archive/samfunctions/getExtraFilesV2/template.yaml b/archive/lambdas/getExtraFilesV2/template.yaml similarity index 100% rename from archive/samfunctions/getExtraFilesV2/template.yaml rename to archive/lambdas/getExtraFilesV2/template.yaml diff --git a/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/localTest.ps1 similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 rename to archive/lambdas/getExtraFilesV2/tests/localTest.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 similarity index 97% rename from archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 rename to archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 index 0a8098317..fa8c832dd 100644 --- a/archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 +++ b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 @@ -15,4 +15,3 @@ else { aws lambda invoke --profile ukmonshared --function-name getExtraOrbitFilesV2 --log-type Tail --cli-binary-format raw-in-base64-out --payload file://tests/dummy.json --region eu-west-2 ./ftpdetect.log Remove-Item tests\dummy.json } - diff --git a/archive/samfunctions/getExtraFilesV2/tests/templateEvent.json b/archive/lambdas/getExtraFilesV2/tests/templateEvent.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/templateEvent.json rename to archive/lambdas/getExtraFilesV2/tests/templateEvent.json diff --git a/archive/samfunctions/getExtraFilesV2/tests/testEvent.json b/archive/lambdas/getExtraFilesV2/tests/testEvent.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/testEvent.json rename to archive/lambdas/getExtraFilesV2/tests/testEvent.json diff --git a/archive/samfunctions/getExtraFilesV2/tests/testEvent2.json b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json similarity index 60% rename from archive/samfunctions/getExtraFilesV2/tests/testEvent2.json rename to archive/lambdas/getExtraFilesV2/tests/testEvent2.json index fd940082b..9aa49b7f2 100644 --- a/archive/samfunctions/getExtraFilesV2/tests/testEvent2.json +++ b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json @@ -7,7 +7,7 @@ "arn": "arn:aws:s3:::ukmda-shared" }, "object": { - "key": "matches/RMSCorrelate/trajectories/2024/202407/20240714/20240714_000041.319_UK/20240714_000041_trajectory.pickle" + "key": "matches/distrib/test/trajectories/2026/202604/20260412/20260412_232324.162_UK/20260412_232324_trajectory.pickle" } } } diff --git a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh b/archive/lambdas/getExtraFilesV2/update_wmpl.sh similarity index 68% rename from archive/samfunctions/getExtraFilesV2/update_wmpl.sh rename to archive/lambdas/getExtraFilesV2/update_wmpl.sh index 57d3fb6fa..002e1f93d 100644 --- a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh +++ b/archive/lambdas/getExtraFilesV2/update_wmpl.sh @@ -4,14 +4,14 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" cd $here/pythoncode -ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout forcontainer && git pull" +ssh ukmonhelper2 "cd ~/src/wmpldev && git stash && git checkout distrib_processing && git pull" rsync -avz ukmonhelper2:src/wmpldev/* ./WesternMeteorPyLib/ --exclude "build/" --exclude "*.egg*" --exclude "dist/" --exclude ".git/" --exclude "__pycache__/" ssh ukmonhelper2 "cd src/wmpldev && git checkout - && git stash apply" -mv -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so.0 ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so +rm -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so +cp -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so.0 ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so # remove modules and code that don't work in a container because theres no GUI rm -Rf ./WesternMeteorPyLib/wmpl/MetSim rm -Rf ./WesternMeteorPyLib/wmpl/CAMO rm -f ./WesternMeteorPyLib/wmpl/Utils/DynamicMassFit.py -rm -f ./WesternMeteorPyLib/wmpl/Utils/PlotMap_OSM.py -cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py +#cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py b/archive/lambdas/getExtraFilesV2/wmpl__init__.py_fixed.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py rename to archive/lambdas/getExtraFilesV2/wmpl__init__.py_fixed.py diff --git a/archive/samfunctions/getLiveImages/getLiveImages.py b/archive/lambdas/getLiveImages/getLiveImages.py similarity index 90% rename from archive/samfunctions/getLiveImages/getLiveImages.py rename to archive/lambdas/getLiveImages/getLiveImages.py index 164ecc605..5d589131d 100644 --- a/archive/samfunctions/getLiveImages/getLiveImages.py +++ b/archive/lambdas/getLiveImages/getLiveImages.py @@ -16,7 +16,10 @@ def getLiveImages(dtstr, ddb=None): resp = table.query(IndexName='month-image_name-index', KeyConditionExpression=Key('month').eq(dtstr[4:6]) & Key('image_name').begins_with(f'M{dtstr}'), ProjectionExpression='image_name') - return resp + resp2 = table.query(IndexName='month-image_name-index', + KeyConditionExpression=Key('month').eq(dtstr[4:6]) & Key('image_name').begins_with(f'FF_{dtstr}'), + ProjectionExpression='image_name') + return resp['Items'] + resp2['Items'] def filterImages(d1, d2, statid=None, maxitems=-1, ddb=None): @@ -103,10 +106,10 @@ def lambda_handler(event, context): patt = qs['pattern'] print(f'searching for {patt}') ecsvstr = getLiveImages(patt, ddb=ddb) - print(f"found {ecsvstr['Items']}") + print(f"found {len(ecsvstr)} items") return { 'statusCode': 200, - 'body': json.dumps(ecsvstr['Items']) + 'body': json.dumps(ecsvstr) } else: if 'dtstr' in qs: @@ -119,7 +122,7 @@ def lambda_handler(event, context): if 'enddtstr' in qs: dtstr2 = qs['enddtstr'] else: - dtstr2 = datetime.datetime.now().strftime('M%Y%m%d_%H') + dtstr2 = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.000Z') statid = None if 'statid' in qs: statid = qs['statid'] diff --git a/archive/samfunctions/getLiveImages/requirements.txt b/archive/lambdas/getLiveImages/requirements.txt similarity index 100% rename from archive/samfunctions/getLiveImages/requirements.txt rename to archive/lambdas/getLiveImages/requirements.txt diff --git a/archive/samfunctions/getLiveImages/samconfig.toml b/archive/lambdas/getLiveImages/samconfig.toml similarity index 100% rename from archive/samfunctions/getLiveImages/samconfig.toml rename to archive/lambdas/getLiveImages/samconfig.toml diff --git a/archive/samfunctions/getLiveImages/template.yml b/archive/lambdas/getLiveImages/template.yml similarity index 100% rename from archive/samfunctions/getLiveImages/template.yml rename to archive/lambdas/getLiveImages/template.yml diff --git a/archive/samfunctions/matchDataApi/localTest.ps1 b/archive/lambdas/matchDataApi/localTest.ps1 similarity index 100% rename from archive/samfunctions/matchDataApi/localTest.ps1 rename to archive/lambdas/matchDataApi/localTest.ps1 diff --git a/archive/samfunctions/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py similarity index 82% rename from archive/samfunctions/matchDataApi/matchDataApi.py rename to archive/lambdas/matchDataApi/matchDataApi.py index b7c2023ef..73c393db9 100644 --- a/archive/samfunctions/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -9,6 +9,33 @@ import pymysql.cursors +def fileToJsonString(flis): + hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', + 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', + 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] + ptsarray='[' + gotpts = False + for fli in flis: + if 'Points' in fli: + gotpts = True + continue + elif '------' in fli or ' No' in fli: + continue + elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): + gotpts=False + break + elif gotpts is True: + spls = fli.split(',') + thisrow = '{' + for h, s in zip(hdr, spls): + thisrow = thisrow + f'"{h}": "{s.strip()}",' + thisrow = thisrow[:-1] + '},' + ptsarray = ptsarray + thisrow + ptsarray = ptsarray[:-1] + ']' + ptsarray = '{' + f'"points": {ptsarray}' + '}' + return ptsarray + + def getSqlLoginDetails(): # retrieve password and host from SSM. This allows me to manage them from Terraform ssm = boto3.client('ssm', region_name='eu-west-1') @@ -22,15 +49,25 @@ def getSqlLoginDetails(): return host, user, password, db -def getStationData(statid, dtstr): +def periodToSqlFragment(period): + if period == 'am': + frag = "and s._h_ut < 12" + elif period == 'pm': + frag = "and s._h_ut >= 12" + elif '-' in period: + sh,eh = period.split('-') + frag = f"and s._h_ut >={sh} and s._h_ut < {eh}" + return frag + + +def getStationData(statid, dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() - connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) try: + statfrag = f"and s.stations like '%{statid}%' " if statid is not None else "" + perfrag = periodToSqlFragment(period) if period is not None else "" with connection.cursor() as cursor: - if statid is None: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%'" - else: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' and s.stations like '%{statid}%'" + sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}" cursor.execute(sql) result = cursor.fetchall() finally: @@ -41,13 +78,14 @@ def getStationData(statid, dtstr): return res -def getSummaryData(dtstr): +def getSummaryData(dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() - connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + perfrag = periodToSqlFragment(period) if period is not None else "" + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' - expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%'" + expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%' {perfrag}" result=[] try: with connection.cursor() as cursor: @@ -88,12 +126,10 @@ def lambda_handler(event, context): if qs is None: return { 'statusCode': 200, - 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1]' + 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1][&period=am|pm|h-h]' } reqtyp = qs['reqtyp'] - points = False - if 'points' in qs: - points = True + points = True if 'points' in qs else False if reqtyp == 'station': statid = qs['statid'] @@ -102,12 +138,14 @@ def lambda_handler(event, context): res = getStationData(statid, dtstr) elif reqtyp == 'summary': dtstr = qs['reqval'] + period = qs['period'] if 'period' in qs else None print(f'summary data requested for {dtstr}') - res = getSummaryData(dtstr) + res = getSummaryData(dtstr, period) elif reqtyp == 'matches': dtstr = qs['reqval'] + period = qs['period'] if 'period' in qs else None print(f'match data requested for {dtstr}') - res = getStationData(None, dtstr) + res = getStationData(None, dtstr, period) elif reqtyp == 'detail': orbname = qs['reqval'] print(f'detail requested for {orbname}') @@ -140,30 +178,3 @@ def lambda_handler(event, context): 'statusCode': 200, 'body': res } - - -def fileToJsonString(flis): - hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', - 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', - 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] - ptsarray='[' - gotpts = False - for fli in flis: - if 'Points' in fli: - gotpts = True - continue - elif '------' in fli or ' No' in fli: - continue - elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): - gotpts=False - break - elif gotpts is True: - spls = fli.split(',') - thisrow = '{' - for h, s in zip(hdr, spls): - thisrow = thisrow + f'"{h}": "{s.strip()}",' - thisrow = thisrow[:-1] + '},' - ptsarray = ptsarray + thisrow - ptsarray = ptsarray[:-1] + ']' - ptsarray = '{' + f'"points": {ptsarray}' + '}' - return ptsarray diff --git a/archive/samfunctions/matchDataApi/requirements.txt b/archive/lambdas/matchDataApi/requirements.txt similarity index 100% rename from archive/samfunctions/matchDataApi/requirements.txt rename to archive/lambdas/matchDataApi/requirements.txt diff --git a/archive/samfunctions/matchDataApi/samconfig.toml b/archive/lambdas/matchDataApi/samconfig.toml similarity index 100% rename from archive/samfunctions/matchDataApi/samconfig.toml rename to archive/lambdas/matchDataApi/samconfig.toml diff --git a/archive/samfunctions/matchDataApi/template.yml b/archive/lambdas/matchDataApi/template.yml similarity index 93% rename from archive/samfunctions/matchDataApi/template.yml rename to archive/lambdas/matchDataApi/template.yml index d07529b3a..83f98f961 100644 --- a/archive/samfunctions/matchDataApi/template.yml +++ b/archive/lambdas/matchDataApi/template.yml @@ -31,6 +31,8 @@ Resources: Required: false - method.request.querystring.statid: Required: false + - method.request.querystring.period: + Required: false Tags: billingtag: "ukmda" typetag: "api" diff --git a/archive/samfunctions/matchDataApi/testDetail.json b/archive/lambdas/matchDataApi/testDetail.json similarity index 100% rename from archive/samfunctions/matchDataApi/testDetail.json rename to archive/lambdas/matchDataApi/testDetail.json diff --git a/archive/samfunctions/matchDataApi/testMatches.json b/archive/lambdas/matchDataApi/testMatches.json similarity index 100% rename from archive/samfunctions/matchDataApi/testMatches.json rename to archive/lambdas/matchDataApi/testMatches.json diff --git a/archive/lambdas/matchDataApi/testMatchesPeriod.json b/archive/lambdas/matchDataApi/testMatchesPeriod.json new file mode 100644 index 000000000..6f61467d2 --- /dev/null +++ b/archive/lambdas/matchDataApi/testMatchesPeriod.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "matches", + "reqval": "20251213", + "period": "0-3" + } +} \ No newline at end of file diff --git a/archive/samfunctions/matchDataApi/testPoints.json b/archive/lambdas/matchDataApi/testPoints.json similarity index 100% rename from archive/samfunctions/matchDataApi/testPoints.json rename to archive/lambdas/matchDataApi/testPoints.json diff --git a/archive/samfunctions/matchDataApi/testStation.json b/archive/lambdas/matchDataApi/testStation.json similarity index 100% rename from archive/samfunctions/matchDataApi/testStation.json rename to archive/lambdas/matchDataApi/testStation.json diff --git a/archive/lambdas/matchDataApi/testSumPeriod.json b/archive/lambdas/matchDataApi/testSumPeriod.json new file mode 100644 index 000000000..81c4b4194 --- /dev/null +++ b/archive/lambdas/matchDataApi/testSumPeriod.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "summary", + "reqval": "20251213", + "period": "0-3" + } +} \ No newline at end of file diff --git a/archive/samfunctions/matchDataApi/testSummary.json b/archive/lambdas/matchDataApi/testSummary.json similarity index 100% rename from archive/samfunctions/matchDataApi/testSummary.json rename to archive/lambdas/matchDataApi/testSummary.json diff --git a/archive/samfunctions/matchPickle/__init__.py b/archive/lambdas/matchPickle/__init__.py similarity index 100% rename from archive/samfunctions/matchPickle/__init__.py rename to archive/lambdas/matchPickle/__init__.py diff --git a/archive/samfunctions/matchPickle/localTest.ps1 b/archive/lambdas/matchPickle/localTest.ps1 similarity index 100% rename from archive/samfunctions/matchPickle/localTest.ps1 rename to archive/lambdas/matchPickle/localTest.ps1 diff --git a/archive/samfunctions/matchPickle/pythoncode/Dockerfile b/archive/lambdas/matchPickle/pythoncode/Dockerfile similarity index 75% rename from archive/samfunctions/matchPickle/pythoncode/Dockerfile rename to archive/lambdas/matchPickle/pythoncode/Dockerfile index a7caae450..cb6745d50 100644 --- a/archive/samfunctions/matchPickle/pythoncode/Dockerfile +++ b/archive/lambdas/matchPickle/pythoncode/Dockerfile @@ -3,13 +3,13 @@ FROM public.ecr.aws/lambda/python:3.8 COPY requirements.txt ./ RUN python3.8 -m pip install -r requirements.txt -t . -ENV LD_LIBRARY_PATH ./ +ENV LD_LIBRARY_PATH=./ # WMPL required even tho only one function called, because # pickle files reference structures defined in the library COPY WesternMeteorPyLib/ ./WesternMeteorPyLib -COPY *.npy ./WesternMeteorPyLib/wmpl/share/ +#COPY *.npy ./WesternMeteorPyLib/wmpl/share/ COPY *.py ./ -ENV PYTHONPATH ./WesternMeteorPyLib -ENV PROJ_LIB ./ +ENV PYTHONPATH=./WesternMeteorPyLib +ENV PROJ_LIB=./ CMD ["matchPickleApi.lambda_handler"] diff --git a/archive/samfunctions/matchPickle/pythoncode/__init__.py b/archive/lambdas/matchPickle/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/__init__.py rename to archive/lambdas/matchPickle/pythoncode/__init__.py diff --git a/archive/samfunctions/matchPickle/pythoncode/matchPickleApi.py b/archive/lambdas/matchPickle/pythoncode/matchPickleApi.py similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/matchPickleApi.py rename to archive/lambdas/matchPickle/pythoncode/matchPickleApi.py diff --git a/archive/samfunctions/matchPickle/pythoncode/requirements.txt b/archive/lambdas/matchPickle/pythoncode/requirements.txt similarity index 94% rename from archive/samfunctions/matchPickle/pythoncode/requirements.txt rename to archive/lambdas/matchPickle/pythoncode/requirements.txt index 58aa7aa9c..92834776d 100644 --- a/archive/samfunctions/matchPickle/pythoncode/requirements.txt +++ b/archive/lambdas/matchPickle/pythoncode/requirements.txt @@ -12,4 +12,5 @@ PyQt5 ; platform_machine != 'aarch64' pyyaml pyswarms keras -pytz \ No newline at end of file +pytz +paramiko \ No newline at end of file diff --git a/archive/lambdas/matchPickle/remoteTest.ps1 b/archive/lambdas/matchPickle/remoteTest.ps1 new file mode 100644 index 000000000..861de301a --- /dev/null +++ b/archive/lambdas/matchPickle/remoteTest.ps1 @@ -0,0 +1,5 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# test + + curl "https://api.ukmeteors.co.uk/pickle/getpickle?reqval=20260421_000017.759_UK" > ./testresult.json + diff --git a/archive/samfunctions/matchPickle/samconfig.toml b/archive/lambdas/matchPickle/samconfig.toml similarity index 100% rename from archive/samfunctions/matchPickle/samconfig.toml rename to archive/lambdas/matchPickle/samconfig.toml diff --git a/archive/samfunctions/matchPickle/template.yml b/archive/lambdas/matchPickle/template.yml similarity index 100% rename from archive/samfunctions/matchPickle/template.yml rename to archive/lambdas/matchPickle/template.yml diff --git a/archive/samfunctions/matchPickle/testEvent.json b/archive/lambdas/matchPickle/testEvent.json similarity index 100% rename from archive/samfunctions/matchPickle/testEvent.json rename to archive/lambdas/matchPickle/testEvent.json diff --git a/archive/lambdas/matchPickle/testresult.json b/archive/lambdas/matchPickle/testresult.json new file mode 100644 index 000000000..7c1230d27 --- /dev/null +++ b/archive/lambdas/matchPickle/testresult.json @@ -0,0 +1,4157 @@ +{ + "jdt_ref": 2461151.5002055545, + "meastype": 2, + "output_dir": ".", + "max_toffset": 10.0, + "verbose": false, + "v_init_part": 0.4, + "v_init_ht": null, + "fixed_time_offsets": {}, + "fixed_time_offsets_copy": {}, + "estimate_timing_vel": true, + "fixed_times": null, + "monte_carlo": true, + "mc_runs": 10, + "mc_runs_max": 20, + "mc_pick_multiplier": 1, + "mc_noise_std": 1.0, + "enable_OSM_plot": true, + "geometric_uncert": true, + "filter_picks": false, + "calc_orbit": true, + "show_plots": false, + "show_jacchia": false, + "save_results": false, + "gravity_correction": true, + "gravity_factor": 1.0, + "plot_all_spatial_residuals": false, + "plot_file_type": "png", + "traj_id": "20260421000017_W5MFS", + "reject_n_sigma_outliers": 2, + "mc_cores": 2, + "file_name": "20260421_000017", + "meas_count": 19, + "observations": [ + { + "ObservedPoints.UK002Z": { + "meas1": [ + 2.3968813154069024, + 2.398794550755332, + 2.401040877732498, + 2.403248309562171, + 2.405262341887118, + 2.407007598336504, + 2.4083661435454724, + 2.4104380988088447, + 2.4124495492369737, + 2.414010003498822, + 2.4151889792750527, + 2.4171917886793013, + 2.417909396727613, + 2.42033600533781, + 2.4223435953587114, + 2.427942162403076, + 2.4300097550164326, + 2.4318363974955406, + 2.433899657791362, + 2.436516452224417, + 2.437238417883184 + ], + "meas2": [ + 0.44034695311399874, + 0.4392383275873981, + 0.4382869800294502, + 0.43791610200467623, + 0.43703226417084373, + 0.4362760437170148, + 0.4357724143366082, + 0.4343759101176623, + 0.4336724668163452, + 0.4324539154277791, + 0.4328902971191608, + 0.43031636173280585, + 0.43144285337433486, + 0.43049915328211963, + 0.4295252455056575, + 0.42636260506048096, + 0.4263728887871292, + 0.4259131817698758, + 0.4244646349323693, + 0.42476734916852354, + 0.4238170894671063 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 0.2905716440411789, + 0.33053564404117924, + 0.41047964404117904, + 0.4504116440411789, + 0.4903636440411791, + 0.5703196440411791, + 0.6102876440411793, + 0.6502516440411792, + 0.690195644041179, + 0.7301756440411791, + 0.7701196440411788, + 0.8101196440411789, + 0.850059644041179, + 0.889999644041179, + 0.929951644041179, + 1.0498356440411787, + 1.0897596440411788, + 1.169703644041179, + 1.209671644041179, + 1.249563644041179, + 1.2895636440411786 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 21, + "JD_data": [ + 2461151.500208891, + 2461151.5002093534, + 2461151.5002102787, + 2461151.500210741, + 2461151.5002112035, + 2461151.5002121287, + 2461151.500212591, + 2461151.500213054, + 2461151.500213516, + 2461151.500213979, + 2461151.500214441, + 2461151.500214904, + 2461151.5002153665, + 2461151.500215829, + 2461151.5002162913, + 2461151.5002176785, + 2461151.500218141, + 2461151.500219066, + 2461151.5002195286, + 2461151.5002199905, + 2461151.5002204534 + ], + "lat": 0.9344075681486413, + "lon": -0.006680073179083097, + "ele": 34.0, + "station_id": "UK002Z", + "azim_data": [ + 4.737799374816502, + 4.735428543040449, + 4.732911263205364, + 4.730867396557772, + 4.728583356038361, + 4.726610236803339, + 4.725138590050284, + 4.722411691527732, + 4.720262091239325, + 4.718078528987862, + 4.717466569845322, + 4.713889250782277, + 4.7141741061193105, + 4.711500673159142, + 4.709139283371754, + 4.702204122090407, + 4.700540864881191, + 4.698711574889019, + 4.695932771463546, + 4.69404324715463, + 4.692730245169177 + ], + "elev_data": [ + 0.5397212284763535, + 0.540021754973832, + 0.5406386453085916, + 0.5416703853261442, + 0.5422000352665981, + 0.5426661854062742, + 0.5430931842237807, + 0.5432695876107108, + 0.5439333938247444, + 0.5439396765993363, + 0.5449702160484455, + 0.5442139525051555, + 0.54549253181179, + 0.5462208350060278, + 0.5466770226216804, + 0.5476093934732065, + 0.5488458699002796, + 0.549583029220384, + 0.5497111435989721, + 0.5514955495795086, + 0.5512040280201354 + ], + "ra_data": [ + 2.396881315406902, + 2.3987945507553325, + 2.401040877732498, + 2.403248309562171, + 2.4052623418871177, + 2.407007598336504, + 2.4083661435454733, + 2.4104380988088443, + 2.4124495492369737, + 2.4140100034988214, + 2.4151889792750527, + 2.417191788679302, + 2.417909396727613, + 2.4203360053378105, + 2.422343595358711, + 2.427942162403076, + 2.430009755016433, + 2.431836397495541, + 2.433899657791363, + 2.4365164522244163, + 2.4372384178831847 + ], + "dec_data": [ + 0.44034695311399874, + 0.43923832758739784, + 0.4382869800294501, + 0.43791610200467607, + 0.43703226417084373, + 0.4362760437170146, + 0.43577241433660796, + 0.4343759101176626, + 0.4336724668163451, + 0.43245391542777895, + 0.432890297119161, + 0.43031636173280596, + 0.43144285337433513, + 0.4304991532821195, + 0.4295252455056574, + 0.42636260506048085, + 0.42637288878712926, + 0.42591318176987575, + 0.4244646349323693, + 0.4247673491685236, + 0.4238170894671061 + ], + "magnitudes": [ + 3.6, + 5.02, + 3.44, + 3.24, + 4.28, + 5.1, + 5.72, + 3.45, + 3.98, + 5.66, + 4.45, + 5.81, + 3.55, + 4.2, + 5.66, + 4.01, + 4.53, + 5.21, + 3.72, + 5.72, + 6.0 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK002Z_20260421-000018.239614_8851", + "comment": "{\"ff_name\": \"FF_UK002Z_20260421_000016_194_0375808.fits\"}", + "incident_angle": 0.5002051863784102, + "weight": 0.23002152819250413, + "h_residuals": [ + -3.8052913962245976, + 20.398349033358095, + 20.60146887676896, + -17.142668686892375, + -12.648791991966329, + -13.029467986359393, + -17.831075981704, + 20.861748677458234, + 11.816255498445493, + 51.257765767545706, + -14.989881371214677, + 106.20898562360125, + 5.1376366928819195, + 0.8473289069201014, + 9.980312274554471, + 63.8895794707902, + 3.432361870992362, + -21.53507262422365, + 17.085715282877228, + -78.20881756637142, + -36.77494843909438 + ], + "h_res_rms": 29.722498774935328, + "v_residuals": [ + 6.309529854796034, + -33.877011893083605, + -34.22656685546831, + 28.4211657875748, + 20.979912654651937, + 21.618339676041263, + 29.581808972747485, + -34.69571283606128, + -19.644738726302496, + -85.4334144263602, + 24.889885886707884, + -177.67230920382528, + -8.544025956103757, + -1.4090186969998264, + -16.60820767834652, + -106.71982725101019, + -5.713621282471512, + 35.80745108911481, + -28.480033854767836, + 129.65104400143866, + 61.12507390508546 + ], + "v_res_rms": 49.431213522476874, + "velocities": [ + 0.0, + 16249.106425374774, + 8967.863527321228, + 15682.132184664548, + 15881.2387686525, + 6875.364245819561, + 10338.63052400254, + 17524.6942293953, + 14943.888606668643, + 13556.02313277404, + 6169.26635208947, + 20213.790509050847, + 1179.9960627586386, + 17843.383954063283, + 15237.361660009743, + 14427.322364972191, + 12382.078647459066, + 6167.611210861438, + 16369.803445691245, + 14542.207740346299, + 6974.687234953151 + ], + "velocities_prev_point": [ + 11765.84594020067, + 11765.84594020067, + 11765.84594020067, + 11765.84594020067, + 12515.334991192343, + 11548.839904206608, + 11165.164595519736, + 11379.517422838991, + 11657.45887779439, + 11886.958654011225, + 11826.65374047157, + 12038.458636794907, + 11892.690809748638, + 11932.525875899712, + 12027.59652074097, + 12255.042346574837, + 12396.814201326875, + 12294.978224650129, + 12284.761091501927, + 12304.68522942524, + 12262.39262157771 + ], + "length": [ + 0.0, + 649.3792891836829, + 1366.3061710118493, + 1992.5250734098718, + 2627.0123246950798, + 3176.7389483338284, + 3589.953333117164, + 4290.310213300716, + 4887.228899805485, + 5429.198704653792, + 5675.623879821653, + 6484.175500183687, + 6531.304542930267, + 7243.9692980555565, + 7852.7323710962655, + 9582.337485498587, + 10076.679593419743, + 10569.743104060852, + 11224.01140817824, + 11804.129159356135, + 12083.116648754258 + ], + "state_vect_dist": [ + 3587.0536516011, + 4236.432940784783, + 4953.35982261295, + 5579.5787250109715, + 6214.0659762961795, + 6763.792599934928, + 7177.006984718264, + 7877.363864901816, + 8474.282551406584, + 9016.252356254892, + 9262.677531422753, + 10071.229151784788, + 10118.358194531367, + 10831.022949656655, + 11439.786022697366, + 13169.391137099687, + 13663.733245020843, + 14156.796755661951, + 14811.06505977934, + 15391.182810957234, + 15670.170300355358 + ], + "lag": [ + 63.37447151468132, + 194.5118468950268, + -125.25258238414244, + -16.860626788108675, + 99.54032331926737, + -387.5799767748267, + -492.65937666999525, + -310.54441028977544, + -231.60828321249573, + -208.08787566819592, + -479.64525992782546, + -189.8023912456356, + -660.6040370513838, + -465.86997047842124, + -375.1931986155396, + -200.2100838728802, + -223.59118100337764, + -767.2189814696103, + -631.2444620307251, + -568.4349489031392, + -808.1562111848561 + ], + "lag_line": [ + 11370.947134066559, + -3165.029756068875 + ], + "v_init": 11370.947134066559, + "v_init_stddev": null, + "jacchia_fit": [ + 63.688001193520016, + 1.8900846614547042 + ], + "model_ra": [ + 2.396881593167252, + 2.3989304099368014, + 2.4011844016151613, + 2.4031860875196687, + 2.4052274845571198, + 2.406976087071311, + 2.4083103572784346, + 2.4106035391564897, + 2.412566930909117, + 2.4143572664202093, + 2.41515920660715, + 2.417869111447515, + 2.418000831463778, + 2.420405613794107, + 2.4224695120777024, + 2.428399440764061, + 2.430109452127189, + 2.4317929026796112, + 2.4340906148854335, + 2.4361376027592625, + 2.4371099642448937 + ], + "model_dec": [ + 0.440302678909872, + 0.4394764728451123, + 0.43852840306686475, + 0.4377147344407611, + 0.43688315067561767, + 0.4361219869381554, + 0.4355611737515261, + 0.4346241691042197, + 0.4338134534871204, + 0.4330683131149763, + 0.43271070454145943, + 0.431599113939899, + 0.43150463507581566, + 0.4305093179638242, + 0.4296461972321007, + 0.4271459610813783, + 0.4264149242154197, + 0.42564839310551755, + 0.42467559658948373, + 0.42380137844967836, + 0.4233614958813678 + ], + "model_azim": [ + 4.73778196828684, + 4.735521732614828, + 4.7330053965160985, + 4.730788912250429, + 4.728525435294195, + 4.726550570784113, + 4.725056923942933, + 4.722507072008819, + 4.720316125854996, + 4.7183126731808445, + 4.717397902721562, + 4.714373400873156, + 4.714197584063138, + 4.711504522278045, + 4.709184901171225, + 4.702495517776749, + 4.700556524882403, + 4.69861306303917, + 4.696010730582082, + 4.6936850389188365, + 4.692562119253269 + ], + "model_elev": [ + 0.5396755224600758, + 0.5402679008661397, + 0.5408884116776695, + 0.5414620307648099, + 0.5420456131089348, + 0.5425065260896258, + 0.5428741613485486, + 0.5435273909554157, + 0.5440798903382158, + 0.5445789635092166, + 0.5447834940136214, + 0.5455517621083557, + 0.5455568612650257, + 0.5462314278769935, + 0.5468031898818537, + 0.548429156889144, + 0.5488898437745016, + 0.5493059467086585, + 0.549932237410441, + 0.5504839878814398, + 0.5507264098873919 + ], + "model_fit1": [ + 4.73778196828684, + 4.735521732614828, + 4.7330053965160985, + 4.730788912250429, + 4.728525435294195, + 4.726550570784113, + 4.725056923942933, + 4.722507072008819, + 4.720316125854996, + 4.7183126731808445, + 4.717397902721562, + 4.714373400873156, + 4.714197584063138, + 4.711504522278045, + 4.709184901171225, + 4.702495517776749, + 4.700556524882403, + 4.69861306303917, + 4.696010730582082, + 4.6936850389188365, + 4.692562119253269 + ], + "model_fit2": [ + 0.5396755224600758, + 0.5402679008661397, + 0.5408884116776695, + 0.5414620307648099, + 0.5420456131089348, + 0.5425065260896258, + 0.5428741613485486, + 0.5435273909554157, + 0.5440798903382158, + 0.5445789635092166, + 0.5447834940136214, + 0.5455517621083557, + 0.5455568612650257, + 0.5462314278769935, + 0.5468031898818537, + 0.548429156889144, + 0.5488898437745016, + 0.5493059467086585, + 0.549932237410441, + 0.5504839878814398, + 0.5507264098873919 + ], + "meas_eci": [ + [ + -0.6651403376230054, + 0.6131039197684128, + 0.42625334583010854 + ], + [ + -0.6666597964124698, + 0.6121494659158085, + 0.42525021719721046 + ], + [ + -0.6683315036211338, + 0.610923065694228, + 0.4243889832103093 + ], + [ + -0.6697948071375551, + 0.6095521783058309, + 0.424053131404759 + ], + [ + -0.6712985427876046, + 0.6084534187936955, + 0.42325252935984004 + ], + [ + -0.6725967633254724, + 0.6074952687408567, + 0.42256726378361675 + ], + [ + -0.6735794940210694, + 0.6067233106956108, + 0.4221107550073526 + ], + [ + -0.6752733035008325, + 0.6057194052358751, + 0.42084435091821104 + ], + [ + -0.6767109144101443, + 0.6045569835569059, + 0.42020613031078136 + ], + [ + -0.6780353559270645, + 0.6038403657657955, + 0.41910007013205 + ], + [ + -0.6786100121852274, + 0.6029190303333297, + 0.41949623862900476 + ], + [ + -0.6806225551669195, + 0.602272242271424, + 0.41715834354283843 + ], + [ + -0.6807019943374223, + 0.6014721244956676, + 0.4181818723470973 + ], + [ + -0.6824555864862213, + 0.6000788818635572, + 0.4173244637151418 + ], + [ + -0.6839643602217045, + 0.5989750663284349, + 0.4164392198909243 + ], + [ + -0.6882992447918803, + 0.5959956375357374, + 0.41356178457069387 + ], + [ + -0.6895268279492358, + 0.5945684648394516, + 0.41357114763571556 + ], + [ + -0.6907558791802513, + 0.5934317844296477, + 0.4131525536730498 + ], + [ + -0.6924328423375028, + 0.592393747301069, + 0.41183298436502674 + ], + [ + -0.6938856684688789, + 0.5904989435368124, + 0.4121088166678862 + ], + [ + -0.6946099128507072, + 0.5902511457747254, + 0.41124281620613995 + ] + ], + "meas_eci_los": [ + [ + -0.6651532201448609, + 0.6130899435642051, + 0.42625334583010854 + ], + [ + -0.6666744422176303, + 0.6121335155564811, + 0.42525021719721046 + ], + [ + -0.668349681390559, + 0.6109031791657809, + 0.4243889832103093 + ], + [ + -0.669814719865365, + 0.6095302968659562, + 0.424053131404759 + ], + [ + -0.6713201921678998, + 0.6084295324671147, + 0.42325252935984004 + ], + [ + -0.6726219198904363, + 0.6074674151436691, + 0.42256726378361675 + ], + [ + -0.6736063861027676, + 0.6066934539853648, + 0.4221107550073526 + ], + [ + -0.6753019174198893, + 0.6056875041052527, + 0.42084435091821104 + ], + [ + -0.6767412328073986, + 0.6045230449433344, + 0.42020613031078136 + ], + [ + -0.6780673992284806, + 0.6038043833219787, + 0.41910007013205 + ], + [ + -0.6786437629715607, + 0.602881040302245, + 0.41949623862900476 + ], + [ + -0.6806580260024921, + 0.6022321546139711, + 0.41715834354283843 + ], + [ + -0.6807391702046832, + 0.6014300489577449, + 0.4181818723470973 + ], + [ + -0.6824944243246932, + 0.6000347096215143, + 0.4173244637151418 + ], + [ + -0.6840048714786812, + 0.598928803723841, + 0.4164392198909243 + ], + [ + -0.6883447630503693, + 0.5959430656730842, + 0.41356178457069387 + ], + [ + -0.6895739692035289, + 0.5945137902859511, + 0.41357114763571556 + ], + [ + -0.6908063894172288, + 0.5933729853420778, + 0.4131525536730498 + ], + [ + -0.6924849898642252, + 0.5923327880522399, + 0.41183298436502674 + ], + [ + -0.693939367574386, + 0.5904358367808104, + 0.4121088166678862 + ], + [ + -0.6946653105480682, + 0.590185947342021, + 0.41124281620613995 + ] + ], + "model_eci": [ + [ + -3431648.7433502977, + -1734673.2501258308, + 5171746.249168466 + ], + [ + -3431502.168458378, + -1735198.4026242748, + 5171393.320457905 + ], + [ + -3431340.2471023854, + -1735778.206261865, + 5171003.513213945 + ], + [ + -3431198.8672727705, + -1736284.6365797631, + 5170663.117429066 + ], + [ + -3431055.612694895, + -1736797.753523634, + 5170318.214222535 + ], + [ + -3430931.3319215793, + -1737242.3632842037, + 5170019.113061614 + ], + [ + -3430837.965680444, + -1737576.5501712172, + 5169794.374518736 + ], + [ + -3430679.813194559, + -1738142.9361567015, + 5169413.623255734 + ], + [ + -3430544.9854288357, + -1738625.679263341, + 5169089.050559572 + ], + [ + -3430422.545034539, + -1739063.9846602748, + 5168794.316972965 + ], + [ + -3430366.765328334, + -1739263.305532724, + 5168660.1248029135 + ], + [ + -3430184.1694901455, + -1739917.1765316478, + 5168220.538002146 + ], + [ + -3430173.328761385, + -1739955.342499328, + 5168194.58388284 + ], + [ + -3430012.3423658106, + -1740531.6864444455, + 5167807.052084736 + ], + [ + -3429874.786286485, + -1741024.0084887384, + 5167475.953263732 + ], + [ + -3429483.8746772027, + -1742422.7843117185, + 5166535.0961491605 + ], + [ + -3429372.090910218, + -1742822.589670693, + 5166266.09232802 + ], + [ + -3429260.304079202, + -1743221.4092456542, + 5165997.30074232 + ], + [ + -3429112.4197078072, + -1743750.5190035156, + 5165641.380819524 + ], + [ + -3428981.246454044, + -1744219.694618384, + 5165325.712035401 + ], + [ + -3428918.0015465165, + -1744445.3384067302, + 5165173.638196536 + ] + ], + "meas_lat": [ + 0.9346450225936334, + 0.9345970660398534, + 0.9345465778299221, + 0.9345060097444198, + 0.9344609020518903, + 0.9344222008936756, + 0.9343935294446254, + 0.9343405561033522, + 0.9342993156178634, + 0.9342574203614202, + 0.93424620996589, + 0.9341778905737628, + 0.9341839571465814, + 0.9341340643559439, + 0.9340902381901774, + 0.9339630599390059, + 0.9339337530240859, + 0.9339012154522863, + 0.933851362550423, + 0.9338192346503082, + 0.9337956204733615 + ], + "meas_lon": [ + -0.04088550365419589, + -0.04075548711489885, + -0.04060781279524608, + -0.04046702814824051, + -0.04033520755003031, + -0.04022319461679362, + -0.040136396159376767, + -0.03999918021151192, + -0.03987197780910561, + -0.03976888838620386, + -0.03970217229821398, + -0.03956286802268136, + -0.03952989329711795, + -0.03937914973331329, + -0.039254062049537145, + -0.03890628729167373, + -0.0387879885582501, + -0.03868188951868613, + -0.03855458929929464, + -0.038409166927020906, + -0.03836284386104872 + ], + "meas_ht": [ + 80166.50920069027, + 79907.89847155106, + 79663.6709640296, + 79509.71227376115, + 79287.06050564267, + 79100.56625424455, + 78967.5446651115, + 78669.14774349949, + 78480.50679384157, + 78234.48391237574, + 78254.36663415054, + 77789.32418704791, + 77932.08594644774, + 77696.87267580134, + 77475.89917794727, + 76804.02455018043, + 76731.24392918951, + 76602.42998524354, + 76320.02753907315, + 76272.04281635238, + 76112.63523459999 + ], + "meas_range": [ + 153339.33047024682, + 152779.23373854544, + 152169.97299907313, + 151631.42779261048, + 151086.25079201206, + 150624.91058615095, + 150273.74436083622, + 149672.74252612054, + 149162.45846822963, + 148700.46758205735, + 148495.59961464474, + 147802.94181198507, + 147771.35775808938, + 147162.96362779627, + 146645.26441739433, + 145179.17570669635, + 144762.42956183938, + 144356.17304021947, + 143802.8283496073, + 143313.7343373524, + 143083.5198313615 + ], + "model_lat": [ + 0.9346446667677631, + 0.9345989704981107, + 0.9345485008239762, + 0.9345044075468767, + 0.9344597200105587, + 0.9344209834416608, + 0.9343918634192264, + 0.9343425020668983, + 0.9343004177428308, + 0.9342621973971849, + 0.9342448099303645, + 0.9341877726487242, + 0.9341844358943037, + 0.9341341428363436, + 0.9340911683984181, + 0.9339690073179128, + 0.9339340724672882, + 0.9338992059698612, + 0.9338529539754011, + 0.9338119290348905, + 0.9337921890458436 + ], + "model_lon": [ + -0.04088647065639903, + -0.0407503038625055, + -0.04060257766467196, + -0.040471381075913575, + -0.0403384201294265, + -0.04022650428958335, + -0.04014092463279356, + -0.03999387866505028, + -0.039868976417382226, + -0.03975584834373117, + -0.03970598017255938, + -0.03953580064279074, + -0.03952858935807945, + -0.0393789360113246, + -0.03925152725986469, + -0.03889002516887339, + -0.038787118004688446, + -0.038687359650227564, + -0.03855024718026426, + -0.03842899481740237, + -0.038372180120937446 + ], + "model_ht": [ + 80160.57129329837, + 79939.75564689818, + 79695.8571107965, + 79482.97408082303, + 79267.32029777888, + 79080.2240190833, + 78939.71062025194, + 78701.77767818491, + 78498.9785627082, + 78314.84618101093, + 78230.94224464067, + 77956.46774587498, + 77940.11468171058, + 77698.18875523485, + 77491.51556920519, + 76904.43232644908, + 76736.61042515974, + 76568.72510363306, + 76346.81825810629, + 76150.02295057768, + 76055.10224458351 + ], + "model_range": [ + 153339.33064750832, + 152779.23885477215, + 152169.9782410282, + 151631.43142680972, + 151086.25277939526, + 150624.9127025781, + 150273.74833255954, + 149672.74799874792, + 149162.46022825968, + 148700.50095141708, + 148495.6024593428, + 147803.08674493877, + 147771.3580936107, + 147162.9636368488, + 146645.26569589667, + 145179.22897841557, + 144762.42971472227, + 144356.17909111048, + 143802.8321819995, + 143313.81433522768, + 143083.53761941093 + ], + "rbeg_lat": 0.9346446667677631, + "rbeg_lon": -0.04088647065639903, + "rbeg_ele": 80160.57129329837, + "rbeg_jd": 2461151.500208891, + "rend_lat": 0.9337921890458436, + "rend_lon": -0.038372180120937446, + "rend_ele": 76055.10224458351, + "rend_jd": 2461151.5002204534, + "htmin_lat": 0.9337921890458436, + "htmin_lon": -0.038372180120937446, + "htmin_ele": 76055.10224458351, + "htmin_jd": 2461151.5002204534, + "absolute_magnitudes": [ + 2.671732184227931, + 4.099678289776929, + 2.528355107417953, + 2.3360538259627734, + 3.3838752499018385, + 4.210515958921451, + 4.835584403455401, + 2.5742863380034136, + 3.111702310827387, + 4.798437841987752, + 3.591432036646971, + 4.961582479831281, + 2.7020486763676725, + 3.3610073736334165, + 4.828659766224398, + 3.200477571944522, + 3.726720681188872, + 4.412823108180669, + 2.931162802418613, + 4.938559724900953, + 5.222051654133499 + ], + "ra_data_los": [ + 2.396902327617516, + 2.3988184762780778, + 2.401070632813468, + 2.403280977947147, + 2.4052979235842242, + 2.4070490095918355, + 2.408410468104811, + 2.4104853396142762, + 2.4124997004205735, + 2.414063070928515, + 2.415244960008752, + 2.417250685659096, + 2.4179712070195376, + 2.420400728941866, + 2.42241123226677, + 2.42801853924753, + 2.43008904516489, + 2.431921517202225, + 2.433987690810178, + 2.4366073956132723, + 2.4373322775181343 + ], + "dec_data_los": [ + 0.44034695311399874, + 0.43923832758739784, + 0.4382869800294501, + 0.43791610200467607, + 0.43703226417084373, + 0.4362760437170146, + 0.43577241433660796, + 0.4343759101176626, + 0.4336724668163451, + 0.43245391542777895, + 0.432890297119161, + 0.43031636173280596, + 0.43144285337433513, + 0.4304991532821195, + 0.4295252455056574, + 0.42636260506048085, + 0.42637288878712926, + 0.42591318176987575, + 0.4244646349323693, + 0.4247673491685236, + 0.4238170894671061 + ], + "x_eci": [ + -0.6651403376230054, + -0.6666597964124698, + -0.6683315036211338, + -0.6697948071375551, + -0.6712985427876046, + -0.6725967633254724, + -0.6735794940210694, + -0.6752733035008325, + -0.6767109144101443, + -0.6780353559270645, + -0.6786100121852274, + -0.6806225551669195, + -0.6807019943374223, + -0.6824555864862213, + -0.6839643602217045, + -0.6882992447918803, + -0.6895268279492358, + -0.6907558791802513, + -0.6924328423375028, + -0.6938856684688789, + -0.6946099128507072 + ], + "y_eci": [ + 0.6131039197684128, + 0.6121494659158085, + 0.610923065694228, + 0.6095521783058309, + 0.6084534187936955, + 0.6074952687408567, + 0.6067233106956108, + 0.6057194052358751, + 0.6045569835569059, + 0.6038403657657955, + 0.6029190303333297, + 0.602272242271424, + 0.6014721244956676, + 0.6000788818635572, + 0.5989750663284349, + 0.5959956375357374, + 0.5945684648394516, + 0.5934317844296477, + 0.592393747301069, + 0.5904989435368124, + 0.5902511457747254 + ], + "z_eci": [ + 0.42625334583010854, + 0.42525021719721046, + 0.4243889832103093, + 0.424053131404759, + 0.42325252935984004, + 0.42256726378361675, + 0.4221107550073526, + 0.42084435091821104, + 0.42020613031078136, + 0.41910007013205, + 0.41949623862900476, + 0.41715834354283843, + 0.4181818723470973, + 0.4173244637151418, + 0.4164392198909243, + 0.41356178457069387, + 0.41357114763571556, + 0.4131525536730498, + 0.41183298436502674, + 0.4121088166678862, + 0.41124281620613995 + ], + "x_eci_los": [ + -0.6651532201448609, + -0.6666744422176303, + -0.668349681390559, + -0.669814719865365, + -0.6713201921678998, + -0.6726219198904363, + -0.6736063861027676, + -0.6753019174198893, + -0.6767412328073986, + -0.6780673992284806, + -0.6786437629715607, + -0.6806580260024921, + -0.6807391702046832, + -0.6824944243246932, + -0.6840048714786812, + -0.6883447630503693, + -0.6895739692035289, + -0.6908063894172288, + -0.6924849898642252, + -0.693939367574386, + -0.6946653105480682 + ], + "y_eci_los": [ + 0.6130899435642051, + 0.6121335155564811, + 0.6109031791657809, + 0.6095302968659562, + 0.6084295324671147, + 0.6074674151436691, + 0.6066934539853648, + 0.6056875041052527, + 0.6045230449433344, + 0.6038043833219787, + 0.602881040302245, + 0.6022321546139711, + 0.6014300489577449, + 0.6000347096215143, + 0.598928803723841, + 0.5959430656730842, + 0.5945137902859511, + 0.5933729853420778, + 0.5923327880522399, + 0.5904358367808104, + 0.590185947342021 + ], + "z_eci_los": [ + 0.42625334583010854, + 0.42525021719721046, + 0.4243889832103093, + 0.424053131404759, + 0.42325252935984004, + 0.42256726378361675, + 0.4221107550073526, + 0.42084435091821104, + 0.42020613031078136, + 0.41910007013205, + 0.41949623862900476, + 0.41715834354283843, + 0.4181818723470973, + 0.4173244637151418, + 0.4164392198909243, + 0.41356178457069387, + 0.41357114763571556, + 0.4131525536730498, + 0.41183298436502674, + 0.4121088166678862, + 0.41124281620613995 + ], + "x_stat": -3329692.8394425996, + "y_stat": -1828618.1638918526, + "z_stat": 5106390.987831098, + "stat_eci": [ + -3329692.8394425996, + -1828618.1638918526, + 5106390.987831098 + ], + "stat_eci_los": [ + [ + -3329654.415397559, + -1828688.127695391, + 5106390.987831098 + ], + [ + -3329649.0878441236, + -1828697.8280102296, + 5106390.987831098 + ], + [ + -3329638.4272870813, + -1828717.2383623847, + 5106390.987831098 + ], + [ + -3329633.099663696, + -1828726.9386035914, + 5106390.987831098 + ], + [ + -3329627.771997192, + -1828736.6388563307, + 5106390.987831098 + ], + [ + -3329617.1112139, + -1828756.049084223, + 5106390.987831098 + ], + [ + -3329611.783477447, + -1828765.7492633294, + 5106390.987831098 + ], + [ + -3329606.450332315, + -1828775.4592229126, + 5106390.987831098 + ], + [ + -3329601.1278900444, + -1828785.1496290863, + 5106390.987831098 + ], + [ + -3329595.7946883356, + -1828794.859557595, + 5106390.987831098 + ], + [ + -3329590.466838816, + -1828804.5596746, + 5106390.987831098 + ], + [ + -3329585.1335805035, + -1828814.2695720182, + 5106390.987831098 + ], + [ + -3329579.805659577, + -1828823.9696850183, + 5106390.987831098 + ], + [ + -3329574.4777103914, + -1828833.6697824954, + 5106390.987831098 + ], + [ + -3329569.1497478094, + -1828843.369837391, + 5106390.987831098 + ], + [ + -3329553.1656459146, + -1828872.4699901317, + 5106390.987831098 + ], + [ + -3329547.837570296, + -1828882.1699829372, + 5106390.987831098 + ], + [ + -3329537.17593862, + -1828901.5797448508, + 5106390.987831098 + ], + [ + -3329531.847763332, + -1828911.2797181348, + 5106390.987831098 + ], + [ + -3329526.5249406365, + -1828920.9698801264, + 5106390.987831098 + ], + [ + -3329521.191342843, + -1828930.679591074, + 5106390.987831098 + ] + ], + "plane_N": [ + -0.010758941735242446, + 0.5626841701025131, + -0.8266019416192922 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 4.808344044228582e-05, + 0.00025879609182125133, + 0.0002624805684568251, + 0.0002189398121328159, + 0.00016219719514936625, + 0.00016763637746942077, + 0.0002299126917867094, + 0.0002704217170759464, + 0.0001536191301664089, + 0.000669935090213715, + 0.0001957385228524681, + 0.0014004153194447633, + 6.738763283318817e-05, + 1.1091740167342496e-05, + 0.00013204797495846036, + 0.0008566651982668407, + 4.595851975374786e-05, + 0.0002895389345591846, + 0.0002308693235996496, + 0.0010565990502203683, + 0.0004986368036220028 + ], + "ang_res_std": 0.0003960746824252553 + } + }, + { + "ObservedPoints.UK00AS": { + "meas1": [ + 4.2684960666937455, + 4.275705660567851, + 4.282316120282617, + 4.287776816608948, + 4.295439857062819, + 4.301929096125738, + 4.308588575262349, + 4.314596261320367, + 4.3215806860608525, + 4.327444662401953, + 4.331822069286927, + 4.335799177502339 + ], + "meas2": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.0734864670857704, + 1.0676655293254065, + 1.0625353098743915, + 1.0568295338648426, + 1.0530700757470863, + 1.047988416812209, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 0.8142981686851195, + 0.8543421686851195, + 0.8943901686851182, + 0.934414168685119, + 0.97446616868512, + 1.0145101686851197, + 1.054550168685119, + 1.09461416868512, + 1.1346821686851198, + 1.1747101686851193, + 1.2147341686851183, + 1.2547741686851195 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 12, + "JD_data": [ + 2461151.500214981, + 2461151.5002154447, + 2461151.500215908, + 2461151.5002163714, + 2461151.5002168347, + 2461151.5002172985, + 2461151.500217762, + 2461151.5002182256, + 2461151.5002186894, + 2461151.5002191523, + 2461151.5002196156, + 2461151.500220079 + ], + "lat": 0.9311541522498763, + "lon": -0.045808173347230974, + "ele": 57.0, + "station_id": "UK00AS", + "azim_data": [ + 0.8832534008354802, + 0.8974325902214364, + 0.9104409422636692, + 0.9232847050629855, + 0.9391342537659964, + 0.9528309723365957, + 0.9677775061336478, + 0.9774560061825399, + 0.9902964441331861, + 1.0053988251477577, + 1.01598915152872, + 1.0237734767338456 + ], + "elev_data": [ + 1.191408858155571, + 1.1885973861086219, + 1.1859092541919156, + 1.1836589415935101, + 1.1802988113469086, + 1.1773284999232583, + 1.1741538069154087, + 1.1711813003457707, + 1.1676161578412807, + 1.164451512475476, + 1.162008573079885, + 1.1597749824215797 + ], + "ra_data": [ + 4.2684960666937455, + 4.275705660567851, + 4.282316120282618, + 4.2877768166089485, + 4.295439857062819, + 4.301929096125738, + 4.308588575262349, + 4.314596261320368, + 4.3215806860608525, + 4.327444662401954, + 4.331822069286927, + 4.33579917750234 + ], + "dec_data": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.07348646708577, + 1.0676655293254065, + 1.0625353098743917, + 1.0568295338648421, + 1.0530700757470863, + 1.0479884168122084, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "magnitudes": [ + 2.07, + 2.55, + 2.28, + 2.28, + 2.54, + 2.07, + 2.44, + 2.91, + 1.92, + 2.53, + 4.48, + 3.45 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK00AS_20260421-000018.455603_8295", + "comment": "{\"ff_name\": \"FF_UK00AS_20260421_000008_555_0363264.fits\"}", + "incident_angle": 1.265480375315784, + "weight": 0.909642928791794, + "h_residuals": [ + 5.292367312510447, + -1.2325180959965676, + -8.34854165501151, + 18.532634348422874, + 21.298045268951483, + 26.66314727338721, + 45.7974867531929, + 1.1143548526059945, + -25.281726091833455, + 20.538142904874615, + 42.62245203257627, + 32.87553355400295 + ], + "h_res_rms": 25.272385757689367, + "v_residuals": [ + 1.948174626316228, + -0.453757025563914, + -3.0739954328321524, + 6.815581499143073, + 7.830632268485128, + 9.7997429292699, + 16.81714105751497, + 0.40987265024026254, + -9.307426018837527, + 7.546207273811804, + 15.64451435403657, + 12.069949587749356 + ], + "v_res_rms": 9.28371695850286, + "velocities": [ + 0.0, + 12969.297700466464, + 12063.810480597225, + 11535.301400186596, + 14810.349609688103, + 12948.52617841113, + 14077.237634374165, + 10248.745079827027, + 13145.40515860947, + 14305.426366502534, + 10388.296265044806, + 8232.259675064872 + ], + "velocities_prev_point": [ + 12177.028722194334, + 12177.028722194334, + 12177.028722194334, + 12177.028722194334, + 12635.927988514766, + 12811.691915261166, + 13010.049718796889, + 12862.45411623616, + 12812.459983391418, + 12868.413677161267, + 12786.266891221863, + 12563.748843977468 + ], + "length": [ + 0.0, + 519.3425571174787, + 1002.4740392444213, + 1464.162942485498, + 2057.347065052742, + 2575.8578473410325, + 3139.510442221366, + 3550.116165099566, + 4076.826258994727, + 4649.4438655930835, + 5065.225035305226, + 5394.844712694833 + ], + "state_vect_dist": [ + 9921.354086256408, + 10440.696643373885, + 10923.828125500828, + 11385.517028741906, + 11978.70115130915, + 12497.21193359744, + 13060.864528477774, + 13471.470251355973, + 13998.180345251134, + 14570.79795184949, + 14986.579121561634, + 15316.19879895124 + ], + "lag": [ + -393.8633893220631, + -393.8001635112796, + -429.99988356618087, + -487.3309572559665, + -413.52990774576574, + -414.29845676416335, + -369.87332231534674, + -478.8062851196937, + -471.68674778223067, + -418.1409889898896, + -521.3797962085864, + -710.987579250519 + ], + "lag_line": [ + 12179.051046767432, + -9899.353326496881 + ], + "v_init": 12179.051046767432, + "v_init_stddev": null, + "jacchia_fit": [ + 169.25151905025842, + 0.9545719833694175 + ], + "model_ra": [ + 4.268675306470282, + 4.275740112640083, + 4.282193626963838, + 4.288259757151979, + 4.295985212896367, + 4.302594155354585, + 4.3096754275247084, + 4.3147005165367975, + 4.321109894501091, + 4.327977157888565, + 4.332834122852471, + 4.336600836612544 + ], + "model_dec": [ + 1.0877269738198503, + 1.0827457055324, + 1.078088202406111, + 1.073615583501862, + 1.067812547679924, + 1.0627178988374508, + 1.0571404830083215, + 1.0530775705703026, + 1.0478192667242059, + 1.0420648001571293, + 1.0378881555178725, + 1.0345818657151975 + ], + "model_azim": [ + 0.8831700010505981, + 0.8974524277636791, + 0.9105774389431591, + 0.9229769805934369, + 0.9387740337613688, + 0.9523731955673052, + 0.9669789957103925, + 0.9774364487803786, + 0.9907468066797422, + 1.0050282483732862, + 1.0152133154617018, + 1.0231714421513614 + ], + "model_elev": [ + 1.1913491255580793, + 1.188611241077182, + 1.1860024765005819, + 1.1834532326767415, + 1.1800641024878358, + 1.1770365570460057, + 1.1736559939503042, + 1.171169280976094, + 1.1678878621525661, + 1.1642326384892834, + 1.161556949829732, + 1.1594281762863772 + ], + "model_fit1": [ + 0.8831700010505981, + 0.8974524277636791, + 0.9105774389431591, + 0.9229769805934369, + 0.9387740337613688, + 0.9523731955673052, + 0.9669789957103925, + 0.9774364487803786, + 0.9907468066797422, + 1.0050282483732862, + 1.0152133154617018, + 1.0231714421513614 + ], + "model_fit2": [ + 1.1913491255580793, + 1.188611241077182, + 1.1860024765005819, + 1.1834532326767415, + 1.1800641024878358, + 1.1770365570460057, + 1.1736559939503042, + 1.171169280976094, + 1.1678878621525661, + 1.1642326384892834, + 1.161556949829732, + 1.1594281762863772 + ], + "meas_eci": [ + [ + -0.19949752969882315, + -0.41951346273949935, + 0.885555865106421 + ], + [ + -0.19831370967000816, + -0.42489567883123996, + 0.8832527014775893 + ], + [ + -0.19719537224748762, + -0.42989189519454285, + 0.8810828244894001 + ], + [ + -0.19653440624741567, + -0.43469911459331356, + 0.8788691068257918 + ], + [ + -0.19526604393742353, + -0.44086253314794654, + 0.8760772790977996 + ], + [ + -0.19419198311969488, + -0.44623568023084337, + 0.8735921195735208 + ], + [ + -0.1931713762708322, + -0.45209532151521414, + 0.8708011481697114 + ], + [ + -0.19171870682741307, + -0.45626258010862597, + 0.8689467161138501 + ], + [ + -0.19020697856453836, + -0.4616673137779947, + 0.8664205657152649 + ], + [ + -0.18946461302037865, + -0.4676323484404185, + 0.8633789128216748 + ], + [ + -0.18879990840657096, + -0.4719169246748595, + 0.8611904613911451 + ], + [ + -0.18794699339414994, + -0.4752570683607163, + 0.8595386242905415 + ] + ], + "meas_eci_los": [ + [ + -0.19947261818984635, + -0.41952530837615254, + 0.885555865106421 + ], + [ + -0.19828723693595823, + -0.4249080335800251, + 0.8832527014775893 + ], + [ + -0.1971673332669236, + -0.42990475582643994, + 0.8810828244894001 + ], + [ + -0.19650478472685406, + -0.4347125057399289, + 0.8788691068257918 + ], + [ + -0.19523471543822826, + -0.44087640777926657, + 0.8760772790977996 + ], + [ + -0.19415896881448527, + -0.4462500458800414, + 0.8735921195735208 + ], + [ + -0.19313660866544666, + -0.4521101754434669, + 0.8708011481697114 + ], + [ + -0.19168228546062463, + -0.4562778824308138, + 0.8689467161138501 + ], + [ + -0.19016877668854015, + -0.4616830511058688, + 0.8664205657152649 + ], + [ + -0.18942455378532386, + -0.4676485767305336, + 0.8633789128216748 + ], + [ + -0.18875810448810454, + -0.4719336470299235, + 0.8611904613911451 + ], + [ + -0.18790350619325405, + -0.4752742636762536, + 0.8595386242905415 + ] + ], + "model_eci": [ + [ + -3430218.4459287277, + -1739794.238366333, + 5168303.099138084 + ], + [ + -3430101.130186622, + -1740214.0687953206, + 5168020.730054632 + ], + [ + -3429991.976360051, + -1740604.6083564344, + 5167758.024153548 + ], + [ + -3429887.652948811, + -1740977.7996632662, + 5167506.958303962 + ], + [ + -3429753.6593147092, + -1741457.3125123843, + 5167184.447316998 + ], + [ + -3429636.5043168305, + -1741876.432179816, + 5166902.494989177 + ], + [ + -3429509.159525174, + -1742332.044553453, + 5166596.010709642 + ], + [ + -3429416.332842804, + -1742663.8893094896, + 5166372.661447595 + ], + [ + -3429297.3084989507, + -1743089.6082505668, + 5166086.229687476 + ], + [ + -3429167.922097079, + -1743552.438223741, + 5165774.850941512 + ], + [ + -3429073.9084734865, + -1743888.437613092, + 5165548.665302157 + ], + [ + -3428999.3243624857, + -1744154.7560413252, + 5165369.277862051 + ] + ], + "meas_lat": [ + 0.9341977291845097, + 0.9341620422361886, + 0.9341289887201897, + 0.9340925154995868, + 0.9340502484747027, + 0.9340128678846222, + 0.9339702969792177, + 0.9339477745675356, + 0.9339143956069487, + 0.9338673009272427, + 0.9338347231487523, + 0.9338128429395532 + ], + "meas_lon": [ + -0.039569337714437845, + -0.0394604803690885, + -0.039359318234566246, + -0.03926606572328635, + -0.03914232800703642, + -0.039034834123200954, + -0.03891903030012694, + -0.03882974941824365, + -0.03871741853890708, + -0.038602319686930085, + -0.03851846883407553, + -0.03844994217817617 + ], + "meas_ht": [ + 78009.79883316693, + 77831.20354207544, + 77664.69644717604, + 77517.24209363233, + 77316.91530664213, + 77142.80788407566, + 76958.1916129066, + 76803.36867867864, + 76615.55707637589, + 76437.24094479789, + 76303.77256128346, + 76188.47275826824 + ], + "meas_range": [ + 83839.35913329039, + 83740.33676324955, + 83650.64655986936, + 83567.18288250422, + 83464.48024344677, + 83377.45673232341, + 83286.60977747764, + 83221.62612094023, + 83142.13428744262, + 83059.64575911379, + 83000.93009824904, + 82954.89554656077 + ], + "model_lat": [ + 0.9341984959671219, + 0.9341618633579815, + 0.9341277785486444, + 0.9340952007177374, + 0.934053334269257, + 0.9340167308727484, + 0.9339769318673277, + 0.9339479355618667, + 0.9339107317015435, + 0.9338702759470549, + 0.9338408971786679, + 0.933817604920053 + ], + "model_lon": [ + -0.039568822172874184, + -0.039460600665898946, + -0.03936013225744678, + -0.03926425828503005, + -0.03914025008272261, + -0.03903223179007913, + -0.03891455786744328, + -0.03882964093347483, + -0.03871988735385722, + -0.03860031305219093, + -0.03851430205713854, + -0.03844672807852617 + ], + "model_ht": [ + 78007.96659065987, + 77831.63104327311, + 77667.58915264205, + 77510.83050120615, + 77309.5485524352, + 77133.5883195504, + 76942.36899348421, + 76802.98414373519, + 76624.31618154436, + 76430.14134564155, + 76289.05225369317, + 76177.11611285266 + ], + "model_range": [ + 83839.35932285733, + 83740.33677357878, + 83650.64703317253, + 83567.18521487652, + 83464.4833274639, + 83377.46157053398, + 83286.62406500315, + 83221.62612936008, + 83142.13865346192, + 83059.64864007127, + 83000.94251389406, + 82954.90293702218 + ], + "rbeg_lat": 0.9341984959671219, + "rbeg_lon": -0.039568822172874184, + "rbeg_ele": 78007.96659065987, + "rbeg_jd": 2461151.500214981, + "rend_lat": 0.933817604920053, + "rend_lon": -0.03844672807852617, + "rend_ele": 76177.11611285266, + "rend_jd": 2461151.500220079, + "htmin_lat": 0.933817604920053, + "htmin_lon": -0.03844672807852617, + "htmin_ele": 76177.11611285266, + "htmin_jd": 2461151.500220079, + "absolute_magnitudes": [ + 2.4527602455285358, + 2.935326484323726, + 2.667653477222837, + 2.6698211294198684, + 2.9324914534964, + 2.4647566555887437, + 2.837123706123827, + 3.3088190122834766, + 2.3208940419532453, + 2.9330495519000843, + 4.884584879911243, + 3.855789702473728 + ], + "ra_data_los": [ + 4.268555447761289, + 4.2757679637412895, + 4.282381342636115, + 4.287844958142504, + 4.295510917776434, + 4.302003078945247, + 4.3086654772619175, + 4.314676085433959, + 4.321663432280336, + 4.327530324867534, + 4.331910650932565, + 4.3358906783280355 + ], + "dec_data_los": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.07348646708577, + 1.0676655293254065, + 1.0625353098743917, + 1.0568295338648421, + 1.0530700757470863, + 1.0479884168122084, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "x_eci": [ + -0.19949752969882315, + -0.19831370967000816, + -0.19719537224748762, + -0.19653440624741567, + -0.19526604393742353, + -0.19419198311969488, + -0.1931713762708322, + -0.19171870682741307, + -0.19020697856453836, + -0.18946461302037865, + -0.18879990840657096, + -0.18794699339414994 + ], + "y_eci": [ + -0.41951346273949935, + -0.42489567883123996, + -0.42989189519454285, + -0.43469911459331356, + -0.44086253314794654, + -0.44623568023084337, + -0.45209532151521414, + -0.45626258010862597, + -0.4616673137779947, + -0.4676323484404185, + -0.4719169246748595, + -0.4752570683607163 + ], + "z_eci": [ + 0.885555865106421, + 0.8832527014775893, + 0.8810828244894001, + 0.8788691068257918, + 0.8760772790977996, + 0.8735921195735208, + 0.8708011481697114, + 0.8689467161138501, + 0.8664205657152649, + 0.8633789128216748, + 0.8611904613911451, + 0.8595386242905415 + ], + "x_eci_los": [ + -0.19947261818984635, + -0.19828723693595823, + -0.1971673332669236, + -0.19650478472685406, + -0.19523471543822826, + -0.19415896881448527, + -0.19313660866544666, + -0.19168228546062463, + -0.19016877668854015, + -0.18942455378532386, + -0.18875810448810454, + -0.18790350619325405 + ], + "y_eci_los": [ + -0.41952530837615254, + -0.4249080335800251, + -0.42990475582643994, + -0.4347125057399289, + -0.44087640777926657, + -0.4462500458800414, + -0.4521101754434669, + -0.4562778824308138, + -0.4616830511058688, + -0.4676485767305336, + -0.4719336470299235, + -0.4752742636762536 + ], + "z_eci_los": [ + 0.885555865106421, + 0.8832527014775893, + 0.8810828244894001, + 0.8788691068257918, + 0.8760772790977996, + 0.8735921195735208, + 0.8708011481697114, + 0.8689467161138501, + 0.8664205657152649, + 0.8633789128216748, + 0.8611904613911451, + 0.8595386242905415 + ], + "x_stat": -3413601.4235978713, + "y_stat": -1704419.3296659163, + "z_stat": 5094057.194026681, + "stat_eci": [ + -3413601.4235978713, + -1704419.3296659163, + 5094057.194026681 + ], + "stat_eci_los": [ + [ + -3413500.207340225, + -1704622.0299575112, + 5094057.194026681 + ], + [ + -3413495.2262395704, + -1704632.004559309, + 5094057.194026681 + ], + [ + -3413490.2500972687, + -1704641.969159245, + 5094057.194026681 + ], + [ + -3413485.273925883, + -1704651.9337446468, + 5094057.194026681 + ], + [ + -3413480.2977254046, + -1704661.89831553, + 5094057.194026681 + ], + [ + -3413475.316508251, + -1704671.8728591497, + 5094057.194026681 + ], + [ + -3413470.340249567, + -1704681.8374009654, + 5094057.194026681 + ], + [ + -3413465.358960294, + -1704691.8119432305, + 5094057.194026681 + ], + [ + -3413460.37765573, + -1704701.7864431958, + 5094057.194026681 + ], + [ + -3413455.4063112848, + -1704711.7409264494, + 5094057.194026681 + ], + [ + -3413450.4299362227, + -1704721.7054101357, + 5094057.194026681 + ], + [ + -3413445.4535320713, + -1704731.6698792966, + 5094057.194026681 + ] + ], + "plane_N": [ + -0.9633203118488703, + -0.08160610934519683, + -0.2556451049735453 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 6.724691477696396e-05, + 1.5706629728077543e-05, + 0.00010637754358927273, + 0.0002362632170843941, + 0.00027184572472634675, + 0.00034066933402676907, + 0.0005857413387279998, + 1.4225497938347832e-05, + 0.0003240762673397417, + 0.0002633833955488076, + 0.0005469632219021634, + 0.00042211398237902714 + ], + "ang_res_std": 0.0003236383693663661 + } + }, + { + "ObservedPoints.UK00CD": { + "meas1": [ + 3.2350197840765165, + 3.2477072959811264, + 3.2552292551574604, + 3.2758807935362744, + 3.293926737375255, + 3.3099222963095567, + 3.3239878762449746, + 3.3380897824049134, + 3.357389348527242, + 3.371805691892109, + 3.3873157784796977, + 3.4044770059058163, + 3.4212650205456225, + 3.4374557844393463 + ], + "meas2": [ + 1.2609999783504144, + 1.2612395779929737, + 1.2595198256953215, + 1.260472338963183, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814451, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153048, + 1.2547119136331997, + 1.2533115917530044, + 1.253010663890778 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 6.187651883518428e-05, + 0.0399378765188352, + 0.0798018765188349, + 0.11962187651883464, + 0.15939787651883444, + 0.1992378765188352, + 0.2390978765188352, + 0.27892987651883483, + 0.3187098765188352, + 0.3585698765188352, + 0.3983738765188345, + 0.4381978765188348, + 0.4779978765188344, + 0.5178458765188345 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 14, + "JD_data": [ + 2461151.500205559, + 2461151.5002060207, + 2461151.500206482, + 2461151.500206943, + 2461151.500207403, + 2461151.500207864, + 2461151.5002083257, + 2461151.5002087867, + 2461151.500209247, + 2461151.5002097087, + 2461151.5002101692, + 2461151.5002106302, + 2461151.500211091, + 2461151.500211552 + ], + "lat": 0.9302909996683025, + "lon": -0.039105358527994384, + "ele": 80.0, + "station_id": "UK00CD", + "azim_data": [ + 5.966419815139067, + 5.97542519016121, + 5.977778599822299, + 5.993954146606859, + 6.005275633071222, + 6.017243491584776, + 6.02792804365347, + 6.0375395619757395, + 6.051683132695393, + 6.063393040906969, + 6.074541567562709, + 6.0886289147110215, + 6.101951058174785, + 6.116263187282847 + ], + "elev_data": [ + 1.2028290600570986, + 1.2049672270465874, + 1.2077118383916157, + 1.2105467452617509, + 1.214565742676128, + 1.2171622436903107, + 1.2193966508469822, + 1.2223914732590124, + 1.226048237869094, + 1.2281100846973485, + 1.2315217824941145, + 1.2341667869259583, + 1.2373483805420769, + 1.2393083048928824 + ], + "ra_data": [ + 3.2350197840765156, + 3.2477072959811255, + 3.2552292551574595, + 3.275880793536273, + 3.293926737375254, + 3.3099222963095567, + 3.323987876244974, + 3.3380897824049134, + 3.3573893485272417, + 3.371805691892108, + 3.387315778479697, + 3.404477005905816, + 3.421265020545622, + 3.4374557844393454 + ], + "dec_data": [ + 1.2609999783504147, + 1.2612395779929737, + 1.2595198256953215, + 1.2604723389631833, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814453, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153052, + 1.2547119136331992, + 1.2533115917530042, + 1.2530106638907785 + ], + "magnitudes": [ + 4.86, + 4.32, + 5.89, + 4.98, + 2.84, + 2.54, + 2.93, + 2.47, + 2.12, + 2.31, + 2.72, + 2.1, + 2.7, + 2.14 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK00CD_20260421-000017.700500_9470", + "comment": "{\"ff_name\": \"FF_UK00CD_20260421_000013_479_0364800.fits\"}", + "incident_angle": 0.9673542667254906, + "weight": 0.6779664749397492, + "h_residuals": [ + 10.60965356642866, + -24.36076247965052, + 105.75947002249839, + -7.220710828385594, + 52.4106292421939, + 13.980469426264557, + -21.11566161888541, + 16.385999544642043, + 31.159896386556547, + -15.566908035957312, + 42.18031417134474, + 11.207776909509969, + 34.794840093942796, + -31.51281526197431 + ], + "h_res_rms": 38.65757618379624, + "v_residuals": [ + 3.2446947715231613, + -7.4621198683776795, + 32.17069118416699, + -2.20894625977602, + 15.980407194763492, + 4.270405033174591, + -6.460331591938528, + 5.002493265014071, + 9.503554253729916, + -4.7583795692783655, + 12.852019409716682, + 3.4197354351588864, + 10.601440013163899, + -9.632696675175211 + ], + "v_res_rms": 11.775952423901554, + "velocities": [ + 0.0, + 10471.680103624529, + 6537.224031186331, + 16633.282812297697, + 14826.52683173458, + 12873.383577289029, + 11286.270727626033, + 11554.888061040885, + 15564.781179446425, + 11390.115494189105, + 12773.925244016458, + 13582.991536140642, + 13615.88234822533, + 12542.882660933778 + ], + "velocities_prev_point": [ + 10743.85866816658, + 10743.85866816658, + 10743.85866816658, + 10743.85866816658, + 12007.055445941307, + 12492.235701727113, + 12536.279099510239, + 12472.845218100547, + 12649.080739643105, + 12670.109215040093, + 12686.294204401467, + 12730.115433503099, + 12785.779374174901, + 12813.830897857222 + ], + "length": [ + 0.0, + 417.56871581213187, + 678.1686145913418, + 1340.5059361770318, + 1930.2458674361035, + 2443.121469155308, + 2892.992220358482, + 3353.246521605858, + 3972.4135169242427, + 4426.4235205226205, + 4934.8768409354425, + 5475.8058958707115, + 6017.718013330074, + 6517.526801602965 + ], + "state_vect_dist": [ + 37.76837659225623, + 379.80033921987564, + 640.4002379990856, + 1302.7375595847757, + 1892.4774908438474, + 2405.3530925630516, + 2855.223843766226, + 3315.4781450136015, + 3934.6451403319866, + 4388.655143930364, + 4897.108464343187, + 5438.037519278456, + 5979.949636737819, + 6479.758425010708 + ], + "lag": [ + 281.3381680637624, + 106.26937614174375, + -150.0758670031762, + -4.113107714768603, + 69.82284087386734, + 66.06452591993548, + -0.9579939258560444, + -57.2338676012655, + 46.077274171508634, + -16.805993279080212, + -24.51975166286138, + -0.017129900050349534, + 25.779779637870888, + 8.85090948729794 + ], + "lag_line": [ + 12015.663931514431, + -83.75025379184109 + ], + "v_init": 12015.663931514431, + "v_init_stddev": null, + "jacchia_fit": [ + 0.0, + 0.0 + ], + "model_ra": [ + 3.2350317233883077, + 3.2476714150887465, + 3.255438140444483, + 3.2758702514549296, + 3.2941153488378436, + 3.3099924216832672, + 3.323910252516376, + 3.3381929059797426, + 3.3575930020386857, + 3.371733421626219, + 3.3876369233400556, + 3.404593711003503, + 3.4215849629232125, + 3.437217005959056 + ], + "model_dec": [ + 1.2611273499274973, + 1.2609464139140631, + 1.260793448991946, + 1.2603850642422985, + 1.259911335665489, + 1.2594122969591925, + 1.2589067455139373, + 1.2583285427696425, + 1.25746144668329, + 1.2567431069600323, + 1.2558671504145162, + 1.254848700705218, + 1.253736238689485, + 1.252625998752515 + ], + "model_azim": [ + 5.966643957880864, + 5.974911280194201, + 5.9800178789208704, + 5.993802064108692, + 6.006378939289549, + 6.017536643092251, + 6.0274870372718645, + 6.03788102220137, + 6.052329074285987, + 6.0630724131073555, + 6.075406678946376, + 6.088856757848179, + 6.102652853784429, + 6.115634949478984 + ], + "model_elev": [ + 1.202730398539852, + 1.2051959385233475, + 1.2067159430592809, + 1.210615868504016, + 1.2140581450105756, + 1.2170252460038622, + 1.2196057067116586, + 1.2222277443684135, + 1.225732770549091, + 1.2282693521279788, + 1.2310859268705145, + 1.234049575685928, + 1.2369803907988324, + 1.239645469881098 + ], + "model_fit1": [ + 5.966643957880864, + 5.974911280194201, + 5.9800178789208704, + 5.993802064108692, + 6.006378939289549, + 6.017536643092251, + 6.0274870372718645, + 6.03788102220137, + 6.052329074285987, + 6.0630724131073555, + 6.075406678946376, + 6.088856757848179, + 6.102652853784429, + 6.115634949478984 + ], + "model_fit2": [ + 1.202730398539852, + 1.2051959385233475, + 1.2067159430592809, + 1.210615868504016, + 1.2140581450105756, + 1.2170252460038622, + 1.2196057067116586, + 1.2222277443684135, + 1.225732770549091, + 1.2282693521279788, + 1.2310859268705145, + 1.234049575685928, + 1.2369803907988324, + 1.239645469881098 + ], + "meas_eci": [ + [ + -0.3035351281043153, + -0.02844121514996133, + 0.9523956758025997 + ], + [ + -0.3029229405820039, + -0.03226575808186291, + 0.9524686939341127 + ], + [ + -0.3042986687014469, + -0.034729084187786134, + 0.9519433864143417 + ], + [ + -0.3026179106160443, + -0.04088405047247195, + 0.9522346846189463 + ], + [ + -0.30295616797952424, + -0.046510880186298745, + 0.9518688451186214 + ], + [ + -0.30220529917399286, + -0.05135608475551451, + 0.9518584504587548 + ], + [ + -0.30152721261512533, + -0.05561523142857777, + 0.9518341694253778 + ], + [ + -0.30167832324719224, + -0.06005383655938584, + 0.9515165400544865 + ], + [ + -0.30143648571506654, + -0.0660779059164778, + 0.9511938579645285 + ], + [ + -0.3005904430157494, + -0.07044880838966484, + 0.9511479122429234 + ], + [ + -0.30091929891322594, + -0.07546789342433696, + 0.9506588097754449 + ], + [ + -0.3001680129514678, + -0.08077893212098153, + 0.9504598508754388 + ], + [ + -0.3000486040229262, + -0.08617384954180235, + 0.9500236328002793 + ], + [ + -0.2988875713280024, + -0.09110370995523792, + 0.9499296467307672 + ] + ], + "meas_eci_los": [ + [ + -0.30353512752019685, + -0.028441221383888878, + 0.9523956758025997 + ], + [ + -0.3029228461071481, + -0.03226664503462707, + 0.9524686939341127 + ], + [ + -0.30429846603725963, + -0.03473085989960501, + 0.9519433864143417 + ], + [ + -0.3026175532804857, + -0.040886695332201205, + 0.9522346846189463 + ], + [ + -0.30295562663884373, + -0.04651440615989257, + 0.9518688451186214 + ], + [ + -0.30220455226634335, + -0.05136047974617341, + 0.9518584504587548 + ], + [ + -0.301526242054227, + -0.05562049322827857, + 0.9518341694253778 + ], + [ + -0.30167710078673465, + -0.06005997721985251, + 0.9515165400544865 + ], + [ + -0.3014349488853584, + -0.06608491628985995, + 0.9511938579645285 + ], + [ + -0.30058859968688656, + -0.07045667302821534, + 0.9511479122429234 + ], + [ + -0.30091710526470994, + -0.07547663979962246, + 0.9506588097754449 + ], + [ + -0.30016543028477377, + -0.08078852849133539, + 0.9504598508754388 + ], + [ + -0.30004559881096554, + -0.08618431269743403, + 0.9500236328002793 + ], + [ + -0.2988841295616315, + -0.09111500073064646, + 0.9499296467307672 + ] + ], + "model_eci": [ + [ + -3432466.7540115756, + -1731741.8935464248, + 5173716.011629781 + ], + [ + -3432372.547769191, + -1732079.5683393998, + 5173489.145111228 + ], + [ + -3432313.743999133, + -1732290.3063399354, + 5173347.5431581 + ], + [ + -3432164.301102949, + -1732825.91596933, + 5172987.668751478 + ], + [ + -3432031.227056555, + -1733302.8168760038, + 5172667.221548597 + ], + [ + -3431915.4844218343, + -1733717.5583187286, + 5172388.519694615 + ], + [ + -3431813.9466015864, + -1734081.3479430394, + 5172144.033616263 + ], + [ + -3431710.0576202665, + -1734453.532469572, + 5171893.892688835 + ], + [ + -3431570.3102787603, + -1734954.2219146248, + 5171557.403865901 + ], + [ + -3431467.812772388, + -1735321.3520204683, + 5171310.628589792 + ], + [ + -3431353.024745032, + -1735732.5059074648, + 5171034.262150875 + ], + [ + -3431230.901985564, + -1736169.9185074358, + 5170740.239346225 + ], + [ + -3431108.549160772, + -1736608.122839488, + 5170445.669753482 + ], + [ + -3430995.6861842214, + -1737012.276212167, + 5170173.961306149 + ] + ], + "meas_lat": [ + 0.9348979840563066, + 0.9348736638383488, + 0.9348366981997408, + 0.9348063479155164, + 0.9347563423139719, + 0.9347257901580164, + 0.9346991840939688, + 0.9346614322637298, + 0.9346157521163507, + 0.9345905006711562, + 0.934546433761751, + 0.9345127904652669, + 0.9344712488826498, + 0.9344455500907388 + ], + "meas_lon": [ + -0.04164301451607974, + -0.04155291066409006, + -0.041513252115616614, + -0.04136276158768624, + -0.04124541609544114, + -0.041134549051738925, + -0.041037524018862015, + -0.040945674933504726, + -0.040817480073928235, + -0.040718389405427956, + -0.040618299375673295, + -0.04050220635063718, + -0.040391477328569035, + -0.04028058930887304 + ], + "meas_ht": [ + 81396.9675099336, + 81244.77789502541, + 81193.34068859647, + 80935.62631653334, + 80752.07851757792, + 80566.58131972801, + 80403.44378474863, + 80257.65942465847, + 80051.33834901436, + 79883.51877213769, + 79727.19354210321, + 79534.42486416693, + 79356.9731703905, + 79168.05194906335 + ], + "meas_range": [ + 87069.72999388228, + 86836.55913972364, + 86692.05673251014, + 86325.53975271186, + 86002.07075931023, + 85723.10812734772, + 85480.18898973276, + 85233.27293127988, + 84903.47168914345, + 84663.79807745767, + 84397.2537452506, + 84115.98487302613, + 83836.60845417317, + 83581.14830424251 + ], + "model_lat": [ + 0.934899505502217, + 0.9348701702014254, + 0.9348518594457105, + 0.9348053124719798, + 0.9347638562520294, + 0.9347277946223416, + 0.934696156319921, + 0.934663781456173, + 0.9346202190594269, + 0.9345882687471724, + 0.9345524799775028, + 0.9345143970336911, + 0.9344762361830826, + 0.9344410323178205 + ], + "model_lon": [ + -0.04164193466936269, + -0.04155538966397436, + -0.041502471430448676, + -0.04136349692543704, + -0.04124007338001663, + -0.04113312415058746, + -0.04103967585574465, + -0.04094400396225524, + -0.04081430089807224, + -0.040719977257233045, + -0.04061399294714445, + -0.0405010622126221, + -0.04038792316631062, + -0.0402838067317805 + ], + "model_ht": [ + 81393.91662581416, + 81251.79438980817, + 81163.08976953034, + 80937.70349190966, + 80737.05100419743, + 80562.56553501167, + 80409.51917280334, + 80252.95499555573, + 80042.40065984003, + 79887.99406011793, + 79715.1061188881, + 79531.20864436727, + 79347.00196171085, + 79177.11246162215 + ], + "model_range": [ + 87069.7307007453, + 86836.56287739017, + 86692.12721183259, + 86325.54008297449, + 86002.08821370563, + 85723.10937370706, + 85480.19184196735, + 85233.27465311192, + 84903.4779387753, + 84663.79964238926, + 84397.26526402768, + 84115.98568912731, + 83836.61634464488, + 83581.15480030599 + ], + "rbeg_lat": 0.934899505502217, + "rbeg_lon": -0.04164193466936269, + "rbeg_ele": 81393.91662581416, + "rbeg_jd": 2461151.500205559, + "rend_lat": 0.9344410323178205, + "rend_lon": -0.0402838067317805, + "rend_ele": 79177.11246162215, + "rend_jd": 2461151.500211552, + "htmin_lat": 0.9344410323178205, + "htmin_lon": -0.0402838067317805, + "htmin_ele": 79177.11246162215, + "htmin_jd": 2461151.500211552, + "absolute_magnitudes": [ + 5.160663993871118, + 4.6264868745522065, + 6.2001017019905555, + 5.2993034793540605, + 3.1674550176961667, + 2.8745104224398865, + 3.270672559130019, + 2.816954128451293, + 2.47537259597214, + 2.6715112232095644, + 3.0883581282452037, + 2.4756073077614653, + 3.082831290906207, + 2.529458163572881 + ], + "ra_data_los": [ + 3.235019804614262, + 3.247710223963076, + 3.2552350905836143, + 3.275889533472668, + 3.293938375945835, + 3.3099368393903763, + 3.324005326769998, + 3.3381101374401774, + 3.3574126051387827, + 3.371831855939726, + 3.387344844103591, + 3.404508976039951, + 3.4212998922560343, + 3.4374935606518706 + ], + "dec_data_los": [ + 1.2609999783504147, + 1.2612395779929737, + 1.2595198256953215, + 1.2604723389631833, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814453, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153052, + 1.2547119136331992, + 1.2533115917530042, + 1.2530106638907785 + ], + "x_eci": [ + -0.3035351281043153, + -0.3029229405820039, + -0.3042986687014469, + -0.3026179106160443, + -0.30295616797952424, + -0.30220529917399286, + -0.30152721261512533, + -0.30167832324719224, + -0.30143648571506654, + -0.3005904430157494, + -0.30091929891322594, + -0.3001680129514678, + -0.3000486040229262, + -0.2988875713280024 + ], + "y_eci": [ + -0.02844121514996133, + -0.03226575808186291, + -0.034729084187786134, + -0.04088405047247195, + -0.046510880186298745, + -0.05135608475551451, + -0.05561523142857777, + -0.06005383655938584, + -0.0660779059164778, + -0.07044880838966484, + -0.07546789342433696, + -0.08077893212098153, + -0.08617384954180235, + -0.09110370995523792 + ], + "z_eci": [ + 0.9523956758025997, + 0.9524686939341127, + 0.9519433864143417, + 0.9522346846189463, + 0.9518688451186214, + 0.9518584504587548, + 0.9518341694253778, + 0.9515165400544865, + 0.9511938579645285, + 0.9511479122429234, + 0.9506588097754449, + 0.9504598508754388, + 0.9500236328002793, + 0.9499296467307672 + ], + "x_eci_los": [ + -0.30353512752019685, + -0.3029228461071481, + -0.30429846603725963, + -0.3026175532804857, + -0.30295562663884373, + -0.30220455226634335, + -0.301526242054227, + -0.30167710078673465, + -0.3014349488853584, + -0.30058859968688656, + -0.30091710526470994, + -0.30016543028477377, + -0.30004559881096554, + -0.2988841295616315 + ], + "y_eci_los": [ + -0.028441221383888878, + -0.03226664503462707, + -0.03473085989960501, + -0.040886695332201205, + -0.04651440615989257, + -0.05136047974617341, + -0.05562049322827857, + -0.06005997721985251, + -0.06608491628985995, + -0.07045667302821534, + -0.07547663979962246, + -0.08078852849133539, + -0.08618431269743403, + -0.09111500073064646 + ], + "z_eci_los": [ + 0.9523956758025997, + 0.9524686939341127, + 0.9519433864143417, + 0.9522346846189463, + 0.9518688451186214, + 0.9518584504587548, + 0.9518341694253778, + 0.9515165400544865, + 0.9511938579645285, + 0.9511479122429234, + 0.9506588097754449, + 0.9504598508754388, + 0.9500236328002793, + 0.9499296467307672 + ], + "x_stat": -3406048.6136386427, + "y_stat": -1729266.1246223755, + "z_stat": 5090787.796276712, + "stat_eci": [ + -3406048.6136386427, + -1729266.1246223755, + 5090787.796276712 + ], + "stat_eci_los": [ + [ + -3406048.5781234126, + -1729266.1945749386, + 5090787.796276712 + ], + [ + -3406043.5503640417, + -1729276.0974638269, + 5090787.796276712 + ], + [ + -3406038.5225758795, + -1729286.0003380955, + 5090787.796276712 + ], + [ + -3406033.4998326153, + -1729295.8932045705, + 5090787.796276712 + ], + [ + -3406028.4872221127, + -1729305.766042447, + 5090787.796276712 + ], + [ + -3406023.464421443, + -1729315.6588797665, + 5090787.796276712 + ], + [ + -3406018.4365182575, + -1729325.5616956379, + 5090787.796276712 + ], + [ + -3406013.413660088, + -1729335.454503771, + 5090787.796276712 + ], + [ + -3406008.39584702, + -1729345.3373042115, + 5090787.796276712 + ], + [ + -3406003.3678716016, + -1729355.24004859, + 5090787.796276712 + ], + [ + -3405998.350001125, + -1729365.1228198803, + 5090787.796276712 + ], + [ + -3405993.3270280496, + -1729375.0155696734, + 5090787.796276712 + ], + [ + -3405988.309100192, + -1729384.8983118308, + 5090787.796276712 + ], + [ + -3405983.286083734, + -1729394.791004777, + 5090787.796276712 + ] + ], + "plane_N": [ + -0.9508405323585105, + -0.0543339152478599, + -0.30487720098102244 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 0.00012742337477004607, + 0.0002934025890114486, + 0.001275134678871608, + 8.747317337504222e-05, + 0.000637107786152886, + 0.00017052484520883212, + 0.00025833001748669024, + 0.00020100450053262754, + 0.0003836891026201303, + 0.00019227104764286688, + 0.0005224611490777694, + 0.00013929892102162514, + 0.00043386000256940454, + 0.00039426294165447807 + ], + "ang_res_std": 0.0004704327154024583 + } + } + ], + "los_mini_status": true, + "t_ref_station": 2, + "time_diffs_final": [ + 0.0, + -0.0019011817811903332, + -0.00014963188985710543 + ], + "intersection_list": [ + { + "PlaneIntersection.UK002Z_UK00AS": { + "conv_angle": 1.3977829287229966, + "radiant_eci": [ + -0.2119721838741445, + 0.805179044558886, + 0.5538542222163385 + ], + "radiant_eq": [ + 1.8282164010696587, + 0.5869862134998625 + ], + "w1": [ + -0.9772483431518503, + -0.17887979587700067, + -0.1139635662567909 + ], + "w2": [ + -0.16103171664643293, + -0.5877633547814152, + 0.792844893412324 + ], + "rcpa_stat1": [ + -73631.49434216082, + -13477.82963291005, + -8586.668622006024 + ], + "rcpa_stat2": [ + -13242.80545538603, + -48336.04163995439, + 65201.38329524699 + ], + "cpa_eci": [ + -3403324.3337847604, + -1842095.9935247626, + 5097804.319209092 + ], + "weight": 0.0024214538252984794 + } + }, + { + "PlaneIntersection.UK002Z_UK00CD": { + "conv_angle": 1.3408619526894816, + "radiant_eci": [ + -0.2224668519112711, + 0.8032092732613852, + 0.5525969264731806 + ], + "radiant_eq": [ + 1.8409953958825356, + 0.5854769115887319 + ], + "w1": [ + -0.9749128323753357, + -0.18752636752381052, + -0.11991176236393772 + ], + "w2": [ + -0.2150761118164841, + -0.5932692097626505, + 0.7757408786917855 + ], + "rcpa_stat1": [ + -72116.41498877588, + -13871.731802662609, + -8870.132928298488 + ], + "rcpa_stat2": [ + -15374.393477834303, + -42408.9602147798, + 55452.67396327532 + ], + "cpa_eci": [ + -3401809.2544313753, + -1842489.8956945152, + 5097520.854902799 + ], + "weight": 0.002371116245910391 + } + }, + { + "PlaneIntersection.UK00AS_UK00CD": { + "conv_angle": 0.0578872564894713, + "radiant_eci": [ + -0.17933529144848642, + 0.8982410335263312, + 0.4012504192279862 + ], + "radiant_eq": [ + 1.767856898383979, + 0.4128815721461222 + ], + "w1": [ + -0.19672893468820898, + -0.4323680153644721, + 0.8799747868809822 + ], + "w2": [ + -0.25216678471696985, + -0.43620698939268593, + 0.8637912798184956 + ], + "rcpa_stat1": [ + -16494.934626699258, + -36252.32942686249, + 73782.36763061871 + ], + "rcpa_stat2": [ + -19567.098351078548, + -33847.856181592375, + 67026.62662722637 + ], + "cpa_eci": [ + -3430096.3582245708, + -1740671.6590927788, + 5167839.561657299 + ], + "weight": 1.3053179448789485e-05 + } + } + ], + "rbeg_lat": 0.934899505502217, + "rbeg_lon": -0.04164193466936269, + "rbeg_ele": 81393.91662581416, + "rbeg_ele_wgs84": 81445.53494089004, + "rbeg_jd": 2461151.500205559, + "rend_lat": 0.9337921890458436, + "rend_lon": -0.038372180120937446, + "rend_ele": 76055.10224458351, + "rend_ele_wgs84": 76106.4147786973, + "rend_jd": 2461151.5002204534, + "htmin_lat": 0.9337921890458436, + "htmin_lon": -0.038372180120937446, + "htmin_ele": 76055.10224458351, + "htmin_ele_wgs84": 76106.4147786973, + "htmin_jd": 2461151.5002204534, + "state_vect": [ + -3432369.342207372, + -1731768.1585065187, + 5173694.939644102 + ], + "incident_angles": [ + 0.5027492433632643, + 1.2638605479641993, + 0.9659504756451831 + ], + "state_vect_mini": [ + -3432458.2336270753, + -1731772.435690051, + 5173695.492595176 + ], + "radiant_eci_mini": [ + -0.22559573032168215, + 0.8086697491907738, + 0.5432861154165067 + ], + "radiant_eq_mini": [ + 1.842850952886513, + 0.5743463248927236 + ], + "v_init": 12967.718791996132, + "v0z": -4413.997141557832, + "v_avg": 12312.85904311762, + "timing_minimization_successful": true, + "velocity_fit": [ + 12967.718791996132, + -244.37218876758848 + ], + "jacchia_fit": [ + 43.65312744412464, + 2.1924329546040098 + ], + "timing_res": 7.041064582452035e-05, + "timing_stddev": 0.008391105161092927, + "state_vect_avg": [ + -3430549.6763656787, + -1738607.335042225, + 5169100.684310519 + ], + "jd_avg": 2461151.5002130065, + "orbit": { + "Orbit": { + "ra": 1.842850952886513, + "dec": 0.5743463248927236, + "azimuth_apparent": 5.215879750489071, + "elevation_apparent": 0.3473192026194278, + "v_avg": 12312.85904311762, + "v_init": 12967.718791996132, + "v_init_stddev": 10.13578330407507, + "ra_norot": 1.837726586059179, + "dec_norot": 0.5860709340821146, + "azimuth_apparent_norot": 5.227200613194143, + "elevation_apparent_norot": 0.3538606337963674, + "v_avg_norot": 12082.953555046177, + "v_init_norot": 12737.81330392469, + "jd_ref": 2461151.500205559, + "jd_dyn": 2461151.5010063, + "lst_ref": 3.6088563643950953, + "lon_ref": -0.0416338437576483, + "lat_ref": 0.9348968523981452, + "ht_ref": 81381.06170645228, + "ht_ref_wgs84": 81432.6793071907, + "lat_geocentric": 0.9317248461260144, + "zc": 1.2250089673888134, + "zg": 1.6682930856125058, + "v_inf": 12967.718791996132, + "v_g": 6669.855942091, + "ra_g": 1.560157053440776, + "dec_g": 0.21033070922631758, + "L_g": 1.560182664795458, + "B_g": -0.19871170575289931, + "meteor_pos": [ + -129457249.42153406, + -76394902.82501395, + 10345.503483171698 + ], + "v_h": 35477.25261044726, + "v_h_x": 14.598935218756163, + "v_h_y": -32.30740600750524, + "v_h_z": 1.3183551608964306, + "L_h": 1.9952090815491528, + "B_h": -0.037169134413427715, + "la_sun": 0.5331351764388832, + "a": 1.749324945375272, + "e": 0.4367632022782812, + "i": 0.037382200258429005, + "peri": 5.925529697530141, + "node": 3.6728731864642414, + "pi": 3.3152175768147956, + "b": -0.0130840548179203, + "q": 0.9852841804078888, + "Q": 2.513365710342655, + "true_anomaly": 0.35949713851885046, + "eccentric_anomaly": 0.22656761811223144, + "mean_anomaly": 0.12845566895513721, + "last_perihelion": "2026-04-03 17:22:02.404589", + "n": 0.007434915611796666, + "Tj": 4.017379995926152, + "T": 2.313696397548099 + } + }, + "uncertainties": { + "MCUncertainties": { + "ci": 95, + "mc_traj_list": null, + "state_vect_mini": [ + 23.302927645551236, + 37.2070223921984, + 47.8951574781733 + ], + "state_vect_mini_ci": [ + [ + -3432495.617382003, + -3432423.837728241 + ], + [ + -1731807.3913284293, + -1731696.1065867343 + ], + [ + 5173640.759083595, + 5173792.756573728 + ] + ], + "x": 23.302927645551236, + "x_ci": [ + -3432495.617382003, + -3432423.837728241 + ], + "y": 37.2070223921984, + "y_ci": [ + -1731807.3913284293, + -1731696.1065867343 + ], + "z": 47.8951574781733, + "z_ci": [ + 5173640.759083595, + 5173792.756573728 + ], + "vx": 13.858206008974927, + "vx_ci": [ + -2938.2878636842593, + -2861.212929938162 + ], + "vy": 59.54333260289223, + "vy_ci": [ + 10483.746446609672, + 10492.11339262698 + ], + "vz": 57.58213998290366, + "vz_ci": [ + 6945.863075051403, + 7122.386599949237 + ], + "radiant_eci_mini": [ + 0.0016920905173864757, + 0.002356945317379389, + 0.002939081393705436 + ], + "radiant_eci_mini_ci": [ + [ + -0.22617399930035348, + -0.2214846318885415 + ], + [ + 0.8059754297844866, + 0.8128571058966497 + ], + [ + 0.5381818809865024, + 0.5473502583390504 + ] + ], + "rbeg_lon": 2.2088929690296427e-06, + "rbeg_lon_ci": [ + 6.2415409901294865, + 6.241546962406092 + ], + "rbeg_lon_m": 8.38540985943337, + "rbeg_lat": 2.638433710383294e-06, + "rbeg_lat_ci": [ + 0.9348949134986161, + 0.9349036888013773 + ], + "rbeg_lat_m": 16.864870452262917, + "rbeg_ele": 45.558677417463386, + "rbeg_ele_ci": [ + 81325.7017569644, + 81464.2043463111 + ], + "rbeg_ele_wgs84": 45.5588599685256, + "rbeg_ele_wgs84_ci": [ + 81377.31980654877, + 81515.822891308 + ], + "rend_lon": 3.407449543509572e-06, + "rend_lon_ci": [ + 6.244800260516159, + 6.244811360843754 + ], + "rend_lon_m": 12.954725654542509, + "rend_lat": 1.0453520900156793e-06, + "rend_lat_ci": [ + 0.933790446617204, + 0.9337939283069673 + ], + "rend_lat_m": 6.681867643440438, + "rend_ele": 32.15807998869947, + "rend_ele_ci": [ + 76036.61348161136, + 76133.00603886494 + ], + "rend_ele_wgs84": 32.15826496087123, + "rend_ele_wgs84_ci": [ + 76087.92684692098, + 76184.31958180107 + ], + "htmin_lon": 3.407449543509572e-06, + "htmin_lon_ci": [ + 6.244800260516159, + 6.244811360843754 + ], + "htmin_lon_m": 12.954725654542509, + "htmin_lat": 1.0453520900156793e-06, + "htmin_lat_ci": [ + 0.933790446617204, + 0.9337939283069673 + ], + "htmin_lat_m": 6.681867643440438, + "htmin_ele": 32.15807998869947, + "htmin_ele_ci": [ + 76036.61348161136, + 76133.00603886494 + ], + "htmin_ele_wgs84": 32.15826496087123, + "htmin_ele_wgs84_ci": [ + 76087.92684692098, + 76184.31958180107 + ], + "ra": 0.002611503709103583, + "ra_ci": [ + 1.8370325403202352, + 1.8440354813447923 + ], + "dec": 0.003499436816389006, + "dec_ci": [ + 0.5682786814957027, + 0.579194832887812 + ], + "azimuth_apparent": 0.0011488927073448102, + "azimuth_apparent_ci": [ + 5.2149278320957935, + 5.218617334102857 + ], + "elevation_apparent": 0.003985943562293819, + "elevation_apparent_ci": [ + 0.3401376381944182, + 0.35146749322636645 + ], + "v_avg": 25.363986414427266, + "v_avg_ci": [ + 12263.698119250277, + 12331.272064862434 + ], + "v_init": 35.83555406641995, + "v_init_ci": [ + 12906.156161188343, + 13012.85741906931 + ], + "lon_ref": 9.746851701222101e-06, + "lon_ref_ci": [ + 6.241530852167341, + 6.241562416968685 + ], + "lat_ref": 4.037368389248857e-06, + "lat_ref_ci": [ + 0.9348930977264116, + 0.9349048456367035 + ], + "ht_ref": 46.23031409446649, + "ht_ref_ci": [ + 81327.7139370313, + 81465.6113865276 + ], + "lat_geocentric": 4.058469875306498e-06, + "lat_geocentric_ci": [ + 0.9317210791689499, + 0.9317328894806066 + ], + "zc": 0.003984415255032236, + "zc_ci": [ + 1.2208682718192776, + 1.2321888843316788 + ], + "zg": 0.010437731100424239, + "zg_ci": [ + 1.6565960535519642, + 1.6867956381526732 + ], + "v_inf": 35.83555406641995, + "v_inf_ci": [ + 12906.156161188343, + 13012.85741906931 + ], + "v_g": 69.86456353616636, + "v_g_ci": [ + 6549.278440215807, + 6757.316401539255 + ], + "ra_g": 0.005218583891039907, + "ra_g_ci": [ + 1.550331619290657, + 1.5642537989451053 + ], + "dec_g": 0.009175375088677046, + "dec_g_ci": [ + 0.19423320526384985, + 0.22168953806563507 + ], + "L_g": 0.005261542789721588, + "L_g_ci": [ + 1.550262999253746, + 1.5642996777101477 + ], + "B_g": 0.009147790639302643, + "B_g_ci": [ + -0.2147525041002339, + -0.18736660983219178 + ], + "meteor_pos": [ + 0.023402287765977765, + 0.04500344767424416, + 0.04074976099956943 + ], + "meteor_pos_ci": [ + [ + -129457249.45909907, + -129457249.3870311 + ], + [ + -76394902.87402098, + -76394902.73235357 + ], + [ + 10345.446462113892, + 10345.578152204636 + ] + ], + "v_h": 85.48609930680463, + "v_h_ci": [ + 35328.09302105182, + 35577.45936832089 + ], + "v_h_x": 0.03321975068903373, + "v_h_x_ci": [ + 14.53657918185901, + 14.62520287469976 + ], + "v_h_y": 0.08104467976059955, + "v_h_y_ci": [ + -32.40782446857918, + -32.16672196346313 + ], + "v_h_z": 0.04568102257541548, + "v_h_z_ci": [ + 1.2602424833462527, + 1.3973518963978855 + ], + "L_h": 0.00021478363911639652, + "L_h_ci": [ + 1.9947075813557262, + 1.9953602674305384 + ], + "B_h": 0.0013798257879346354, + "B_h_ci": [ + -0.03956396089577628, + -0.035430681123816095 + ], + "la_sun": 0.0, + "la_sun_ci": [ + 0.5331351764388832, + 0.5331351764388832 + ], + "a": 0.020764918399534503, + "a_ci": [ + 1.7136415146239252, + 1.7742356355051931 + ], + "e": 0.00662985086508511, + "e_ci": [ + 0.4252246127134539, + 0.44463172993888717 + ], + "i": 0.0013875907380813083, + "i_ci": [ + 0.03563537240700222, + 0.03979064362004738 + ], + "peri": 0.0036974620554850617, + "peri_ci": [ + 5.918617880304961, + 5.928510238183485 + ], + "node": 6.736619195035191e-05, + "node_ci": [ + 3.6727828832139906, + 3.6729846226879324 + ], + "pi": 0.0036316227110077645, + "pi_ci": [ + 3.308403222800056, + 3.3181100489377644 + ], + "b": 0.0006164034149704704, + "b_ci": [ + -0.01417379179358153, + -0.01237568509476685 + ], + "q": 0.0001887248437724549, + "q_ci": [ + 0.9848735253457906, + 0.9853733247776466 + ], + "Q": 0.04135376621516677, + "Q_ci": [ + 2.4423263887544406, + 2.563117103921014 + ], + "true_anomaly": 0.003631669498448071, + "true_anomaly_ci": [ + 0.3566046072044634, + 0.3663115643921616 + ], + "eccentric_anomaly": 0.00415997564946805, + "eccentric_anomaly_ci": [ + 0.22255528058867302, + 0.23405984987107875 + ], + "mean_anomaly": 0.0038950653377452164, + "mean_anomaly_ci": [ + 0.12441501198245487, + 0.13542723129154152 + ], + "last_perihelion": 0.21162646186227413, + "last_perihelion_ci": [ + 2461133.838177647, + 2461134.4084006962 + ], + "n": 0.00013353213752772863, + "n_ci": [ + 0.00727888466701047, + 0.007668381602780202 + ], + "T": 0.04112839790029051, + "T_ci": [ + 2.243267062584157, + 2.3632930877217384 + ], + "Tj": 0.03300359258971792, + "Tj_ci": [ + 3.978562778709054, + 4.074873090067962 + ], + "ra_norot": 0.0026819047506464275, + "ra_norot_ci": [ + 1.831751910635052, + 1.8389425768418368 + ], + "dec_norot": 0.003535179504345732, + "dec_norot_ci": [ + 0.5799393752075872, + 0.590966744099576 + ], + "v_avg_norot": 25.73922847929732, + "v_avg_norot_ci": [ + 12033.412162374418, + 12102.036345823391 + ], + "v_init_norot": 36.25000387087065, + "v_init_norot_ci": [ + 12675.549576533595, + 12783.621700030268 + ], + "azimuth_apparent_norot": 0.0011626219913197264, + "azimuth_apparent_norot_ci": [ + 5.226249501183861, + 5.229971656717279 + ], + "elevation_apparent_norot": 0.004037622005774978, + "elevation_apparent_norot_ci": [ + 0.3465823432479639, + 0.35805015466321544 + ], + "ht_ref_wgs84": 46.23066689867216, + "ht_ref_wgs84_ci": [ + 81379.33149963012, + 81517.23000879795 + ] + } + }, + "orbit_cov": [ + [ + 4.839509762694922e-05, + 1.2839048270422571e-06, + 0.001529597070645031, + -2.8134845951441427e-05, + 0.0015235437188076953, + -0.0005796707377483675 + ], + [ + 1.2839048270422571e-06, + 3.92532878805147e-08, + 4.283334419962881e-05, + -7.366720739670027e-07, + 4.318110319692552e-05, + -1.5189142360366112e-05 + ], + [ + 0.001529597070645031, + 4.283334419962881e-05, + 0.04932644359606424, + -0.0008848174909047756, + 0.049355279485603636, + -0.018239993412701383 + ], + [ + -2.8134845951441427e-05, + -7.366720739670027e-07, + -0.0008848174909047754, + 1.640345974762046e-05, + -0.0008803958803679891, + 0.00033761695055935486 + ], + [ + 0.0015235437188076953, + 4.318110319692551e-05, + 0.04935527948560364, + -0.0008803958803679891, + 0.049434753433829214, + -0.018149168163731216 + ], + [ + -0.0005796707377483675, + -1.5189142360366112e-05, + -0.018239993412701383, + 0.00033761695055935486, + -0.018149168163731216, + 0.006957026666261073 + ] + ], + "state_vect_cov": [ + [ + 594.8872861480645, + -256.66487170381055, + -967.7624393923977, + 583.6215942851601, + 26.71726004970638, + -985.1200079805594 + ], + [ + -256.66487170381055, + 1522.6343263519718, + 718.6734107124669, + 359.1627911110367, + 46.990250811313, + -641.6507010433324 + ], + [ + -967.7624393923977, + 718.673410712467, + 2514.098708757615, + -931.2754297849413, + -11.982008124620437, + 2312.506952848055 + ], + [ + 583.6215942851601, + 359.1627911110367, + -931.2754297849413, + 923.3221697191502, + 62.170755663341666, + -1621.7192914483069 + ], + [ + 26.71726004970638, + 46.99025081131299, + -11.982008124620435, + 62.170755663341666, + 7.208948184864825, + -91.24287656294936 + ], + [ + -985.1200079805594, + -641.6507010433324, + 2312.5069528480544, + -1621.719291448307, + -91.24287656294938, + 3635.949624000401 + ] + ], + "phase_1_only": false, + "avg_radiant": [ + -0.21707459130810688, + 0.8045078859660982, + 0.5528523159278415 + ], + "radiant_eq": [ + 1.8343430182556113, + 0.5857833676735065 + ], + "best_conv_inter": { + "PlaneIntersection.UK002Z_UK00AS": { + "conv_angle": 1.3977829287229966, + "radiant_eci": [ + -0.2119721838741445, + 0.805179044558886, + 0.5538542222163385 + ], + "radiant_eq": [ + 1.8282164010696587, + 0.5869862134998625 + ], + "w1": [ + -0.9772483431518503, + -0.17887979587700067, + -0.1139635662567909 + ], + "w2": [ + -0.16103171664643293, + -0.5877633547814152, + 0.792844893412324 + ], + "rcpa_stat1": [ + -73631.49434216082, + -13477.82963291005, + -8586.668622006024 + ], + "rcpa_stat2": [ + -13242.80545538603, + -48336.04163995439, + 65201.38329524699 + ], + "cpa_eci": [ + -3403324.3337847604, + -1842095.9935247626, + 5097804.319209092 + ], + "weight": 0.0024214538252984794 + } + }, + "stations_time_dict": { + "OrderedDict": {} + }, + "stations_time_dict_copy": { + "OrderedDict": {} + }, + "v_init_stddev": 10.13578330407507, + "time_diffs": [ + 0.0, + 0.00026701713965955896, + 6.187651883518428e-05 + ], + "longname": "20260421_000017.759_UK", + "pre_mc_longname": "20260421_000017.759_UK", + "save_date": "2026-04-21 09:33:47.459946+00:00" +} \ No newline at end of file diff --git a/archive/samfunctions/searchArchive/requirements.txt b/archive/lambdas/searchArchive/requirements.txt similarity index 100% rename from archive/samfunctions/searchArchive/requirements.txt rename to archive/lambdas/searchArchive/requirements.txt diff --git a/archive/samfunctions/searchArchive/samconfig.toml b/archive/lambdas/searchArchive/samconfig.toml similarity index 100% rename from archive/samfunctions/searchArchive/samconfig.toml rename to archive/lambdas/searchArchive/samconfig.toml diff --git a/archive/samfunctions/searchArchive/searchArchive.py b/archive/lambdas/searchArchive/searchArchive.py similarity index 100% rename from archive/samfunctions/searchArchive/searchArchive.py rename to archive/lambdas/searchArchive/searchArchive.py diff --git a/archive/samfunctions/searchArchive/template.yml b/archive/lambdas/searchArchive/template.yml similarity index 100% rename from archive/samfunctions/searchArchive/template.yml rename to archive/lambdas/searchArchive/template.yml diff --git a/archive/samfunctions/searchArchive/testEvent.json b/archive/lambdas/searchArchive/testEvent.json similarity index 100% rename from archive/samfunctions/searchArchive/testEvent.json rename to archive/lambdas/searchArchive/testEvent.json diff --git a/archive/samfunctions/updateDnsRecords/lambda_function.py b/archive/lambdas/updateDnsRecords/lambda_function.py similarity index 100% rename from archive/samfunctions/updateDnsRecords/lambda_function.py rename to archive/lambdas/updateDnsRecords/lambda_function.py diff --git a/archive/samfunctions/updateDnsRecords/samconfig.toml b/archive/lambdas/updateDnsRecords/samconfig.toml similarity index 100% rename from archive/samfunctions/updateDnsRecords/samconfig.toml rename to archive/lambdas/updateDnsRecords/samconfig.toml diff --git a/archive/samfunctions/updateDnsRecords/template.yaml b/archive/lambdas/updateDnsRecords/template.yaml similarity index 100% rename from archive/samfunctions/updateDnsRecords/template.yaml rename to archive/lambdas/updateDnsRecords/template.yaml diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index 7676e445e..fa6eb3529 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -4,6 +4,7 @@ alias h='history' alias df='df -h' alias du='du -h' +alias ls='ls --color=auto' alias data='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/data && pwd ; fi' alias logs='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/logs && pwd ; fi' @@ -16,30 +17,32 @@ alias stats='if [ "$DATADIR" == "" ] ; then echo select env first; else tail $DA alias matchstatus='if [ "$SRC" == "" ] ; then echo select env first; else grep "Running" $SRC/logs/matchJob.log && grep TRAJ $SRC/logs/matchJob.log | grep SOLVING && echo -n "Completed " && grep Observations: $SRC/logs/matchJob.log | wc -l && grep "nightlyJob" $SRC/logs/nightlyJob.log ; fi ' alias spacecalc='ls -1 | egrep -v "ukmon-shared" | while read i ; do \du -s $i ; done | sort -n' -alias startcalc='~/prod/utils/stopstart-calcengine.sh start' -alias stopcalc='~/prod/utils/stopstart-calcengine.sh stop' +alias startcalc='$SRC/utils/stopstartCalcengine.sh start' +alias stopcalc='$SRC/utils/stopstartCalcengine.sh stop' function dev { - source ~/dev/config.ini >/dev/null - conda activate $HOME/miniconda3/envs/${WMPL_ENV} - PS1="(wmpl) (dev) [\W]\$ " + source ~/dev/config.ini + #conda activate $HOME/miniconda3/envs/${WMPL_ENV} + conda activate ${WMPL_ENV} + PS1="(dev) [\W]\$ " cd ~/dev } function prd { - source ~/prod/config.ini >/dev/null - conda activate $HOME/miniconda3/envs/${WMPL_ENV} - PS1="(wmpl) (prd) [\W]\$ " + source ~/prod/config.ini #>/dev/null + #conda activate $HOME/miniconda3/envs/${WMPL_ENV} + conda activate ${WMPL_ENV} + PS1="(prd) [\W]\$ " cd ~/prod } function calcserver { - sts=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State --output text --profile ukmonshared) - ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) + sts=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State --output text) + ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) isrunning=$(echo $sts | cut -d " " -f 1) if [ $isrunning -ne 16 ] ; then - /home/ec2-user/prod/utils/stopstart-calcengine.sh start + $HOME/prod/utils/stopstartCalcengine.sh start echo "starting server on ${ipaddr}... waiting 10s..." sleep 10 fi - ssh -i ~/.ssh/markskey.pem ec2-user@$ipaddr + ssh -i ~/.ssh/markskey.pem $SERVERUSERID@$ipaddr } diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index b772d578b..a7f39fa35 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -22,8 +22,7 @@ export PATH=$PATH:/usr/local/geos/bin:/usr/local/proj4/bin if shopt -q login_shell ; then echo "" - echo "Type 'dev' to activate the dev environment" - echo "Type 'prd' to activate the dev environment" + echo "THIS IS THE PROD SERVER!!" echo "" echo " Some handy aliases that work in either environment are" echo " logs => go to the logs folder" @@ -34,23 +33,36 @@ if shopt -q login_shell ; then echo " tnj => tail the nightly job log" echo " spacecalc => display space usage in the current folder" echo "" - fi - export NVM_DIR="$HOME/.nvm" + prd +fi +export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion export PATH=$PATH:$(dirname $(which node)) # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! -__conda_setup="$('/home/ec2-user/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +__conda_setup="$('/home/ubuntu/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" else - if [ -f "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" ]; then - . "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" + if [ -f "/home/ubuntu/miniconda3/etc/profile.d/conda.sh" ]; then + . "/home/ubuntu/miniconda3/etc/profile.d/conda.sh" else - export PATH="/home/ec2-user/miniconda3/bin:$PATH" + export PATH="/home/ubuntu/miniconda3/bin:$PATH" fi fi unset __conda_setup # <<< conda initialize <<< + +# If this is an xterm set the title to user@host:dir +case "$TERM" in + xterm*|rxvt*) + PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" + ;; + *) + ;; +esac +export rundate=$(date +%Y%m%d) +conda activate wmpl + diff --git a/archive/server_setup/.condaon b/archive/server_setup/.condaon index b69b186cd..a8766953f 100644 --- a/archive/server_setup/.condaon +++ b/archive/server_setup/.condaon @@ -1,13 +1,14 @@ # >>> conda initialize >>> -__conda_setup="$('/home/ec2-user/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +__conda_setup="$('~/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" else - if [ -f "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" ]; then - . "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" + if [ -f "~/miniconda3/etc/profile.d/conda.sh" ]; then + . "~/miniconda3/etc/profile.d/conda.sh" else - export PATH="/home/ec2-user/miniconda3/bin:$PATH" + export PATH="~/miniconda3/bin:$PATH" fi fi unset __conda_setup +. ~/miniconda3/etc/profile.d/conda.sh # <<< conda initialize <<< diff --git a/archive/server_setup/.vimrc b/archive/server_setup/.vimrc new file mode 100644 index 000000000..2fd7a2b6c --- /dev/null +++ b/archive/server_setup/.vimrc @@ -0,0 +1 @@ +colo slate diff --git a/archive/server_setup/logs.conf b/archive/server_setup/logs.conf new file mode 100644 index 000000000..910c3bf2b --- /dev/null +++ b/archive/server_setup/logs.conf @@ -0,0 +1,24 @@ +daily +rotate 7 +notifempty +~/prod/logs/*.log { + daily + rotate 45 + compress + delaycompress + dateext + notifempty + maxage 45 + # create +} + +~/dev/logs/*.log { + daily + rotate 45 + compress + delaycompress + dateext + notifempty + maxage 45 + # create +} diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh new file mode 100755 index 000000000..52d5add0f --- /dev/null +++ b/archive/server_setup/migrateSftpAccts.sh @@ -0,0 +1,44 @@ +#!/bin/bash + + +addOneUser() { + userid=$1 + srchost=$2 + grep -w $userid /etc/passwd + if [ $? -eq 1 ] ; then + dt=$(date +%Y-%m-%d) + logger -s -t addSftpUser "Creating unix user $userid" + sudo useradd --system --shell /usr/sbin/nologin --groups sftp --home /var/sftp/$userid --comment "${dt}" $userid + sudo mkdir /var/sftp/$userid + sudo chown root:sftp /var/sftp/$userid + sudo chmod 751 /var/sftp/$userid + # create the .ssh folder, platepar folder and empty client copy of the ini file + sudo mkdir /var/sftp/$userid/.ssh + sudo mkdir /var/sftp/$userid/platepar + sudo touch /var/sftp/$userid/ukmon.ini.client + # make these three writeable by the client + sudo chown $userid:$userid /var/sftp/$userid/platepar /var/sftp/$userid/.ssh /var/sftp/$userid/ukmon.ini.client + else + logger -s -t addSftpUser "Unix user $userid already exists" + fi + sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid + sudo cat /var/sftp/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini + sudo mv /tmp/$userid.ini /var/sftp/$userid/ukmon.ini + sudo scp /var/sftp/$userid/ukmon.ini $srchost:/var/sftp/$userid/ +} + +if [ $# -lt 2 ] ; then + echo "Usage: ./migrateSftpAccounts.sh oldservername userfile" + exit +fi + +echo "Warning: this must only be run on the new server" +read -p "press ctrl-c to quit or enter to continue" + +oldserver=$1 +srcfile=$2 + +cat $srcfile | while read stn +do + addOneUser $stn $oldserver +done \ No newline at end of file diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index cd413ef7e..1bf7db16e 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -1,51 +1,172 @@ -# how to move accounts to a new server -The basic process is to extract the user accounts from the current server along with -group and password info, then import it back in on the new server. Its important to avoid -accidentally overwriting system or otherwise-existing accounts on the new server. +# Replacing the UKMON helper server +The ukmon helper server provides two services. +* batch processing +* camera authentication and key management -On most systems, accoutns below 500 are system accounts. Check though, as some AWS servers create -new accounts starting at 1000 and working both upwards and downwards! +## how to move batch processing +* Build a new Ubuntu server from the Terraform by cloning the existing batch server details in `terraform/ukmda/batchserver.tf` and making any necessary changes then deploying it with terraform in the normal way. +* I do not recommend using Amazon Linux as this doesn't contain many of the base requirements like C++, development libraries, GEOS and PROJ, so you would need to install and/or build these from source which is tedious. +* If you want to set the hostname to eg `batchserver1`, then do the following + * edit `/etc/cloud/cloud.cfg` and set `preserve_hostname: true` + * run `sudo hostnamctl hostname batchserver1` + * now the new hostname should be preserved through boot -Steps in brief. NB must all be done as root, of course. +### Prerequisites +* install some prerequisites via apt +``` bash +sudo apt-get install unzip net-tools dos2unix mariadb-server +``` + +### Miniconda +* Install miniconda with the default settings + +### WMPL +* Install WMPL + * create a 'wmpl' conda environment with at least python 3.13 + * install the python requirements + * See note below for how to get PyQt5 working. + * Alternatively you can delete subfolders of wmpl that rely on QT (`CAM0` `MetSIM` and `Utils/DynamicMassFit.py`). +``` bash +conda create -n wmpl python=3.13 +mkdir -p ~/src +cd ~/src +git clone --recursive git@github.com:markmac99/WesternMeteorPyLib.git +cd WesternMeteorPyLib +conda activate wmpl +pip install -r requirements.txt +python setup.py +``` + +### UKMDA Dataprocessing +* Clone the ukmda git repo and then install all the code using the deployment script. Both dev and prod envs should be created, though arguably the dev env should be on a separate server. +``` bash +cd ~/src +git clone git@github.com:ukmda/ukmda-dataprocessing.git +cd ukmda-dataprocessing +./install_or_upgrade.sh PROD +``` +* double check that all required SSM variables exist in the account holding the server. These +are used to build the config file and are deployed via terraform so should be present! For example: +``` bash +aws ssm get-parameters --region eu-west-2 --names prod_siteurl --query Parameters[0].Value +"https://archive.ukmeteors.co.uk" +``` + +### SSH and other API keys +copy the below files from ~/.ssh on the old server, and make sure permissions are correct (should all be 0600). At a minimum you need the following files +``` bash +gmailcreds.json +gmailtoken.json +``` +These are used to authenticate against gmail. The keys are currently specific to Mark McIntyre's account and will need to be replaced with keys for `ukmeteors@gmail.com` + +You may also need the `github` keys though this depends on how you authenticate with GitHub. + +## data +Replicate `~/prod/data`, `~/prod/logs` and `~/keymgmt` to the new server. (once for prod and once for dev). + + This will have to be repeated every day till golive (strictly, the key data only needs to be replicated if a new camera is added). Also keep the MariaDB SQL database up to date. +``` bash +cd $DATADIR +rsync -avz ukmonhelper2:prod/data/ . +cd $SRC/logs +rsync -avz ukmonhelper2:prod/logs/ . +cd ~ +rsync -avz ukmonhelper2:keymgmt/ ./keymgmt +$SRC/utils/loadSingleCsvMDB.sh +$SRC/utils/loadMatchCsvMDB.sh + +``` +## Mariadb database +Find the mariadb config file (normally in /etc, might ber called `my.cnf`) and ensure that `bind-address` is set to `0.0.0.0`. The default on many installations is to bind to 127.0.0.1 which allows traffic only from localhost. The UKMON APIs access the database externally. After changing the config file, restart the database. + + +Now sudo to root and run the following to set a root password +``` bash +mysql -u root -p +ALTER USER 'root'@'localhost' IDENTIFIED VIA mysql_native_password USING PASSWORD('ROOTPASSWORD'); +FLUSH PRIVILEGES; +quit +``` +Pbviously, replace `ROOTPASSWORD` with the password in the SSM variable `prod_dbpw` -## On the old server +Exit the root shell, and as a 'normal' user execute the following: +``` bash +cd $SRC/database/ +mysql -u root -pROOTPASSWORD < ddl/create_dbs_and_users.sql +mysql -u root -pROOTPASSWORD < ddl/create_tables.sql +mysql -u root -pROOTPASSWORD < ddl/create_brightness_table.sq +``` +Now dump and reload the databases from the old server. +Make sure the old server can connect to the new one then on the old server, run the following: ``` bash -mkdir /root/move/ -export UGIDLIMIT=500 -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/passwd > /root/move/passwd.mig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/group > /root/move/group.mig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534) {print $1}' /etc/passwd | tee - |egrep -f - /etc/shadow > /root/move/shadow.mig -cp /etc/gshadow /root/move/gshadow.mig +mysqldump -u batch -p'passwd' ukmon | ssh newserver mysql -u batch -p'passwd' ukmon +``` -# Also backup the user homedirs. In our case, they're all in /var/sftp -tar cvfz /root/move/varsftp.tar.gz /var/sftp -# now copy the files to target server, -scp /root/move/* newserver:/tmp +### To install PyQt5 +* needs at least 3GB memory so add 2GB swap if the server has less. +``` bash +sudo fallocate -l 2G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab + +sudo apt install qtbase5-dev qt5-qmake +pip install pyqt5 ``` -## On the new server -In summary: backup the existing files, remove any accounts from the .mig files -that are already present in the target, then append the filtered data. +### To get conda working in batch scripts +This is done by sourcing a file `.condaon` which is automatically installed by the ukmda deployment script and sourced from the environments' config.ini files. Make sure any scripts activate conda like this: -When comparing groups, remember new ids will get added to the sftp group. +``` bash +source $HOME/prod/config.ini >/dev/null 2>&1 +conda activate ${WMPL_ENV} +``` +## how to move accounts to a new server + +### Install and configure sftp +First set up the SFTP server on the new host. +Run the following as root: ``` bash -mkdir -p /root/move/bkp -mv /tmp/*.mig /tmp/arsftp.tar.gz /root/move -cp /etc/passwd /etc/group /etc/shadow /etc/gshadow /root/move/bkp +groupadd sftp +mkdir -p /var/sftp +chown root:root /var/sftp +chmod 751 /var/sftp +``` -cd / -tar -xvf /root/move/varsftp.tar.gz . +Add this to /etc/ssh/sshd_config +``` bash +Match group sftp +ChrootDirectory /var/sftp/%u +AllowTCPForwarding no +X11Forwarding no +ForceCommand internal-sftp +``` + +and then reload sshd +``` bash +service sshd reload +``` -export UGIDLIMIT=500 -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/passwd > /root/move/passwd.orig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/group > /root/move/group.orig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534) {print $1}' /etc/passwd | tee - |egrep -f - /etc/shadow > /root/move/shadow.orig +### Now move the user accounts and update camera config -cd /root/move -diff passwd.orig passwd.mig -diff shadow.orig shadow.mig -diff group.orig group.mig +First, ensure that root on the new server can connect to the old server as the batch user by creating a default SSH keypair for root, and adding its public half to root's authorized_keys file on the old server. + +* on the new server as the standard batch user: + * create a folder `move` + * Create a list of the desired SFTP user accounts using +``` bash +ssh oldserver "sudo ls -1 /var/sftp" > ./move/sftp_accts.txt ``` + * Edit the list to exclude defunct accounts and other entries not related to a camera account. + * use `$SRC/$utils/migrateSftpAccts.sh` to create accounts on the new server and copy over the user data +``` bash +$SRC/utils/migrateSftpAccts.sh oldserverFQDN ./move/sftp_accts.txt +``` +This will also update the `ukmon.ini` file for each station on both server. Upon next connection, the station should update its local ini file and thereafter connect to the new server. + +This means that the old server must be kept running for a few days till all cameras have switched over. diff --git a/archive/server_setup/ssh-setup.sh b/archive/server_setup/ssh-setup.sh deleted file mode 100644 index f2c0cbf77..000000000 --- a/archive/server_setup/ssh-setup.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -echo $1 -scp -i ~/.ssh/markskey.pem ~/.ssh/ukmon-shared-keys $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/markskey.pem $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/config $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/gm* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/ukmon* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/marksk* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/s3* $1:.ssh - -ssh -i ~/.ssh/markskey.pem $1 mkdir setup -ssh -i ~/.ssh/markskey.pem $1 mkdir data -ssh -i ~/.ssh/markskey.pem $1 mkdir src - diff --git a/archive/server_setup/install-cartopy.sh b/archive/server_setup/unused/install-cartopy.sh similarity index 100% rename from archive/server_setup/install-cartopy.sh rename to archive/server_setup/unused/install-cartopy.sh diff --git a/archive/server_setup/install-geos.sh b/archive/server_setup/unused/install-geos.sh similarity index 100% rename from archive/server_setup/install-geos.sh rename to archive/server_setup/unused/install-geos.sh diff --git a/archive/server_setup/install-proj4.sh b/archive/server_setup/unused/install-proj4.sh similarity index 100% rename from archive/server_setup/install-proj4.sh rename to archive/server_setup/unused/install-proj4.sh diff --git a/archive/server_setup/install-sqlite3.sh b/archive/server_setup/unused/install-sqlite3.sh similarity index 100% rename from archive/server_setup/install-sqlite3.sh rename to archive/server_setup/unused/install-sqlite3.sh diff --git a/archive/server_setup/install_opencv.sh b/archive/server_setup/unused/install_opencv.sh similarity index 100% rename from archive/server_setup/install_opencv.sh rename to archive/server_setup/unused/install_opencv.sh diff --git a/archive/server_setup/newserver.sh b/archive/server_setup/unused/newserver.sh similarity index 100% rename from archive/server_setup/newserver.sh rename to archive/server_setup/unused/newserver.sh diff --git a/archive/server_setup/remount-s3.sh b/archive/server_setup/unused/remount-s3.sh similarity index 100% rename from archive/server_setup/remount-s3.sh rename to archive/server_setup/unused/remount-s3.sh diff --git a/archive/server_setup/yum-installs.sh b/archive/server_setup/unused/yum-installs.sh similarity index 100% rename from archive/server_setup/yum-installs.sh rename to archive/server_setup/unused/yum-installs.sh diff --git a/archive/shwrinfo/costnotes.html b/archive/shwrinfo/costnotes.html index 09a5714c8..08b211c37 100644 --- a/archive/shwrinfo/costnotes.html +++ b/archive/shwrinfo/costnotes.html @@ -4,19 +4,4 @@

  • Amounts of less than $0.10 per day are consolidated under Other.
  • AWS cost data may lag by up to a day, so yesterday's costs data are incomplete.
  • AWS charge VAT at the start of the month for the previous month's total bill.
    • -
  • Until September 2023, costs were split across two accounts for historical reasons. We've now consolidated into one account.
    • -
    -2023 Costs prior to 13th September were as follows: -
  • S3: $1135
    • -
  • ECS: $212
    • -
  • EC2-other: $159
    • -
  • EC2-instance: $131
    • -
  • Cloudwatch: $44
    • -
  • Registrar: $29
    • -
  • KMS: $23
    • -
  • DynamoDB: $15
    • -
  • Route53: $6
    • -
  • ECR: $5
    • -
  • Other: £8
    • - -Total: $1768 + VAT +
    \ No newline at end of file diff --git a/archive/static_content/js/searchdialog.js b/archive/static_content/js/searchdialog.js index 5b9e0ea52..d5716b036 100644 --- a/archive/static_content/js/searchdialog.js +++ b/archive/static_content/js/searchdialog.js @@ -70,6 +70,17 @@ form.addEventListener("submit", function (event) { op = op + "l:" + strstat + "_"; } //jQuery.support.cors = true; + if (d1 == "" ){ + console.log("missing start date"); + const stdt = new Date(Date.now()); + stdt.setHours(stdt.getHours() - 12); + d1 = stdt.toISOString(); + } + if (d2 == "" ){ + console.log("missing end date"); + const enddt = new Date(Date.now()); + d2 = enddt.toISOString(); + } payload = {"d1": d1, "d2": d2, "opts": op }; console.log(payload); document.getElementById("searchresults").innerHTML = "Searching...."; @@ -96,6 +107,9 @@ function myFunc(myObj) { txt = "" txt += ""; txt += ""; + if (myObj.length == 0){ + alert("No data available, check dates"); + } for (x in myObj) { console.log(myObj[x]); var dtarr = myObj[x].split(','); diff --git a/archive/static_content/search/index.html b/archive/static_content/search/index.html index 5fa067bec..391ede95f 100644 --- a/archive/static_content/search/index.html +++ b/archive/static_content/search/index.html @@ -78,8 +78,9 @@

    Search the Archive.



    This page allows you to search the archive for specific events in the matched and single-station detections.
    - Time should be entered as local time, so in the UK summer you should enter the time in BST. + Time should be entered as local time, so in the UK summer you should enter the time in BST.

    +

    NOTE - default date range is the last 24 hours. This might be a HUGE amount of data during busy showers.

    To search the livestream, go over to this page.

    diff --git a/archive/terraform/mjmm/aws.tf b/archive/terraform/mjmm/aws.tf index e18c7d50e..a9871b9e3 100644 --- a/archive/terraform/mjmm/aws.tf +++ b/archive/terraform/mjmm/aws.tf @@ -6,8 +6,8 @@ provider "aws" { } provider "aws" { - alias = "eeacct" - region = var.remote_region - profile = "ukmonshared" + profile = var.profile + region = "eu-west-1" + alias = "eu-west-1-prov" } diff --git a/archive/terraform/mjmm/crossacct-role.tf b/archive/terraform/mjmm/crossacct-role.tf index fabcd4ecd..858365ade 100644 --- a/archive/terraform/mjmm/crossacct-role.tf +++ b/archive/terraform/mjmm/crossacct-role.tf @@ -1,7 +1,5 @@ # Copyright (C) 2018-2023 Mark McIntyre -data "aws_caller_identity" "eeacct" {provider = aws.eeacct } - # RESTRICT THESE PERMISSIONS!! data "aws_iam_policy" "terraformpol" { name = "AdministratorAccess" diff --git a/archive/terraform/mjmm/dev_ssm_parameters.tf b/archive/terraform/mjmm/dev_ssm_parameters.tf index e622d5475..819844f67 100644 --- a/archive/terraform/mjmm/dev_ssm_parameters.tf +++ b/archive/terraform/mjmm/dev_ssm_parameters.tf @@ -32,7 +32,7 @@ resource "aws_ssm_parameter" "dev_livebucket" { resource "aws_ssm_parameter" "dev_siteurl" { name = "dev_siteurl" type = "String" - value = "http://mjmm-ukmonarchive.co.uk.s3-website.eu-west-2.amazonaws.com" + value = "https://www.ukmeteors.co.uk/dummy/" tags = { "billingtag" = "ukmon" } @@ -50,12 +50,30 @@ resource "aws_ssm_parameter" "dev_envname" { resource "aws_ssm_parameter" "dev_calcinstance" { name = "dev_calcinstance" type = "String" - value = "i-08cd1d5f6e1056f6b" #"i-0da38ed8aea1a1d85" + value = "i-0ab47af23705beb31" tags = { "billingtag" = "ukmon" } } +resource "aws_ssm_parameter" "dev_calcuser" { + name = "dev_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "dev_calcip" { + name = "dev_calcserverip" + type = "String" + value = "172.32.16.137" + tags = { + "billingtag" = "ukmon" + } +} + resource "aws_ssm_parameter" "dev_backupinstance" { name = "dev_backupinstance" type = "String" @@ -118,3 +136,4 @@ resource "aws_ssm_parameter" "dev_batchloggroup" { "billingtag" = "ukmon" } } + diff --git a/archive/terraform/mjmm/iam.tf b/archive/terraform/mjmm/iam.tf index 766e2d668..1da045ac5 100644 --- a/archive/terraform/mjmm/iam.tf +++ b/archive/terraform/mjmm/iam.tf @@ -198,7 +198,7 @@ resource "aws_iam_role_policy" "stsAssumeLambda" { Statement = { Action = "sts:AssumeRole" Effect = "Allow" - Resource = "arn:aws:iam::822069317839:role/service-role/S3FullAccess" + Resource = "arn:aws:iam::183798037734:role/s3AccessForRadio" } Version = "2012-10-17" } diff --git a/archive/terraform/mjmm/s3.tf b/archive/terraform/mjmm/s3.tf index 740ba4cc6..5a05aef5a 100644 --- a/archive/terraform/mjmm/s3.tf +++ b/archive/terraform/mjmm/s3.tf @@ -1,5 +1,6 @@ # Copyright (C) 2018-2023 Mark McIntyre ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmonarchive-co-uk" { bucket = "mjmm-ukmonarchive.co.uk" tags = { @@ -52,8 +53,9 @@ data "aws_iam_policy_document" "websiteacesspolicy" { resources = ["${aws_s3_bucket.mjmm-ukmonarchive-co-uk.arn}/*"] } } - +*/ ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmon-shared" { bucket = "mjmm-ukmon-shared" timeouts {} @@ -113,8 +115,9 @@ resource "aws_s3_bucket_versioning" "shared_versioning" { status = "Suspended" } } - +*/ ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmon-live" { bucket = "mjmm-ukmon-live" tags = { @@ -175,4 +178,4 @@ resource "aws_s3_bucket_versioning" "live_versioning" { status = "Suspended" } } - +*/ diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 683a1367c..8889f3a64 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -1,6 +1,45 @@ # Copyright (C) 2018-2023 Mark McIntyre # SSM parameters for use in the environment config +resource "aws_ssm_parameter" "prod_dbhost" { + provider = aws.eu-west-1-prov + name = "prod_dbhost" + type = "String" + value = "3.11.55.160" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbname" { + provider = aws.eu-west-1-prov + name = "prod_dbname" + type = "String" + value = "ukmon" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbpw" { + provider = aws.eu-west-1-prov + name = "prod_dbpw" + type = "SecureString" + value = "Batch33mdl" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbuser" { + provider = aws.eu-west-1-prov + name = "prod_dbuser" + type = "String" + value = "batch" + tags = { + "billingtag" = "ukmda" + } +} resource "aws_ssm_parameter" "prod_websitebucket" { name = "prod_websitebucket" @@ -50,12 +89,31 @@ resource "aws_ssm_parameter" "prod_envname" { resource "aws_ssm_parameter" "prod_calcinstance" { name = "prod_calcinstance" type = "String" - value = "i-04cd701c3cfc980f5" # "i-08cd1d5f6e1056f6b" + value = "i-0ab47af23705beb31" + #value = "i-04cd701c3cfc980f5" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "prod_calcuser" { + name = "prod_calcuser" + type = "String" + value = "ubuntu" + #value = "ec2-user" # "ubuntu" tags = { "billingtag" = "ukmon" } } +resource "aws_ssm_parameter" "prod_calcip" { + name = "prod_calcserverip" + type = "String" + value = "172.32.16.137" + tags = { + "billingtag" = "ukmon" + } +} resource "aws_ssm_parameter" "prod_backupinstance" { name = "prod_backupinstance" type = "String" @@ -119,3 +177,11 @@ resource "aws_ssm_parameter" "prod_batchloggroup" { } } +resource "aws_ssm_parameter" "prod_gmapsapikey" { + name = "prod_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmon" + } +} diff --git a/archive/terraform/mjmm/ec2.tf b/archive/terraform/mjmm/ukmonhelper_ec2.tf similarity index 100% rename from archive/terraform/mjmm/ec2.tf rename to archive/terraform/mjmm/ukmonhelper_ec2.tf diff --git a/archive/terraform/mjmm/variables.tf b/archive/terraform/mjmm/variables.tf index f1d68fb92..bad991dce 100644 --- a/archive/terraform/mjmm/variables.tf +++ b/archive/terraform/mjmm/variables.tf @@ -11,8 +11,8 @@ variable "webbucket" {default = "ukmda-website"} variable "sharedbucket" {default = "ukmda-shared"} variable "livebucket" {default = "ukmda-live"} -variable "dev_sharedbucket" { default = "mjmm-ukmon-shared" } -variable "dev_webbucket" { default = "mjmm-ukmonarchive.co.uk" } -variable "dev_livebucket" { default = "mjmm-ukmon-live" } +variable "dev_webbucket" { default = "ukmda-website" } +variable "dev_sharedbucket" { default = "ukmda-shared" } +variable "dev_livebucket" { default = "ukmda-live" } variable "vpc_id" { default = "vpc-a19015c8" } diff --git a/archive/terraform/ukmda/batchserver.tf b/archive/terraform/ukmda/batchserver.tf new file mode 100644 index 000000000..7ff5993f8 --- /dev/null +++ b/archive/terraform/ukmda/batchserver.tf @@ -0,0 +1,47 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +resource "aws_instance" "batchserver" { + ami = "ami-06f0cf249257c89b3" + instance_type = "t4g.micro" + iam_instance_profile = aws_iam_instance_profile.calcserverrole.name + key_name = aws_key_pair.marks_key.key_name + force_destroy = false + + root_block_device { + tags = { + "Name" = "batchservervol" + "billingtag" = "ukmda" + } + volume_size = 50 + volume_type = "gp3" + throughput = 125 + iops = 3000 + encrypted = true + kms_key_id = aws_kms_key.container_key.arn + } + + tags = { + "Name" = "batchserver" + "billingtag" = "ukmda" + "Route53FQDN" = "batchserver.ukmeteors.co.uk" + "DNSRecordType" = "A" + } + primary_network_interface { + network_interface_id = aws_network_interface.batchserver_if.id + } + metadata_options { + http_tokens = "required" + } +} + +resource "aws_network_interface" "batchserver_if" { + subnet_id = aws_subnet.ec2_subnet.id + description = "Primary network interface" + private_ips = [var.batchserverip] + security_groups = [aws_security_group.ec2_secgrp.id] + ipv6_address_list_enabled = false + tags = { + "Name" = "batchserver" + "billingtag" = "ukmda" + } +} diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/calcserver.tf similarity index 61% rename from archive/terraform/ukmda/ec2.tf rename to archive/terraform/ukmda/calcserver.tf index 29c5fd71c..a1aa2a468 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/calcserver.tf @@ -1,53 +1,12 @@ # Copyright (C) 2018-2023 Mark McIntyre -resource "aws_instance" "calc_server" { - ami = "ami-0df2d8f6def0bd716" - instance_type = "c8g.2xlarge" - iam_instance_profile = aws_iam_instance_profile.calcserverrole.name - key_name = aws_key_pair.marks_key.key_name - force_destroy = false - tags = { - "Name" = "Calcengine" - "billingtag" = "ukmda" - "Route53FQDN" = "calcengine.ukmeteors.co.uk" - "DNSRecordType" = "A" - } - root_block_device { - tags = { - "Name" = "calcengine" - "billingtag" = "ukmda" - } - volume_size = 100 - } - primary_network_interface { - network_interface_id = aws_network_interface.calcserver_if.id - } - - metadata_options { - http_tokens = "required" - } -} - -# elastic network interface attached to the calc server - -resource "aws_network_interface" "calcserver_if" { - subnet_id = aws_subnet.ec2_subnet.id - description = "Primary network interface" - private_ips = [var.calcserverip] - security_groups = [aws_security_group.ec2_secgrp.id] - ipv6_address_list_enabled = false - tags = { - "Name" = "calcengine" - "billingtag" = "ukmda" - } -} ################################################ # Ubuntu calc server ################################################ resource "aws_instance" "ubuntu_calc_server" { ami = "ami-0bdf149a42243bde8" - instance_type = "c6g.4xlarge" + instance_type = "c8g.2xlarge" iam_instance_profile = aws_iam_instance_profile.calcserverrole.name key_name = aws_key_pair.marks_key.key_name tags = { @@ -58,10 +17,10 @@ resource "aws_instance" "ubuntu_calc_server" { } root_block_device { tags = { - "Name" = "calcengine2" + "Name" = "calcengine_ub" "billingtag" = "ukmda" } - volume_size = 100 + volume_size = 120 } primary_network_interface { network_interface_id = aws_network_interface.ubuntu_calcserver_if.id diff --git a/archive/terraform/ukmda/cloudwatch.tf b/archive/terraform/ukmda/cloudwatch.tf index 61ab690a0..ee336b1dd 100644 --- a/archive/terraform/ukmda/cloudwatch.tf +++ b/archive/terraform/ukmda/cloudwatch.tf @@ -21,7 +21,7 @@ resource "aws_cloudwatch_metric_alarm" "calcServerIdle" { "arn:aws:automate:${var.region}:ec2:stop", ] dimensions = { - "InstanceId" = aws_instance.calc_server.id + "InstanceId" = aws_instance.ubuntu_calc_server.id } tags = { "billingtag" = "ukmda" diff --git a/archive/terraform/ukmda/dev_ssm_variables.tf b/archive/terraform/ukmda/dev_ssm_variables.tf new file mode 100644 index 000000000..bca543597 --- /dev/null +++ b/archive/terraform/ukmda/dev_ssm_variables.tf @@ -0,0 +1,191 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +# SSM parameters for use in Lambdas and the batch + +resource "aws_ssm_parameter" "dev_dbhost" { + provider = aws.eu-west-1-prov + name = "dev_dbhost" + type = "String" + value = "3.11.55.160" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_dbname" { + provider = aws.eu-west-1-prov + name = "dev_dbname" + type = "String" + value = "ukmon" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_dbpw" { + provider = aws.eu-west-1-prov + name = "dev_dbpw" + type = "SecureString" + value = "Batch33mdl" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_rootdbpw" { + provider = aws.eu-west-1-prov + name = "dev_rootdbpw" + type = "SecureString" + value = "Wombat33mdb" + tags = { + "billingtag" = "ukmda" + } +} + + +resource "aws_ssm_parameter" "dev_dbuser" { + provider = aws.eu-west-1-prov + name = "dev_dbuser" + type = "String" + value = "batch" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_websitebucket" { + name = "dev_websitebucket" + type = "String" + value = var.dev_websitebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_sharedbucket" { + name = "dev_sharedbucket" + type = "String" + value = var.dev_sharedbucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_livebucket" { + name = "dev_livebucket" + type = "String" + value = var.dev_livebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_siteurl" { + name = "dev_siteurl" + type = "String" + value = "https://www.ukmeteors.co.uk/dummy/" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_envname" { + name = "dev_envname" + type = "String" + value = "DEV" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_calcinstance" { + name = "dev_calcinstance" + type = "String" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_calcuser" { + name = "dev_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmda" + } +} + + +resource "aws_ssm_parameter" "dev_calcserverip" { + name = "dev_calcserverip" + type = "String" + value = var.ubuntu_calcserverip + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_wmplhome" { + name = "dev_wmplhome" + type = "String" + value = "$HOME/src/wmpldev" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_rmshome" { + name = "dev_rmshome" + type = "String" + value = "$HOME/src/RMS" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_srcdir" { + name = "dev_srcdir" + type = "String" + value = "$HOME/dev" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_caminfo" { + name = "dev_caminfo" + type = "String" + value = "$HOME/dev/data/consolidated/camera-details.csv" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_sshkey" { + name = "dev_sshkey" + type = "String" + value = "$HOME/.ssh/markskey.pem" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_batchloggroup" { + name = "dev_batchloggroup" + type = "String" + value = "/ukmondev/nightlyjob" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_gmapsapikey" { + name = "dev_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmda" + } +} + diff --git a/archive/terraform/ukmda/ecs_vpc.tf b/archive/terraform/ukmda/ecs_vpc.tf index 6c7243599..3f88c92a9 100644 --- a/archive/terraform/ukmda/ecs_vpc.tf +++ b/archive/terraform/ukmda/ecs_vpc.tf @@ -5,10 +5,6 @@ #data used by the code in several places data "aws_region" "current" {} -# some variables for the cluster and task defns -variable "ecsloggroup" { default = "/ecs/trajcont" } -variable "containername" { default = "trajcont" } - # create a VPC for the cluster resource "aws_vpc" "ecs_vpc" { cidr_block = "172.128.0.0/16" diff --git a/archive/terraform/ukmda/ecscluster-test.tf b/archive/terraform/ukmda/ecscluster-test.tf new file mode 100644 index 000000000..44c532659 --- /dev/null +++ b/archive/terraform/ukmda/ecscluster-test.tf @@ -0,0 +1,73 @@ +# terraform to create ECS cluster +# Copyright (C) 2018-2023 Mark McIntyre + +variable "ecsloggrouptest" { default = "/ecs/trajcontest" } +variable "containernametest" { default = "trajconttest" } + +# create a cluster +resource "aws_ecs_cluster" "trajsolvertest" { + name = "trajsolvertest" + tags = { + "billingtag" = "ukmda" + } +} + +# declare the capacity provider type, in this case FARGATE +resource "aws_ecs_cluster_capacity_providers" "trajsolvertest_cap" { + cluster_name = aws_ecs_cluster.trajsolvertest.name + capacity_providers = ["FARGATE", "FARGATE_SPOT"] +} + +data "template_file" "tasktest_json_template" { + template = file("files/trajsolver/trajsolver_container.json") + vars = { + acctid = data.aws_caller_identity.current.account_id + regionid = "eu-west-2" + repoid = "calcengine/trajsolvertest" + contname = var.containernametest + loggrp = var.ecsloggrouptest + } +} + +# define the task +# the definition of the container it runs are in the webapp.json file +resource "aws_ecs_task_definition" "trajsolvertest_task" { + family = "trajsolvertest" + container_definitions = data.template_file.tasktest_json_template.rendered + cpu = 4096 + memory = 8192 + network_mode = "awsvpc" + tags = { + billingtag = "ukmda" + } + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.ecstaskrole.arn + task_role_arn = aws_iam_role.ecstaskrole.arn + runtime_platform { + operating_system_family = "LINUX" + } +} + +# create a local file containing the clustername and a few other details +# +resource "null_resource" "createECSdetailstest" { + triggers = { + clusname = join(",", tolist([aws_ecs_cluster.trajsolvertest.name, + aws_subnet.ecs_subnet.id, + aws_security_group.ecssecgrp.id, + aws_iam_role.ecstaskrole.arn, var.ecsloggrouptest, + var.containernametest])) + } + provisioner "local-exec" { + command = "echo $env:CLUSNAME $env:SECGRP $env:SUBNET $env:IAMROLE $env:LOGGRP $env:CONTNAME > ../../ukmon_pylib/traj/clusdetailstest-ukmda.txt" + interpreter = ["pwsh.exe", "-command"] + environment = { + CLUSNAME = "${aws_ecs_cluster.trajsolvertest.name}" + SECGRP = "${aws_security_group.ecssecgrp.id}" + SUBNET = "${aws_subnet.ecs_subnet.id}" + IAMROLE = "${aws_iam_role.ecstaskrole.arn}" + LOGGRP = "${var.ecsloggrouptest}" + CONTNAME = "${var.containernametest}" + } + } +} diff --git a/archive/terraform/ukmda/ecscluster.tf b/archive/terraform/ukmda/ecscluster.tf index 974d4c596..2dbb16566 100644 --- a/archive/terraform/ukmda/ecscluster.tf +++ b/archive/terraform/ukmda/ecscluster.tf @@ -1,6 +1,10 @@ # terraform to create ECS cluster # Copyright (C) 2018-2023 Mark McIntyre +# some variables for the cluster and task defns +variable "ecsloggroup" { default = "/ecs/trajcont" } +variable "containername" { default = "trajcont" } + # create a cluster resource "aws_ecs_cluster" "trajsolver" { name = "trajsolver" diff --git a/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py b/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py index 654ad06a9..40c3a4214 100644 --- a/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py +++ b/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py @@ -61,28 +61,19 @@ def storeInDDb(evtdets, camdets): return response['ResponseMetadata']['HTTPStatusCode'] -def updateLiveTable(event, dtval): - record = event['Records'][0] - fname = record['s3']['object']['key'] - _, barefname = os.path.split(fname) - if 'P.jpg' not in barefname: - print(f'{barefname} not a jpg') - return +def updateLiveTable(evtdets, camdets, fname): + statname = camdets['camid'] + dtval = evtdets['dtval'] expdate = int((dtval + datetime.timedelta(days=90)).timestamp()) tstamp = str(int(dtval.timestamp()*1000)) yr = dtval.strftime('%Y') mth = dtval.strftime('%m') ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table('live') - if barefname[0] == 'M': - statname = barefname[17:].replace('P.jpg','').replace('_',' ') - else: - statname = barefname[3:9] - - print(f'inserting {barefname} with timestamp {dtval}') + print(f'inserting {fname} with timestamp {dtval}') response = table.put_item( Item={ - 'image_name': barefname, + 'image_name': fname, 'timestamp': tstamp, 'image_timestamp': tstamp, 'station_name': statname, @@ -94,19 +85,15 @@ def updateLiveTable(event, dtval): return response['ResponseMetadata']['HTTPStatusCode'] -def processXml(event): - record = event['Records'][0] - fname = record['s3']['object']['key'] +def processXml(remote_xmlname): buck = 'ukmda-live' s3 = boto3.resource('s3') - if '.xml' not in fname: - fname = fname.replace('P.jpg','.xml') - _, barefname = os.path.split(fname) + _, barefname = os.path.split(remote_xmlname) tmpdir = mkdtemp() xmlname = os.path.join(tmpdir, barefname) try: - s3.meta.client.download_file(buck, fname, xmlname) + s3.meta.client.download_file(buck, remote_xmlname, xmlname) except Exception: print('xml file not available') return None, None @@ -157,7 +144,12 @@ def processXml(event): def lambda_handler(event, context): - evtdets, camdets = processXml(event) - if evtdets is not None: - storeInDDb(evtdets, camdets) - updateLiveTable(event, evtdets['dtval']) + record = event['Records'][0] + fname = record['s3']['object']['key'] + _, barefname = os.path.split(fname) + if ('M' in barefname and 'P.jpg' in barefname) or ('FF' in barefname and '.jpg' in barefname): + xmlname = fname.replace('P.jpg','.xml').replace('.jpg', '.xml') + evtdets, camdets = processXml(xmlname) + if evtdets is not None: + storeInDDb(evtdets, camdets) + updateLiveTable(evtdets, camdets, barefname) diff --git a/archive/terraform/ukmda/secgrp.tf b/archive/terraform/ukmda/secgrp.tf index 1ae17dd00..a6aa5e1d4 100644 --- a/archive/terraform/ukmda/secgrp.tf +++ b/archive/terraform/ukmda/secgrp.tf @@ -46,8 +46,8 @@ resource "aws_security_group" "ec2_secgrp" { description = "Security Group for EC2 instances" vpc_id = aws_vpc.ec2_vpc.id ingress = [ -/* { - cidr_blocks = [data.aws_vpc.mjmm_ec2_vpc.cidr_block] + { + cidr_blocks = [aws_vpc.ec2_vpc.cidr_block] description = "SSH for Admin" from_port = 22 protocol = "tcp" @@ -56,9 +56,9 @@ resource "aws_security_group" "ec2_secgrp" { prefix_list_ids = [] security_groups = [] self = false - },*/ + }, { - cidr_blocks = [aws_vpc.ec2_vpc.cidr_block] + cidr_blocks = ["0.0.0.0/0"] description = "SSH for Admin" from_port = 22 protocol = "tcp" @@ -69,16 +69,38 @@ resource "aws_security_group" "ec2_secgrp" { self = false }, { - cidr_blocks = ["0.0.0.0/0"] - description = "SSH for Admin" - from_port = 22 + cidr_blocks = ["194.0.0.0/8"] + description = "MariaDB" + from_port = 3306 protocol = "tcp" - to_port = 22 + to_port = 3306 ipv6_cidr_blocks = [] prefix_list_ids = [] security_groups = [] self = false - } + }, + { + cidr_blocks = ["0.0.0.0/0"] + description = "icmp" + from_port = -1 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "icmp" + security_groups = [] + self = false + to_port = -1 + }, + { + cidr_blocks = ["0.0.0.0/0"] + description = "mariadb" + from_port = 3306 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "tcp" + security_groups = [] + self = false + to_port = 3306 + }, ] egress = [ { diff --git a/archive/terraform/ukmda/ssm_variables.tf b/archive/terraform/ukmda/ssm_variables.tf index 29fba5ac6..5661b5078 100644 --- a/archive/terraform/ukmda/ssm_variables.tf +++ b/archive/terraform/ukmda/ssm_variables.tf @@ -1,14 +1,14 @@ # Copyright (C) 2018-2023 Mark McIntyre -# SSM parameters for use in Lambdas +# SSM parameters for use in Lambdas and the batch resource "aws_ssm_parameter" "prod_dbhost" { provider = aws.eu-west-1-prov name = "prod_dbhost" type = "String" - value = "3.11.55.160" + value = "batchserver.ukmeteors.co.uk" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } @@ -18,7 +18,7 @@ resource "aws_ssm_parameter" "prod_dbname" { type = "String" value = "ukmon" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } @@ -28,7 +28,17 @@ resource "aws_ssm_parameter" "prod_dbpw" { type = "SecureString" value = "Batch33mdl" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_rootdbpw" { + provider = aws.eu-west-1-prov + name = "prod_rootdbpw" + type = "SecureString" + value = "Wombat33mdb" + tags = { + "billingtag" = "ukmda" } } @@ -38,6 +48,143 @@ resource "aws_ssm_parameter" "prod_dbuser" { type = "String" value = "batch" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_websitebucket" { + name = "prod_websitebucket" + type = "String" + value = var.websitebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_sharedbucket" { + name = "prod_sharedbucket" + type = "String" + value = var.sharedbucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_livebucket" { + name = "prod_livebucket" + type = "String" + value = var.livebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_siteurl" { + name = "prod_siteurl" + type = "String" + value = "https://archive.ukmeteors.co.uk" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_envname" { + name = "prod_envname" + type = "String" + value = "PROD" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_calcinstance" { + name = "prod_calcinstance" + type = "String" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_calcuser" { + name = "prod_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmda" } } + + +resource "aws_ssm_parameter" "prod_calcserverip" { + name = "prod_calcserverip" + type = "String" + value = var.ubuntu_calcserverip + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_wmplhome" { + name = "prod_wmplhome" + type = "String" + value = "$HOME/src/WesternMeteorPyLib" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_rmshome" { + name = "prod_rmshome" + type = "String" + value = "$HOME/src/RMS" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_srcdir" { + name = "prod_srcdir" + type = "String" + value = "$HOME/prod" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_caminfo" { + name = "prod_caminfo" + type = "String" + value = "$HOME/prod/data/consolidated/camera-details.csv" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_sshkey" { + name = "prod_sshkey" + type = "String" + value = "$HOME/.ssh/markskey.pem" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_batchloggroup" { + name = "prod_batchloggroup" + type = "String" + value = "/ukmonbatch/nightlyjob" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_gmapsapikey" { + name = "prod_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmda" + } +} + diff --git a/archive/terraform/ukmda/ukmdalive_bucket.tf b/archive/terraform/ukmda/ukmdalive_bucket.tf index 7e1c405bb..45ea7d50c 100644 --- a/archive/terraform/ukmda/ukmdalive_bucket.tf +++ b/archive/terraform/ukmda/ukmdalive_bucket.tf @@ -9,58 +9,6 @@ resource "aws_s3_bucket" "ukmdalive" { } provider = aws.eu-west-1-prov } -/* -resource "aws_s3_bucket_policy" "ukmdalivebp" { - bucket = aws_s3_bucket.ukmdalive.id - provider = aws.eu-west-1-prov - policy = jsonencode( - { - Id = "ukmda-live-bp" - Statement = [ - { - "Sid": "DataSyncCreateS3LocationAndTaskAccess", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::${var.eeaccountid}:role/DataSyncBetweenAccounts" - }, - "Action": [ - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:AbortMultipartUpload", - "s3:DeleteObject", - "s3:GetObject", - "s3:ListMultipartUploadParts", - "s3:PutObject", - "s3:GetObjectTagging", - "s3:PutObjectTagging" - ], - "Resource": [ - "${aws_s3_bucket.ukmdalive.arn}", - "${aws_s3_bucket.ukmdalive.arn}/*" - ] - }, - { - "Sid": "replicatelive", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::${var.eeaccountid}:role/service-role/replicatelive" - }, - "Action": [ - "s3:Put*", - "s3:Get*" - ], - "Resource": [ - "${aws_s3_bucket.ukmdalive.arn}", - "${aws_s3_bucket.ukmdalive.arn}/*" - ] - } - ] - Version = "2008-10-17" - } - ) -} -*/ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalivelcp" { bucket = aws_s3_bucket.ukmdalive.id diff --git a/archive/terraform/ukmda/variables.tf b/archive/terraform/ukmda/variables.tf index e73774644..01d6aef6b 100644 --- a/archive/terraform/ukmda/variables.tf +++ b/archive/terraform/ukmda/variables.tf @@ -8,6 +8,10 @@ variable "sharedbucket" { default = "ukmda-shared" } variable "livebucket" { default = "ukmda-live" } variable "adminbucket" { default = "ukmda-admin" } +variable "dev_websitebucket" { default = "ukmda-website" } +variable "dev_sharedbucket" { default = "ukmda-shared" } +variable "dev_livebucket" { default = "ukmda-live" } + variable "region" { default = "eu-west-2"} variable "liveregion" { default = "eu-west-1"} @@ -18,14 +22,13 @@ data "aws_canonical_user_id" "current" {} variable "remote_profile" { default = "default"} variable "remote_account_id" { default = "317976261112" } variable "remote_region" {default = "eu-west-2"} -#variable "eeaccountid" {default = "822069317839" } variable "main_cidr" {default = "172.32.0.0/16" } variable mgmt_cidr { default = "172.32.36.0/22" } variable lambda_cidr { default = "172.32.32.0/22" } variable ec2_cidr { default = "172.32.16.0/20" } -variable calcserverip { default = "172.32.16.136" } variable ubuntu_calcserverip { default = "172.32.16.137" } +variable batchserverip { default = "172.32.16.138" } variable archalias { default = "archive.ukmeteors.co.uk"} diff --git a/archive/ukmon_pylib/requirements.txt b/archive/ukmon_pylib/additional_requirements.txt similarity index 66% rename from archive/ukmon_pylib/requirements.txt rename to archive/ukmon_pylib/additional_requirements.txt index aeaba31b9..c5942af22 100644 --- a/archive/ukmon_pylib/requirements.txt +++ b/archive/ukmon_pylib/additional_requirements.txt @@ -1,51 +1,37 @@ # Copyright (C) 2018-2023 Mark McIntyre #cartopy INSTALL THIS WITH CONDA -# skip this -#https://github.com/matplotlib/basemap/archive/master.zip -basemap basemap-data-hires boto3 -Cython gmplot google-auth google-auth-oauthlib google-api-python-client googleapis-common-protos -jplephem -Keras -matplotlib==3.3.2 -numpy oauthlib -pandas Pillow pycparser pyephem -pyproj==2.6.1.post1 -PyQt5 +pyproj pyswarms -pytz -PyYAML -scipy Shapely xmltodict -tweepy fastparquet cryptography python-crontab simplekml astropy -gitpython pymysql mysql-connector-python pre-commit s3fs -# pyarrow +dynamodb_json +requests +paho-mqtt +scp +# testing requirements pytest behave pdoc -dynamodb_json pytest pytest-cov -requests -paho-mqtt \ No newline at end of file diff --git a/archive/ukmon_pylib/analysis/README.md b/archive/ukmon_pylib/analysis/README.md index 732ae6a45..a4b09f7d4 100644 --- a/archive/ukmon_pylib/analysis/README.md +++ b/archive/ukmon_pylib/analysis/README.md @@ -1,9 +1,8 @@ # Analysis -This folder contains functions to analyse the single station and matched data. +This folder contains functions to analyse the single station and matched data for website reporting. * ShowerAnalysis.py - analysis for a named shower for a specific year eg GEM 2022 * stationAnalysis.py - analysis for a named year or year+month, for one or all stations. -* meteorFlux - plot a graph of meteor flux -* gatherDetectionData.py -collects all available data on a detection or detections matching a pattern YYYYMMDD_HHMMSS. If the seconds are omitted, then all detections within the specified minute will be returned. - +* summaryAnalysis.py - summary analysis for a given period + diff --git a/archive/ukmon_pylib/analysis/analyseGmnData.py b/archive/ukmon_pylib/analysis/analyseGmnData.py deleted file mode 100644 index 77920b99c..000000000 --- a/archive/ukmon_pylib/analysis/analyseGmnData.py +++ /dev/null @@ -1,20 +0,0 @@ -# simple scripts to analyse the GMN data in python - -import pandas as pd -import os - -from converters.gmnTxtToPandas import dirpath - - -def findDuplicates(yr, mth=None): - if mth: - datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') - else: - datafile = os.path.join(dirpath, 'parquet', f'gmn_{yr:04d}.parquet.snap') - df = pd.read_parquet(datafile) - df['dupe']=df.duplicated(subset=['id']) - dupeids = df[df.dupe].sort_values(by=['id']).id - duperows = df[df.id.isin(dupeids)] - print(duperows) - print(len(df)) - return duperows diff --git a/archive/ukmon_pylib/analysis/gatherDetectionData.py b/archive/ukmon_pylib/analysis/gatherDetectionData.py deleted file mode 100644 index b9031167b..000000000 --- a/archive/ukmon_pylib/analysis/gatherDetectionData.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Collect data for a potential match at a specific time -# -# Copyright (C) 2018-2023 Mark McIntyre -# -import os -import sys -import datetime -import pandas as pd - -from meteortools.ukmondb import getECSVs, getLiveJpgs, getFBfiles, getDetections - - -def getUncalibratedImageList(dtstr=None): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') - logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') - flines = open(logfile, 'r').readlines() - uncal = [f for f in flines if 'not recalibrated' in f] - imglist = [x.strip(' ').split(' ')[1][:-1] for x in uncal] - with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: - for li in imglist: - outf.write(f'{li}\n') - return imglist - - -def getRawData(idlist, outpth): - for _,row in idlist.iterrows(): - stat =row.ID - dts = datetime.datetime.strptime(row.Dtstamp, '%Y%m%d_%H%M%S') - dt = dts.strftime('%Y-%m-%dT%H:%M:%S') - fboutdir = os.path.join(outpth, stat) - dtstr = row.Filename[10:25] - getLiveJpgs(dtstr, outpth, False) - getFBfiles(f'{stat}_{dtstr}', fboutdir) - getECSVs(stat, dt, True, fboutdir) - return - - -def updateSingleDB(yr, consdt): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') - sngls = pd.read_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv')) - consdf = pd.read_csv(os.path.join(datadir, 'single', 'used', f'consumed_{consdt}.txt'), names=['Filename']) - conslist = list(consdf.Filename) - for ff in conslist: - mtch = sngls.Filename == ff - if mtch.any(): - print(f'updating {ff}') - sngls.loc[mtch, 'status'] = 'Y' - else: - print(f'skipping {ff}') - sngls.to_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv'), index=False) - - -if __name__ == '__main__': - outpth = f'./{sys.argv[1]}' - os.makedirs(outpth, exist_ok=True) - idlist = getDetections(sys.argv[1]) - if not isinstance(idlist, bool): - getRawData(idlist, outpth) - ids = list(idlist.ID) - with open(os.path.join(outpth, 'ids.txt'), 'w') as outf: - outf.writelines(line + '\n' for line in ids) - print(ids) - else: - print('no matches') diff --git a/archive/ukmon_pylib/analysis/getUsedUnused.py b/archive/ukmon_pylib/analysis/getUsedUnused.py new file mode 100644 index 000000000..0541ecad6 --- /dev/null +++ b/archive/ukmon_pylib/analysis/getUsedUnused.py @@ -0,0 +1,65 @@ +# +# Python code to analyse meteor shower data +# +# Copyright (C) 2018-2023 Mark McIntyre + +import os +import boto3 +import json +import pymysql.cursors + + +def getSqlLoginDetails(): + # retrieve password and host from SSM. This allows me to manage them from Terraform + ssm = boto3.client('ssm', region_name='eu-west-1') + res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) + password = res['Parameter']['Value'] + res = ssm.get_parameter(Name='prod_dbhost') + host = res['Parameter']['Value'] + # should really do these too but they won't change often if at all + user = 'batch' + db = 'ukmon' + return host, user, password, db + + +def getAllImages(dtstr): + """ + Retrieve a list of all images for a given date from the SQL database + This is a list of all images containing detections, whether or not used + in a solution. + """ + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + sqlstr = f'select filname from singles where Y={dtstr[:4]} and M={int(dtstr[4:6])} and D={int(dtstr[6:8])}' + with connection.cursor() as cursor: + cursor.execute(sqlstr) + result = cursor.fetchall() + connection.close() + res=[] + for event in result: + res.append(event['filname']) + return res + + +def getKnownImages(dtstr, datatype='consumed'): + """ + Get a list of all images that were involved in a match or were uncalibrated, + depending on the value of datatype (consumed or uncal) + The raw data are generated by a process on the matching engine server + """ + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + srcfile = os.path.join(datadir,'single','used',f'{datatype}_{dtstr}.txt') + imglist = [] + if os.path.isfile(srcfile): + used = open(srcfile,'r').readlines() + for img in used: + imglist.append(img.strip()) + return imglist + + +def getUnusedImages(dtstr): + + allimages = getAllImages(dtstr) + uncal = getKnownImages(dtstr, 'uncal') + used = getKnownImages(dtstr, 'consumed') + unused = list(set(allimages)-set(used + uncal)) \ No newline at end of file diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index 29636148f..7defd5930 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -10,8 +10,8 @@ from matplotlib import pyplot as plt import datetime -from meteortools.utils import getShowerDates as sd -from meteortools.fileformats import imoWorkingShowerList as imo +from utils.getActiveShowers import getShowerDets +from utils.imoWorkingShowerList import IMOshowerList SMALL_SIZE = 8 MEDIUM_SIZE = 10 @@ -97,8 +97,8 @@ def timeGraph(dta, shwrname, outdir, binmins=10): countcol = dta['Shwr'] # resample it - binned = countcol.resample('{}min'.format(binmins)).count() - binned.plot(kind='bar') + binned = countcol.resample(f'{binmins}min').count() + binned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals nbins = max(len(binned)/144, 2) @@ -138,8 +138,8 @@ def matchesGraphs(dta, shwrname, outdir, binmins=60, startdt=None, enddt=None, t #select just the shower ID col mcountcol = mdta['_ID1'] # resample it - mbinned = mcountcol.resample('{}min'.format(binmins)).count() - mbinned.plot(kind='bar') + mbinned = mcountcol.resample(f'{binmins}min').count() + mbinned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals ax = plt.gca() @@ -216,7 +216,7 @@ def velDistribution(dta, shwrname, outdir, vg_or_vs, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -254,7 +254,7 @@ def durationDistribution(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.1f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -292,7 +292,7 @@ def distanceDistribution(dta, shwrname, outdir, binwidth=1.0): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -356,13 +356,6 @@ def radiantDistribution(dta, shwrname, outdir): magdf.plot.scatter(x=idx, y=idx2) ax = plt.gca() ax.set(xlabel='RA (deg)', ylabel="Dec (deg)") - #plt.locator_params(axis='x', nbins=12) - #for lab in ax.get_xticklabels(): - # lab.set_fontsize(SMALL_SIZE) - - # format x-axes - #x_labels=["%.0f" % number for number in bins[:-1]] - #ax.set_xticklabels(x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -394,7 +387,7 @@ def semimajorDistribution(dta, shwrname, outdir, binwidth=0.5): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -424,7 +417,7 @@ def magDistributionAbs(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -454,7 +447,7 @@ def magDistributionVis(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -467,7 +460,7 @@ def magDistributionVis(dta, shwrname, outdir, binwidth=0.2): def showerAnalysis(shwr, dtstr): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) # set up paths, files etc # check if month was passed in @@ -486,7 +479,7 @@ def showerAnalysis(shwr, dtstr): cols = ['Shwr','Dtstamp','Y','M','ID','Mag'] filt = None if shwr != 'ALL': - sl = imo.IMOshowerList() + sl = IMOshowerList() maxdt = sl.getEnd(shwr) + datetime.timedelta(days=10) mindt = sl.getStart(shwr) + datetime.timedelta(days=-10) @@ -498,7 +491,7 @@ def showerAnalysis(shwr, dtstr): # select the required data if shwr != 'ALL': - id, shwrname, sl, dt = sd.getShowerDets(shwr) + id, shwrname, sl, dt = getShowerDets(shwr) sngl = sngl[sngl['Shwr']==shwr] sngl = sngl[sngl.Dtstamp > mindt.timestamp()] sngl = sngl[sngl.Dtstamp < maxdt.timestamp()] @@ -531,7 +524,7 @@ def showerAnalysis(shwr, dtstr): # select the required data if shwr != 'ALL': - id, shwrname, sl, dt = sd.getShowerDets(shwr) + id, shwrname, sl, dt = getShowerDets(shwr) mtch = mtch[mtch['_stream']==shwr] mtch = mtch[mtch.dtstamp > mindt.timestamp()] mtch = mtch[mtch.dtstamp < maxdt.timestamp()] diff --git a/archive/ukmon_pylib/analysis/stationAnalysis.py b/archive/ukmon_pylib/analysis/stationAnalysis.py index 519bf4790..7e888ae47 100644 --- a/archive/ukmon_pylib/analysis/stationAnalysis.py +++ b/archive/ukmon_pylib/analysis/stationAnalysis.py @@ -77,7 +77,7 @@ def timeGraph(dta, loc, outdir, when, sampleinterval): #select just the shower ID col countcol = dta['ID'] # resample it - binned = countcol.resample('{}'.format(sampleinterval)).count() + binned = countcol.resample(f'{sampleinterval}').count() binned.plot(kind='bar') # set ticks and labels every 144 intervals @@ -100,7 +100,7 @@ def timeGraph(dta, loc, outdir, when, sampleinterval): pass fname = os.path.join(outdir, 'station_plot_timeline_single.jpg') - plt.title('Observed stream activity {} intervals for {} in {}'.format(sampleinterval, loc, when)) + plt.title(f'Observed stream activity {sampleinterval} intervals for {loc} in {when}') try: plt.tight_layout() plt.savefig(fname) @@ -140,7 +140,7 @@ def reportOneSite(yr, mth, loc, sngl, mful, idlist, outdir): when = f'{mth:02d}-{yr}' idxfile = os.path.join(outdir,'index.html') - templatedir=os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templatedir=os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) shutil.copyfile(os.path.join(templatedir, 'header.html'), idxfile) outf = open(idxfile, 'a+') @@ -291,7 +291,7 @@ def pushToWebsite(fuloutdir, outdir, websitebucket): matchcols = ['_Y_ut','_M_ut','_mag','_mjd','mjd','_stream','orbname','stations'] snglcols = ['ID','Shwr','Dtstamp','Ver', 'M', 'Y'] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camlist = loadLocationDetails() sngl = pd.read_parquet(os.path.join(datadir, 'single', f'singles-{yr}.parquet.snap'), columns=snglcols) @@ -311,7 +311,7 @@ def pushToWebsite(fuloutdir, outdir, websitebucket): idlist = list(camlistfltr.stationid) if mth is None: - sampleinterval="1M" + sampleinterval="1ME" shortoutdir = os.path.join('reports', str(yr), 'stations', loc) outdir = os.path.join(datadir, shortoutdir) else: diff --git a/archive/ukmon_pylib/analysis/summaryAnalysis.py b/archive/ukmon_pylib/analysis/summaryAnalysis.py index 1aa6117a4..5df930768 100644 --- a/archive/ukmon_pylib/analysis/summaryAnalysis.py +++ b/archive/ukmon_pylib/analysis/summaryAnalysis.py @@ -13,7 +13,7 @@ def showerSummaryByPeriod(dtstr): yr = int(dtstr[:4]) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) cols = ['_stream','_M_ut'] filt = None matchfile = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) @@ -34,7 +34,7 @@ def showerSummaryByPeriod(dtstr): def createSummWebpage(dtstr, outdir=None): yr = dtstr[:4] mth = None - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if len(dtstr) > 4: mth = dtstr[4:6] if outdir is None: @@ -43,7 +43,7 @@ def createSummWebpage(dtstr, outdir=None): if outdir is None: outdir = os.path.join(datadir, 'reports', yr, 'showers') os.makedirs(outdir, exist_ok=True) - templatedir=os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templatedir=os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) idxfile = os.path.join(outdir, 'index.html') shutil.copyfile(os.path.join(templatedir, 'header.html'), idxfile) with open(idxfile, 'a+') as outf: @@ -73,7 +73,7 @@ def createSummWebpage(dtstr, outdir=None): def createSummJS(dtstr, shwrdata, maxlines=None): yr = dtstr[:4] mth = None - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if len(dtstr) > 4: mth = dtstr[4:6] outdir = os.path.join(datadir, 'reports', yr, 'showers', mth) diff --git a/archive/ukmon_pylib/converters/gmnTxtToPandas.py b/archive/ukmon_pylib/converters/gmnTxtToPandas.py index ad98886f9..4e759b6b9 100644 --- a/archive/ukmon_pylib/converters/gmnTxtToPandas.py +++ b/archive/ukmon_pylib/converters/gmnTxtToPandas.py @@ -17,7 +17,7 @@ 'Lat1','Lat1sd','Lon1','Lon1sd','H1','H1sd','Lat2','Lat2sd','Lon2','Lon2sd','H2','H2sd', 'Dur','Amag','PkHt','F1','mass','Qc','MedianFitErr','BegIn','EndIn','NumStat','stats'] -datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') +datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) dirpath = os.path.join(datadir, 'gmndata') @@ -37,7 +37,7 @@ def loadOneFile(fname): df = df.fillna(value=np.nan) df['jd_beg'] = df.jd_beg.astype(float) df['utc_beg'] = pd.to_datetime(df.utc_beg) - df['ian_uo'] = df.iau_no.astype(int) + df['iau_no'] = df.iau_no.astype(int) for c in range(5,82): try: df[colhdrs[c]] = df[colhdrs[c]].astype(float) @@ -138,4 +138,10 @@ def getStats(): if __name__ == '__main__': - doYear(sys.argv[1]) + if sys.argv[1] == 'doYear': + doYear(sys.argv[2]) + if sys.argv[1] == 'convert': + df = loadOneFile(sys.argv[2]) + fn, _ = os.path.splitext(sys.argv[2]) + df.to_parquet(f'{fn}.parquet.snap', index=False) + diff --git a/archive/ukmon_pylib/converters/toParquet.py b/archive/ukmon_pylib/converters/toParquet.py index c9454b7b8..a147501ce 100644 --- a/archive/ukmon_pylib/converters/toParquet.py +++ b/archive/ukmon_pylib/converters/toParquet.py @@ -15,7 +15,7 @@ if 'match' in fn: # fill in any #NAs in the mjd column - df.mjd.fillna(df._mjd, inplace=True) + df.fillna({'mjd': df._mjd}, inplace=True) if 'single' in fn: df = df.drop_duplicates() diff --git a/archive/ukmon_pylib/maintenance/cleanDdbTables.py b/archive/ukmon_pylib/maintenance/cleanDdbTables.py index 464d2be7f..26e17f80d 100644 --- a/archive/ukmon_pylib/maintenance/cleanDdbTables.py +++ b/archive/ukmon_pylib/maintenance/cleanDdbTables.py @@ -9,9 +9,7 @@ def deleteRows(): - archprof = os.getenv('ADM_PROFILE', default='ukmda_admin') - conn = boto3.Session(profile_name=archprof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') tbl = 'uploadtimes' idx = 'uploaddate-stationid-index' table = ddb.Table(tbl) @@ -28,9 +26,7 @@ def deleteRows(): def updateMissingExpiryDate(): - archprof = os.getenv('ADM_PROFILE', default='ukmda_admin') - conn = boto3.Session(profile_name=archprof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') tbl = 'uploadtimes' idx = 'uploaddate-stationid-index' table = ddb.Table(tbl) diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index 96d24dab9..32418b256 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -9,6 +9,13 @@ import paramiko from scp import SCPClient from time import sleep +import pandas as pd +import datetime +import json +import operator + +from wmpl.Utils.TrajConversions import datetime2JD +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase def findInputDataByMonth(yyyymm, archbucket, outdir): @@ -48,18 +55,11 @@ def deleteS3FilesByMonth(flist, archbucket): def deleteFromCalcServerByMonth(outfname): - hname = os.getenv('HOSTNAME', default='none') env = os.getenv('RUNTIME_ENV', default='DEV').lower() - if hname != 'ukmonhelper2': - sess = boto3.Session(profile_name='default') - ssmc = sess.client('ssm', region_name='eu-west-2') - ec2 = boto3.client('ec2', region_name='eu-west-2') - else: - ssmc = boto3.client('ssm', region_name='eu-west-2') - prof = os.getenv('UKMPROFILE',default='ukmonshared') - sess = boto3.Session(profile_name=prof) - ec2 = sess.client('ec2', region_name='eu-west-2') + ssmc = boto3.client('ssm', region_name='eu-west-2') + ec2 = boto3.client('ec2', region_name='eu-west-2') + resp = ssmc.get_parameter(Name=f'{env}_calcinstance') instId = resp['Parameter']['Value'] print('clearing down calcserver') @@ -97,6 +97,126 @@ def deleteFromCalcServerByMonth(outfname): return +def removeDeletedTraj(csvfile): + """ + Remove deleted trajectories from the consolidated match CSV, Parquet and search files + """ + + csvdata = open(csvfile, 'r').readlines() + if 'search' in csvfile: + ts_end = float(csvdata[-1].split(',')[0]) + dt_end = datetime.datetime.fromtimestamp(ts_end, tz=datetime.timezone.utc) + jdt_end = datetime2JD(dt_end) + else: + jdt_end = float(csvdata[-1].split(',')[3]) + 2400000.5 + jdt_beg = jdt_end - 21 + + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + + i=0 + # loop over the deleted trajectories, removing the corresponding one from the CSV file. + # Note that there's no need to update the MariaDB database, as it is populated + # from the CSV file *after* it has been purged + offs = 4 if 'search' in csvfile else 131 + for traj in deltrajs: + cur = masterdb.dbhandle.execute(f'select jdt_ref, participating_stations from trajectories where traj_file_path="{traj[0]}" and status=0') + thistraj = cur.fetchall() + if len(thistraj) == 1: + fldr = os.path.basename(os.path.dirname(traj[0])) + + # find the rows in the CSV file that correspond to the deleted trajectory + # then go through them and compare obs_ids. The one with the same obs_ids + # is the one we want to remove + match = [tr for tr in csvdata if fldr in tr] + obs_ids = thistraj[0][1] + obs_ids_str = ';'.join(json.loads(obs_ids)) + ';' + for thismtch in match: + if thismtch.split(',')[offs] == obs_ids_str: + print(f'removing {fldr}') + idx = csvdata.index(thismtch) + _ = csvdata.pop(idx) + i += 1 + break + else: + print(f'skipping {traj[0]} as there are {len(thistraj)} active trajs with the same path') + masterdb.closeTrajDatabase() + print(f'removed {i} trajectories from the text file') + + # save the CSV file again + open(csvfile, 'w').writelines(csvdata) + + if 'search' not in csvfile: + df = pd.read_csv(csvfile, skipinitialspace=True) + df = df.drop_duplicates(subset=['_mjd','_sol','_ID1','_ra_o','_dc_o','_amag','_ra_t','_dc_t']) + df.to_csv(csvfile, index=False) + + return + + +def removeDeletedTrajCsv(csvloc, targloc): + """ + Remove csv files corresponding to deleted trajectories before they are + consolidated into the annual CSV and Parquet files + """ + srcbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] + + csvfiles = [] + s3 = boto3.client('s3') + print('checking in ', csvloc, 'sending to ', targloc) + res = s3.list_objects(Bucket=srcbucket, Prefix=csvloc) + if 'Contents' in res: + for k in res['Contents']: + csvname = k["Key"] + csv_dt = datetime.datetime.strptime(os.path.basename(k["Key"])[:22], "%Y%m%d-%H%M%S.%f") + csvfiles.append({"name":csvname, "dt":csv_dt}) + csvfiles.sort(key=operator.itemgetter('name')) + jdt_beg = datetime2JD(csvfiles[0]['dt']) + jdt_end = datetime2JD(csvfiles[-1]['dt']) + + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + + i=0 + # loop over the deleted trajectories, removing the corresponding CSV file. + for del_traj in deltrajs: + cur = masterdb.dbhandle.execute(f'select jdt_ref, participating_stations from trajectories where traj_file_path="{del_traj[0]}" and status=0') + thistraj = cur.fetchall() + if len(thistraj) == 1: + obs_ids = thistraj[0][1] + obs_ids_str = ';'.join(json.loads(obs_ids)) + ';' + fldr = os.path.basename(os.path.dirname(del_traj[0]))[:19].replace('_', '-') + + matched_csvs = [x for x in csvfiles if fldr in x['name']] + for csv in matched_csvs: + csvdata = s3.get_object(Bucket=srcbucket, Key=csv['name'])['Body'].read() + csv_obs_ids = csvdata.decode('utf-8').split(',')[131].strip() + if csv_obs_ids == obs_ids_str: + # we have a match to within 0.001s and with the same observations + print(f'removing {fldr}') + bare_csv_name = os.path.basename(matched_csvs[0]['name']) + yr = bare_csv_name[:4] + destkey = f"{targloc}/{yr}/{bare_csv_name}" + s3.copy({"Bucket":srcbucket,"Key":csv['name']}, srcbucket, destkey) + s3.delete_object(Bucket=srcbucket, Key=matched_csvs[0]['name']) + i += 1 + break + else: + print(f'skipping {del_traj[0]} as there are {len(thistraj)} inactive trajs with the same path') + masterdb.closeTrajDatabase() + print(f'removed {i} trajectories from the text file') + else: + print('no fullcsv files to process') + + return + + if __name__ == '__main__': arg_parser = argparse.ArgumentParser(description="Find and clear down historical raw data.") diff --git a/archive/ukmon_pylib/maintenance/getNextBatchStart.py b/archive/ukmon_pylib/maintenance/getNextBatchStart.py index 2dc8f39a6..d89b40304 100644 --- a/archive/ukmon_pylib/maintenance/getNextBatchStart.py +++ b/archive/ukmon_pylib/maintenance/getNextBatchStart.py @@ -6,7 +6,7 @@ import datetime import sys from crontab import CronTab -from meteortools.utils import getNextRiseSet +from utils.getRiseSet import getNextRiseSet offset = 90 diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py index f1b477283..85b3f04bc 100644 --- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py +++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py @@ -25,8 +25,8 @@ csvdir = 'f:/videos/meteorcam/usermgmt/csvkeys' lnxdir='/home/ec2-user/keymgmt/csvkeys' else: - csvdir='/home/ec2-user/keymgmt/csvkeys' - lnxdir='/home/ec2-user/keymgmt/csvkeys' + csvdir=os.path.expanduser('~/keymgmt/csvkeys') + lnxdir=os.path.expanduser('~/keymgmt/csvkeys') def checkIfOnGMN(camid): diff --git a/archive/ukmon_pylib/maintenance/manageTraj.py b/archive/ukmon_pylib/maintenance/manageTraj.py index fae24d735..0a684babd 100644 --- a/archive/ukmon_pylib/maintenance/manageTraj.py +++ b/archive/ukmon_pylib/maintenance/manageTraj.py @@ -8,7 +8,7 @@ # delete an orbit from the database def deleteDuplicate(trajname, jd=None): - datadir = os.getenv('DATADIR', default='/home/pi/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) yr=trajname[:4] if int(yr) > 2021: fname = os.path.join(datadir, 'matched','matches-full-{}.parquet.snap'.format(yr)) diff --git a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py index 28d7b9c02..e90cca86c 100644 --- a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py +++ b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py @@ -6,14 +6,11 @@ import numpy as np import os import sys -import fnmatch import matplotlib.pyplot as plt import json import cartopy.crs as ccrs -from meteortools.fileformats import UFOAnalyzerXML as ua - from PIL import Image @@ -21,15 +18,7 @@ def getBearingsForEvent(stns, fldr): brngs = np.zeros(len(stns)) listOfFiles = os.listdir(fldr) for entry in listOfFiles: - if fnmatch.fnmatch(entry, '*A.XML'): - fullname = os.path.join(fldr, entry) - xmlf = ua.UAXml(fullname) - sta, _, _, _, _, _ = xmlf.getStationDetails() - _, _, _, _, _, _, az1, _ = xmlf.getObjectStart(0) - i = stns.index(sta) - brngs[i] = az1 - - elif entry == 'FTPdetectinfo_UFO.txt': + if entry == 'FTPdetectinfo_UFO.txt': with open(os.path.join(fldr, 'FTPdetectinfo_UFO.txt'), 'r') as f: lis = f.readlines() for s in stns: @@ -107,7 +96,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): # ax.add_feature(cfeature.LAKES, edgecolor='black') # ax.add_feature(cfeature.RIVERS) ax.gridlines() - pylib=os.getenv('PYLIB', default='/home/ec2-user/prod/ukmon_pylib') + pylib=os.getenv('PYLIB', default=os.path.expanduser('~/prod/ukmon_pylib')) os.environ['CARTOPY_USER_BACKGROUNDS'] = os.path.join(pylib, 'share','maps') ax.background_img(name='BM', resolution='high', extent= [minn, maxn, mina, maxa]) @@ -143,7 +132,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): plt.plot(yi, xi, 'bx', markersize=10, linewidth=1) plt.tight_layout() - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if datadir is None: datadir='.' plt.savefig(os.path.join(datadir, 'stations.png'), dpi=200) @@ -158,7 +147,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): plotlabels = False else: plotlabels = True - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) srcfile = os.path.join(datadir, 'admin', 'cameraLocs.json') intersect = False if len(sys.argv) > 2: diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py index 4f5f050cf..c64c746c5 100644 --- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py +++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py @@ -71,8 +71,7 @@ def createExtraJpgtxt(outdir, traj, availableimages): def createExtraJpgHtml(outdir, parentfldr, yr, ym): - mdasess = boto3.Session(profile_name='ukmda_admin') - s3mda = mdasess.client('s3') + s3mda = boto3.client('s3') webfldr = f'img/single/{yr}/{ym}' with open(os.path.join(outdir, 'extrajpgs.html'), 'w') as outf: jpgfldr = os.path.join(parentfldr, 'jpgs') @@ -194,8 +193,7 @@ def recreateOrbitFiles(outdir, pickname, doupload=False): if doupload: files = os.listdir(outdir) - mdasess = boto3.Session(profile_name='ukmda_admin') - s3mda = mdasess.client('s3') + s3mda = boto3.client('s3') if int(yr) > 2020: webfldr = f'reports/{yr}/orbits/{ym}/{ymd}' else: diff --git a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py index feb2a5079..3ef031d57 100644 --- a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py +++ b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py @@ -14,7 +14,7 @@ def findOtherBadEvents(): evts = [] - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'dailyreports', 'latest.txt') with open(lastf, 'r') as inf: lis = inf.readlines() @@ -30,10 +30,9 @@ def findOtherBadEvents(): return badevents -def findFailedEvents(prof='ukmonshared'): - #session = boto3.Session(profile_name=prof) +def findFailedEvents(): logcli = boto3.client('logs', region_name='eu-west-2') - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') if os.path.isfile(lastf): with open(lastf, 'r') as inf: @@ -98,11 +97,10 @@ def checkFails(fails): return realfails -def rerunFails(fails, prof='ukmonshared'): - session=boto3.Session(profile_name=prof) - lambd = session.client('lambda', region_name='eu-west-2') +def rerunFails(fails): + lambd = boto3.client('lambda', region_name='eu-west-2') - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') with open(lastf, 'w') as outf: rundt = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') @@ -120,10 +118,9 @@ def rerunFails(fails, prof='ukmonshared'): return -def resubmitPicklesForDay(dtstr, prof='ukmonshared'): +def resubmitPicklesForDay(dtstr): s3 = boto3.client('s3') - session = boto3.Session(profile_name=prof) - lambd = session.client('lambda', region_name='eu-west-2') + lambd = boto3.client('lambda', region_name='eu-west-2') pref = f'matches/RMSCorrelate/trajectories/{dtstr[:4]}/{dtstr[:6]}/{dtstr}/' buck = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] res = s3.list_objects_v2(Bucket=buck, Prefix=pref) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 09c961dce..36aeaf985 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -71,7 +71,7 @@ def findRowCamTimings(stationid, uploaddate, ddb=None): def getDayCamTimings(uploaddate, ddb=None, outfile=None, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if not ddb: ddb = boto3.resource('dynamodb', region_name='eu-west-2') #, endpoint_url="http://thelinux:8000") table = ddb.Table('uploadtimes') @@ -136,9 +136,10 @@ def deleteRowCamTimings(stationid, dtstamp, ddb=None): def backPopulate(stationid): s3bucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - fldrs = glob.glob1(f'/home/ec2-user/ukmon-shared/matches/RMSCorrelate/{stationid}/', '*') + basepath = os.path.expanduser('~/prod/ukmon-shared/matches/RMSCorrelate') + fldrs = glob.glob('*', root_dir=os.path.join(basepath, stationid)) for fldr in fldrs: - s3objects = glob.glob1(f'/home/ec2-user/ukmon-shared/matches/RMSCorrelate/{stationid}/{fldr}/', 'FTPd*') + s3objects = glob.glob('FTPd*', root_dir=os.path.join(basepath, stationid. fldr)) if len(s3objects) > 0: s3obj = s3objects[0] fullobj = f'matches/RMSCorrelate/{stationid}/{fldr}/{s3obj}' @@ -147,23 +148,9 @@ def backPopulate(stationid): if __name__ == '__main__': - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) ddb = boto3.resource('dynamodb', region_name='eu-west-2') - if os.path.isfile('/sys/devices/virtual/dmi/id/board_asset_tag'): # crude check for EC2 - #print('asset tag file exists') - lis = open('/sys/devices/virtual/dmi/id/board_asset_tag').readlines() - if 'i-' in lis[0]: - sts_client = boto3.client('sts') - assumed_role_object=sts_client.assume_role( - RoleArn="arn:aws:iam::183798037734:role/service-role/S3FullAccess", - RoleSessionName="AssumeRoleSession1") - credentials=assumed_role_object['Credentials'] - - ddb = boto3.resource('dynamodb', region_name='eu-west-2', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) s,d,t,m,r = getDayCamTimings(sys.argv[1], ddb=ddb) newdata=pd.DataFrame(zip(s,d,t,m,r), columns=['stationid','upddate','uploadtime','manual','rundate']) @@ -186,49 +173,48 @@ def backPopulate(stationid): pd.options.mode.chained_assignment = 'warn' caminfo = camlist.drop(columns=['site','direction','oldcode','active','camtype','eMail', 'humanName']) - logindf = pd.read_csv(os.path.join(datadir, 'reports', 'lastlogins.txt'), names=['dateval','timeval','siteid'], skipinitialspace=True) - logindf['timeval'] = logindf.timeval.astype('str').str.pad(6,fillchar='0') - logindf.dateval.fillna('Jan-01',inplace=True) - logindf.timeval.fillna('00:00:00',inplace=True) - # handle case round yearend where the log may have prev year's details as well as current year - nowdt = datetime.datetime.now() - yrval = str(nowdt.year) + '-' - yrvalback = str(nowdt.year-1) + '-' - logindf['lastseen'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrval + logindf.dateval+'_'+logindf.timeval] - try: # will fail on 29th Feb in a leapyear, as previous year is not leap - logindf['lastseen2'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrvalback + logindf.dateval+'_'+logindf.timeval] - except Exception: - logindf['lastseen2'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrval + logindf.dateval+'_'+logindf.timeval] - logindf.loc[logindf.lastseen > nowdt, 'lastseen'] = logindf.lastseen2 - logindf = logindf.sort_values(by=['lastseen']) - logindf.drop_duplicates(subset=['siteid'], inplace=True, keep='last') - logindf.rename(columns={'siteid':'location'}, inplace=True) - logindf.drop(columns = ['lastseen2'], inplace=True) - - # create a merged dataframe with siteid and stationid - intdf = pd.merge(logindf,caminfo, on=['location'], how='outer') - - df = pd.merge(intdf, fulldf, on=['stationid']) - df.dateval.fillna('Jan-01',inplace=True) - df.timeval.fillna('00:00:00',inplace=True) - df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") - df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime - df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] - df = df.drop(columns=['timeval','stationid','manual','rundate', 'upddate','uploadtime', 'dateval']) - df['dateval']=[x.strftime('%b-%d') for x in df.lastupload] - df = df.sort_values(by=['lastupload']) - - outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') - zerodate = datetime.datetime(1970,1,1,0,0,0) - with open(outfile,'w') as outf: - outf.write('Last Upload, StationID, Last Login\n') - for _,rw in df.iterrows(): - dtval = rw.dateval - lastup = rw.lastupload.strftime('%H:%M:%S') - if pd.isnull(rw.lastseen): - lastseen = '> 1 month' - else: - lastseen = rw.lastseen.strftime('%b-%d %H:%M:%S') - if lastseen == 'Jan-01 00:00:00': - lastseen = '> 1 month' - outf.write(f'{dtval}, {lastup}, {rw.location:20s}, {lastseen}\n') + # process the last-login data from the SSHD log + lastlogs = open(os.path.join(datadir, 'reports', 'lastlogins.txt'),'r').readlines() + lodata = [] + now = datetime.datetime.now(tz=datetime.timezone.utc) + for li in lastlogs: + spls = li.split('ssh') + # could be ubuntu (auth.log) or amazon linux style log + dtstr = spls[0][spls[0].find(':')+1:][:19] + if ' ' in dtstr: + dtval = datetime.datetime.strptime(f'{now.year} {dtstr[:15]}', '%Y %b %d %H:%M:%S').replace(tzinfo=datetime.timezone.utc) + if dtval > now: + dtval = dtval.replace(year=now.year-1, tzinfo=datetime.timezone.utc) + targhost = 'ukmonhelper2' + else: + dtval = datetime.datetime.strptime(dtstr, '%Y-%m-%dT%H:%M:%S') + dtval = dtval.replace(tzinfo=datetime.timezone.utc) + targhost = 'batchserver' + location = spls[1].split(' for ')[1].split()[0] + lodata.append({'location':location, 'lastseen':dtval,'host': targhost}) + + + if len(lodata) > 0: + logindf = pd.DataFrame(lodata) + logindf = logindf.sort_values(by=['lastseen']) + logindf.drop_duplicates(subset=['location'], inplace=True, keep='last') + + # create a merged dataframe with siteid and stationid + intdf = pd.merge(logindf,caminfo, on=['location'], how='outer') + + df = pd.merge(intdf, fulldf, on=['stationid']) + df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") + df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime + df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] + df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) + df = df.sort_values(by=['lastupload']) + + outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') + zerodate = datetime.datetime(1970,1,1,0,0,0) + with open(outfile,'w') as outf: + outf.write('Last Upload, StationID, Last Login, Via\n') + for _,rw in df.iterrows(): + lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') + lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') + via = '' if pd.isnull(rw.host) else rw.host + outf.write(f'{lastup} , {rw.location:20s}, {lastlo}, {via}\n') diff --git a/archive/ukmon_pylib/metrics/costMetrics.py b/archive/ukmon_pylib/metrics/costMetrics.py index 36f83dd2b..9b0236fe6 100644 --- a/archive/ukmon_pylib/metrics/costMetrics.py +++ b/archive/ukmon_pylib/metrics/costMetrics.py @@ -17,7 +17,7 @@ def monthlyCostByService(dtstr, acctid): mth = int(dtstr[4:6]) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) df = pd.read_csv(os.path.join(datadir, 'costs', f'costs-{acctid}-90-days.csv')) if acctid != '183798037734': df = df[(df.Tag.str.contains('ukmda')) | (df.Tag.str.contains('ukmon'))] diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index 8114c4e3a..05d098edf 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -5,6 +5,9 @@ from datetime import datetime import sys +import os +import glob +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase def getDailyObsCounts(logf): @@ -33,33 +36,48 @@ def getDailyObsCounts(logf): return -def getMatchStats(logf): - with open(logf) as inf: - loglines = inf.readlines() +def getMatchStats(logf, rundate): + loglines = open(logf).readlines() + + events = [line.strip() for line in loglines if 'Analysing' in line and 'observations' in line] + if len(events) > 0: + # new-style log + addlines = [x for x in events if ' 0 observations' not in x] + addoffs = -5 + oldstyle = False + else: + addlines = [line.strip() for line in loglines if 'Added' in line and 'observations' in line] + addoffs = 1 + oldstyle = True - addlines = [line.strip() for line in loglines if 'Added' in line and 'observations' in line] - nocallines = [line.strip() for line in loglines if 'Skipping' in line and 'recalibrated' in line] - misdflines = [line.strip() for line in loglines if 'Skipping' in line and 'missing data' in line] - uncal = len(nocallines) - missdf = len(misdflines) + uncal = len([line.strip() for line in loglines if 'Skipping' in line and 'recalibrated' in line]) + missdf = len([line.strip() for line in loglines if 'Skipping' in line and 'missing data' in line]) + added=0 for li in addlines: spls=li.split(' ') - added = added + int(spls[1]) + added += int(spls[addoffs]) beglowr = len([line.strip() for line in loglines if 'Begin height lower than the end height' in line]) badalti = len([line.strip() for line in loglines if 'Meteor heights outside allowed range' in line]) badvelo = len([line.strip() for line in loglines if 'Velocity difference too high' in line]) badangl = len([line.strip() for line in loglines if 'Max convergence angle too small' in line]) - trajs = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] - spls = trajs[0].split(' ') - trajs = int(spls[1]) + cands = 0 + if oldstyle: + cands = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] + spls = cands[0].split(' ') + cands = int(spls[1]) + else: + trajlines = [line.strip() for line in loglines if 'Saved' in line and ' 0 candidates' not in line] + for li in trajlines: + spls=li.split(' ') + cands += int(spls[-2]) nonphys = beglowr + badalti + badvelo + badangl tot = added + uncal + missdf - rtims = [line.strip() for line in loglines if 'runDistrib' in line] + rtims = [line.strip() for line in loglines if 'runDistrib' in line and '/prod/' not in line] stim = rtims[0][11:19] etim = rtims[-1][11:19] d1=datetime.strptime(stim,'%H:%M:%S') @@ -73,9 +91,39 @@ def getMatchStats(logf): d2=datetime.strptime(etim,'%H:%M:%S') cstime = str(d2 - d1) - return tot, added, uncal, missdf, nonphys, trajs, runtime, cstime + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + dailydbdir = os.path.join(datadir, 'latest','dailydbs') + trajdb = TrajectoryDatabase(dailydbdir, f'{rundate}_trajectories.db') + trajs = len(trajdb.getTrajBasics('.',[0,9999999])) + + return tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime + + +def updateStats(obscount, candcount, trajcount, runtime, rundate): + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + statsdir = os.path.join(datadir, 'dailyreports') + reports = glob.glob(os.path.join(statsdir, f'{rundate}*.txt')) + + # only update stats if the daily report exists already + if len(reports) > 0: + dailyrep = os.path.split(reports[-1])[1] + statsdata = open(os.path.join(statsdir,'stats.txt'), 'r').readlines() + + # remove any existing entry for today + currentstats = [li for li in statsdata if rundate in li] + if len(currentstats) > 0: + statsdata.pop(statsdata.index(currentstats[0])) + # add the new entry and save + statsdata.append(f'{dailyrep} {obscount} {candcount} {trajcount} {runtime}\n') + open(os.path.join(statsdir,'stats.txt'), 'w').writelines(statsdata) + return if __name__ == '__main__': - tot, added, uncal, missdf, nonphys, trajs, runtime, cstime = getMatchStats(sys.argv[1]) - print(tot, added, uncal, missdf, nonphys, trajs, runtime, cstime) + matchfile = sys.argv[1] + rundate = sys.argv[2] + + tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(matchfile, rundate) + + updateStats(added, cands, trajs, runtime, rundate) + print(tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime) diff --git a/archive/ukmon_pylib/metrics/timingMetrics.py b/archive/ukmon_pylib/metrics/timingMetrics.py index 0f4606da5..d88273d09 100644 --- a/archive/ukmon_pylib/metrics/timingMetrics.py +++ b/archive/ukmon_pylib/metrics/timingMetrics.py @@ -47,7 +47,7 @@ def graphOfData(logf, dtstr): plt.grid(axis='x') plt.gca().invert_yaxis() logname, _ = os.path.splitext(os.path.basename(logf)) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) os.makedirs(os.path.join(datadir, 'batchcharts'), exist_ok=True) plt.savefig(os.path.join(datadir, 'batchcharts',f'./{dtstr}-{logname}.jpg'), dpi=100) plt.close() @@ -130,7 +130,7 @@ def getLogStats(nightlogf, matchlogf, thisdy): if __name__ == '__main__': dtstr = sys.argv[1] - logdir = os.path.join(os.getenv('SRC', default='/home/ec2-user/prod'), 'logs') + logdir = os.path.join(os.getenv('SRC',default=os.path.expanduser('~/prod')), 'logs') nowdt = datetime.datetime.now().strftime('%Y%m%d') if nowdt == dtstr: nightf = os.path.join(logdir, 'nightlyJob.log') diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index 74ce7ac86..3557c103d 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -13,7 +13,7 @@ def getCamLocDirFov(camid, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camdb = json.load(open(os.path.join(datadir, 'admin', 'cameraLocs.json'))) if camid not in camdb.keys(): return False @@ -22,7 +22,7 @@ def getCamLocDirFov(camid, datadir=None): def updateCamLocDirFovDB(datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camdb = {} ppfiles = glob.glob(os.path.join(datadir, 'consolidated','platepars', '*.json')) for ppf in ppfiles: @@ -41,9 +41,7 @@ def updateCamLocDirFovDB(datadir=None): def loadLocationDetails(table='camdetails', ddb=None, loadall=False): if not ddb: - prof = os.getenv('UKMPROFILE',default='ukmonshared') - conn = boto3.Session(profile_name=prof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table(table) res = table.scan() # strictly, should check for LastEvaluatedKey here, in case there was more than 1MB of data, @@ -71,7 +69,7 @@ def findLocationInfo(srchstring, ddb=None, statdets=None): return srchres -def createCDCsv(targetloc): +def createCDCsv(targetloc, srchidxdir): camdets = loadLocationDetails() cd4csv = camdets[['site','stationid','oldcode','direction','camtype','active']] pd.options.mode.chained_assignment = None @@ -80,31 +78,35 @@ def createCDCsv(targetloc): cd4csv = cd4csv[['site','stationid','oldcode','direction','camtype','dummycode','active']] cd4csv = cd4csv.rename(columns={'stationid':'camid', 'oldcode':'lid', 'direction':'sid'}) cd4csv.sort_values(by=['active','camid'], inplace=True) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) outfname = os.path.join(datadir, targetloc, 'camera-details.csv') cd4csv.to_csv(outfname, index=False) - with open(os.path.join(datadir, 'activecamcount.txt'), 'w') as outf: + with open(os.path.join(datadir, targetloc, 'activecamcount.txt'), 'w') as outf: outf.write(str(len(cd4csv[cd4csv.active==1]))) - createStatOptsHtml(camdets, datadir) - createStatOptsHtml(camdets, datadir, True) - createStatOptsHtml(camdets, datadir, True, True) + + outdir = os.path.join(datadir, srchidxdir) + os.makedirs(outdir, exist_ok=True) + + createStatOptsHtml(camdets, outdir) + createStatOptsHtml(camdets, outdir, True) + createStatOptsHtml(camdets, outdir, True, True) return -def createStatOptsHtml(sitedets, datadir, active=False, locs=False): +def createStatOptsHtml(sitedets, outdir, active=False, locs=False): """ Create an HTML file containing a list of stations or locations These files are used by the website search functions """ if active: if locs: - siteidx = os.path.join(datadir, 'activestatlocs.html') + siteidx = os.path.join(outdir, 'activestatlocs.html') sitedets = sitedets.sort_values(by=['site', 'stationid']) else: - siteidx = os.path.join(datadir, 'activestatopts.html') + siteidx = os.path.join(outdir, 'activestatopts.html') sitedets = sitedets[sitedets.active==1] else: - siteidx = os.path.join(datadir, 'statopts.html') + siteidx = os.path.join(outdir, 'statopts.html') with open(siteidx, 'w') as outf: if not active: outf.write('\n') diff --git a/archive/ukmon_pylib/reports/cameraStatusReport.py b/archive/ukmon_pylib/reports/cameraStatusReport.py index 0872feff7..83e4850e0 100644 --- a/archive/ukmon_pylib/reports/cameraStatusReport.py +++ b/archive/ukmon_pylib/reports/cameraStatusReport.py @@ -25,12 +25,12 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): caminfo = caminfo.drop(columns=['direction','oldcode','active','camtype','eMail', 'humanName']) caminfo.rename(columns={'site':'Site'}, inplace=True) if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) tmplist = pd.read_csv(os.path.join(datadir,'reports','camuploadtimes.csv'), index_col=False) fldrlist = pd.merge(tmplist, caminfo, on=['stationid'], how='inner') nowdt = datetime.datetime.now() - fldrlist.rundate.fillna(fldrlist.upddate.astype(str) + '_' + fldrlist.uploadtime.astype(str).str.pad(width=6,fillchar='0'), inplace=True) + fldrlist.fillna({'rundate': fldrlist.upddate.astype(str) + '_' + fldrlist.uploadtime.astype(str).str.pad(width=6,fillchar='0')}, inplace=True) fldrlist['dtstamp'] = [datetime.datetime.strptime(f,'%Y%m%d_%H%M%S') for f in fldrlist.rundate] fldrlist = fldrlist.sort_values(by=['stationid','dtstamp']) fldrlist.drop_duplicates(keep='last', subset=['stationid'], inplace=True) @@ -54,7 +54,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): def createStatusReportJSfile(stati, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) os.makedirs(os.path.join(datadir, 'reports'), exist_ok=True) repfile = os.path.join(datadir, 'reports','camrep.js') with open(repfile, 'w') as outf: diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 05aa32aa5..944c2ba03 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -8,15 +8,16 @@ import matplotlib.pyplot as plt import datetime -from meteortools.utils import jd2Date +from wmpl.Utils.TrajConversions import jd2Date def createBarChart(datadir=None, yr=None): if yr is None: yr=datetime.datetime.now().year if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') - fname = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + yr = str(yr) + fname = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') if not os.path.isfile(fname): print('{} missing', fname) return None @@ -36,19 +37,20 @@ def createBarChart(datadir=None, yr=None): dts.append(dt) fig, ax = plt.subplots() - width = 0.35 + width = 0.35 nowdt=datetime.datetime.now() ax.set_ylabel('# matches') ax.set_xlabel('Date') ax.set_title('Number of matched events per day. Last updated {}'.format(nowdt.strftime('%Y-%m-%d %H:%M:%S'))) - li.append(0) # comes up one short + while len(li) < len(dts): + li.append(0) ax.bar(dts, li, width, label='Events') fig = plt.gcf() fig.set_size_inches(12, 5) fig.tight_layout() #plt.gca().invert_yaxis() - plt.savefig(os.path.join(datadir,f'Annual-{yr}.jpg'), dpi=100) + plt.savefig(os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg'), dpi=100) plt.close() return matches diff --git a/archive/ukmon_pylib/reports/createExchangeFiles.py b/archive/ukmon_pylib/reports/createExchangeFiles.py index 820650ab5..e11a5588a 100644 --- a/archive/ukmon_pylib/reports/createExchangeFiles.py +++ b/archive/ukmon_pylib/reports/createExchangeFiles.py @@ -164,7 +164,7 @@ def createCameraFile(datadir): def createAll(targdate=None, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/datas') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if targdate is None: targdate = datetime.datetime.now() createCameraFile(datadir) diff --git a/archive/ukmon_pylib/reports/createSearchableFormat.py b/archive/ukmon_pylib/reports/createSearchableFormat.py index e2df79c2b..c567fa572 100644 --- a/archive/ukmon_pylib/reports/createSearchableFormat.py +++ b/archive/ukmon_pylib/reports/createSearchableFormat.py @@ -23,7 +23,7 @@ def checkUrl(s3, url): except Exception: #print(e) retval = '/img/missing-white.png' - print(url, retval) + #print(url, retval) return retval @@ -110,7 +110,7 @@ def convertMatchToSrchable(datadir, year, newonly=True): print('usage: python createSearchableFormat.py year mode outdir') exit(1) else: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) year = sys.argv[1] mode = sys.argv[2] diff --git a/archive/ukmon_pylib/reports/createSummaryTable.py b/archive/ukmon_pylib/reports/createSummaryTable.py index cff8e40ce..b31500027 100644 --- a/archive/ukmon_pylib/reports/createSummaryTable.py +++ b/archive/ukmon_pylib/reports/createSummaryTable.py @@ -16,10 +16,13 @@ def createSummaryTable(curryr=None, datadir=None): """ if curryr is None: - curryr = str(datetime.datetime.now().year) + curryr = datetime.datetime.now().year if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') - fname = os.path.join(datadir, 'summarytable.js') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + curryr = str(curryr) + outdir = os.path.join(datadir, 'reports', curryr) + os.makedirs(outdir, exist_ok=True) + fname = os.path.join(outdir, 'summarytable.js') with open(fname, 'w') as f: f.write('$(function() {\n') f.write('var table = document.createElement("table");\n') diff --git a/archive/ukmon_pylib/reports/dailyReport.py b/archive/ukmon_pylib/reports/dailyReport.py index 8c7852070..3c5bc8979 100644 --- a/archive/ukmon_pylib/reports/dailyReport.py +++ b/archive/ukmon_pylib/reports/dailyReport.py @@ -6,7 +6,7 @@ import sys import datetime -from meteortools.utils import sendAnEmail +from utils.sendAnEmail import sendAnEmail def AddHeader(body, bodytext, stats): @@ -55,11 +55,7 @@ def AddRowRMS(body, bodytext, ele): return body, bodytext -def LookForMatchesRMS(doff, dayfile, statsfile): - # get stats - with open(statsfile, 'r') as inf: - lis = inf.readlines() - stats = lis[-1].strip().split(' ') +def LookForMatchesRMS(doff, dayfile, stats): bodytext = 'Daily notification of matches\n\n' body = '' @@ -100,7 +96,14 @@ def LookForMatchesRMS(doff, dayfile, statsfile): dailyreps.sort() dailyrep = dailyreps[-1] statfile = os.path.join(reppth, 'stats.txt') - _, _, body, bodytext = LookForMatchesRMS(doff, dailyrep, statfile) + lis = open(statfile, 'r').readlines() + stats = [li for li in lis if repdtstr in li] + if len(stats) > 0: + stats = stats[0].strip().split(' ') + else: + stats = lis[-1].strip().split(' ') + + _, _, body, bodytext = LookForMatchesRMS(doff, dailyrep, stats) #body = body.replace('assets/img/logo.svg', 'latest/dailyreports/dailyreportsidx.html') if doff == 1: @@ -115,7 +118,7 @@ def LookForMatchesRMS(doff, dayfile, statsfile): # now send the post to groups.io targeturl='https://archive.ukmeteors.co.uk' - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) recs = open(os.path.join(datadir, 'admin','dailyReportRecips.txt'), 'r').readlines() mailFrom = 'markmcintyre99@googlemail.com' mailRecip = recs[0].strip() @@ -123,5 +126,7 @@ def LookForMatchesRMS(doff, dayfile, statsfile): mailFrom = recs[-1].strip() #mailRecip = 'markmcintyre99@googlemail.com' # TODO for testing only yest = (datetime.date.today()-datetime.timedelta(days=doff)).strftime('%Y-%m-%d') - mailSubj = f'Latest Match Report for {yest}' - sendAnEmail(mailRecip, bodytext, mailSubj, mailFrom, msg_html=body.replace('href="/reports', f'href="{targeturl}/reports')) + # only send the email for the most recent report. + if doff == 1: + mailSubj = f'Latest Match Report for {yest}' + sendAnEmail(mailRecip, bodytext, mailSubj, mailFrom, msg_html=body.replace('href="/reports', f'href="{targeturl}/reports')) diff --git a/archive/ukmon_pylib/reports/extractors.py b/archive/ukmon_pylib/reports/extractors.py index d7fc4bb80..8541e7598 100644 --- a/archive/ukmon_pylib/reports/extractors.py +++ b/archive/ukmon_pylib/reports/extractors.py @@ -6,8 +6,8 @@ import os import sys import pandas as pd -from meteortools.fileformats import imoWorkingShowerList as imo -from meteortools.utils import getActiveShowers +from utils.imoWorkingShowerList import IMOshowerList +from utils.getActiveShowers import getActiveShowers def createSplitMatchFile(yr, mth=None, shwr=None, matches=None): @@ -19,7 +19,7 @@ def createSplitMatchFile(yr, mth=None, shwr=None, matches=None): shwr (string): optional shower code. If provided, the data will be filtered """ - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if matches is None: infname = os.path.join(datadir, 'matched',f'matches-full-{yr}.parquet.snap') if not os.path.isfile(infname): @@ -58,7 +58,7 @@ def createUFOSingleMonthlyExtract(yr, mth=None, shwr=None, dta=None): """ # print('ufo singles file') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if dta is None: fname = os.path.join(datadir, 'consolidated','M_{}-unified.csv'.format(yr)) if not os.path.isfile(fname): @@ -94,7 +94,7 @@ def createRMSSingleMonthlyExtract(yr, mth=None, shwr=None, dta=None, withshower= """ #print(f'rms singles file, withshower {withshower}') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if dta is None: fname = os.path.join(datadir, 'single','singles-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): @@ -143,11 +143,11 @@ def extractAllShowersData(ymd): showerlist = getActiveShowers(ymd, retlist=True) showerlist.append('spo') else: - sl = imo.IMOshowerList() + sl = IMOshowerList() showerlist = sl.getMajorShowers(True, True).strip().split(' ') print(f'processing data for {ymd}') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) infname = os.path.join(datadir, 'matched',f'matches-full-{yr}.parquet.snap') if not os.path.isfile(infname): print(f'unable to open {infname}') @@ -159,8 +159,9 @@ def extractAllShowersData(ymd): fname = os.path.join(datadir, 'consolidated','M_{}-unified.csv'.format(yr)) if not os.path.isfile(fname): print(f'unable to open {fname}') - return - ufosingles = pd.read_csv(fname, skipinitialspace=True) + ufosingles = None + else: + ufosingles = pd.read_csv(fname, skipinitialspace=True) fname = os.path.join(datadir, 'single','singles-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): print(f'unable to open {fname}') @@ -174,8 +175,9 @@ def extractAllShowersData(ymd): print(f'processing RMS singles for {shwr}') createRMSSingleMonthlyExtract(yr, shwr=shwr, dta=rmssingles) createRMSSingleMonthlyExtract(yr, shwr=shwr, dta=rmssingles, withshower=True) - print(f'processing UFO singles for {shwr}') - createUFOSingleMonthlyExtract(yr, shwr=shwr, dta=ufosingles) + if ufosingles: + print(f'processing UFO singles for {shwr}') + createUFOSingleMonthlyExtract(yr, shwr=shwr, dta=ufosingles) return diff --git a/archive/ukmon_pylib/reports/findBestMp4s.py b/archive/ukmon_pylib/reports/findBestMp4s.py index a07611ddd..22f533700 100644 --- a/archive/ukmon_pylib/reports/findBestMp4s.py +++ b/archive/ukmon_pylib/reports/findBestMp4s.py @@ -20,7 +20,7 @@ def getBestNMatches(reqdate=None, numtoget=10): tod = reqdate + datetime.timedelta(days=1) yr = reqdate.year - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') # select only the columns we need @@ -81,7 +81,7 @@ def getUrlFromFilename(fname): def getBestNMp4s(yr, mth, numtoget): - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') # select only the columns we need diff --git a/archive/ukmon_pylib/reports/findFailedMatches.py b/archive/ukmon_pylib/reports/findFailedMatches.py index c6005fe7e..1b110a1c6 100644 --- a/archive/ukmon_pylib/reports/findFailedMatches.py +++ b/archive/ukmon_pylib/reports/findFailedMatches.py @@ -46,7 +46,7 @@ def processOneLog(logfile, repfile): srcdir = os.getenv('SRC', default='.') logdir = os.path.join(srcdir, 'logs','distrib') - logs = glob.glob1(logdir, f'{repdt}*.log') + logs = glob.glob(f'{repdt}*.log', root_dir=logdir) for logf in logs: processOneLog(os.path.join(logdir, logf), reportfile) reportfile.close() diff --git a/archive/ukmon_pylib/reports/findFireballs.py b/archive/ukmon_pylib/reports/findFireballs.py index 60b7af0c0..178bd1247 100644 --- a/archive/ukmon_pylib/reports/findFireballs.py +++ b/archive/ukmon_pylib/reports/findFireballs.py @@ -12,14 +12,14 @@ from shutil import rmtree from traj.pickleAnalyser import getBestView -from meteortools.utils import jd2Date +from wmpl.Utils.TrajConversions import jd2Date # # Manually mark a trajectoriy as a "fireball" # def markAsFireball(trajname, tof=True): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) yr=trajname[:4] if int(yr) > 2021: fname = os.path.join(datadir, 'matched','matches-full-{}.parquet.snap'.format(yr)) @@ -142,7 +142,7 @@ def findFBPre2020(df, mag=-4): def findFireballs(dtval, shwr, minmag=-3.99, matchdataset = None): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) orbdir = 'matches' # check if month was passed in diff --git a/archive/ukmon_pylib/reports/getLivestreamData.py b/archive/ukmon_pylib/reports/getLivestreamData.py index 3db0dfe24..18c15b1d3 100644 --- a/archive/ukmon_pylib/reports/getLivestreamData.py +++ b/archive/ukmon_pylib/reports/getLivestreamData.py @@ -72,7 +72,7 @@ def getLatestLiveFiles(daysback=None, df=None): if __name__ == '__main__': - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) df = getLatestLiveFiles(-30) prvfile = os.path.join(datadir, 'ukmonlive', 'livefeed.csv') if os.path.isfile(prvfile): diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index 2bdb89c52..b25f8beca 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -6,8 +6,9 @@ import os import gmplot import glob +import boto3 from cryptography.fernet import Fernet -from meteortools.fileformats import readCameraKML +from utils.kmlHandlers import readCameraKML def decodeApiKey(enckey): @@ -24,16 +25,27 @@ def encodeApiKey(plainkey): return apikey.decode('utf-8') +def getApiKey(): + ssm = boto3.client('ssm') + res = ssm.get_parameter(Name='prod_gmapsapikey', WithDecryption=True) + if 'Parameter' in res: + return res['Parameter']['Value'] + else: + return False + + def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): - apikey = os.getenv('APIKEY') - apikey = decodeApiKey(apikey) + + apikey = getApiKey() + if not apikey: + return kmltempl = os.getenv('KMLTEMPLATE', default='*70km.kml') heightval = kmltempl[1:-4] gmap = gmplot.GoogleMapPlotter(52, -1.0, 5, apikey=apikey, title=f'Camera Coverage at {heightval}', map_type='satellite') - flist = glob.glob1(kmlsource, kmltempl) + flist = glob.glob(kmltempl, root_dir=kmlsource) cols = list(gmplot.color._HTML_COLOR_CODES.keys()) for col, fn in zip(cols,flist): @@ -55,14 +67,15 @@ def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): return -def createCoveragePage(): - apikey = os.getenv('APIKEY') - apikey = decodeApiKey(apikey) - templdir = os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') +def createCoveragePage(targdir): + apikey = getApiKey() + if not apikey: + return + templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) with open(os.path.join(templdir, 'coverage-maps.html'), 'r') as inf: lis = inf.readlines() - with open(os.path.join(datadir, 'latest','coverage-maps.html'), 'w') as outf: + with open(os.path.join(datadir, targdir,'coverage-maps.html'), 'w') as outf: for li in lis: outf.write(li.replace('{{MAPSAPIKEY}}', apikey)) return diff --git a/archive/ukmon_pylib/reports/meteoriteTools.py b/archive/ukmon_pylib/reports/meteoriteTools.py index 4f7c72e7b..e699bb072 100644 --- a/archive/ukmon_pylib/reports/meteoriteTools.py +++ b/archive/ukmon_pylib/reports/meteoriteTools.py @@ -8,7 +8,7 @@ import numpy as np import requests -from meteortools.utils.Math import greatCircleDistance +from wmpl.Utils.Earth import greatCircleDistance def stationsNearPoint(lat, lon, dist=75, email_only=True): diff --git a/archive/ukmon_pylib/reports/reportActiveShowers.py b/archive/ukmon_pylib/reports/reportActiveShowers.py index 7786eafa6..bf2a171a0 100644 --- a/archive/ukmon_pylib/reports/reportActiveShowers.py +++ b/archive/ukmon_pylib/reports/reportActiveShowers.py @@ -9,13 +9,13 @@ import shutil import argparse -from meteortools.utils import getActiveShowers +from utils.getActiveShowers import getActiveShowers from analysis.showerAnalysis import showerAnalysis from reports.findFireballs import findFireballs def createShowerIndexPage(dtstr, shwr, shwrname, outdir, datadir): - templdir = os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) idxfile = os.path.join(outdir, 'index.html') shutil.copyfile(os.path.join(templdir,'header.html'), idxfile) @@ -109,8 +109,8 @@ def createShowerIndexPage(dtstr, shwr, shwrname, outdir, datadir): outf.write('The graphs and histograms below show more information about the velocity, magnitude \n') outf.write('start and end altitude and other parameters. Click for larger view. \n') # add the charts and stuff - jpglist = glob.glob1(outdir, '*.jpg') - pnglist = glob.glob1(outdir, '*.png') + jpglist = glob.glob('*.jpg', root_dir=outdir) + pnglist = glob.glob('*.png', root_dir=outdir) outf.write('
    \n') for j in jpglist: outf.write(f'\n') @@ -175,7 +175,7 @@ def reportActiveShowers(ymd, thisshower=None, thismth=None, includeMinor=False): else: shwrlist = [thisshower] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) # pltdir=os.path.join(datadir, 'showerplots') dtstr = ymd[:4] if thismth is not None: diff --git a/archive/ukmon_pylib/reports/reportBadCameras.py b/archive/ukmon_pylib/reports/reportBadCameras.py index bdf4f6ef2..adf4d64ca 100644 --- a/archive/ukmon_pylib/reports/reportBadCameras.py +++ b/archive/ukmon_pylib/reports/reportBadCameras.py @@ -10,7 +10,7 @@ import textwrap from reports.CameraDetails import loadLocationDetails -from meteortools.utils import sendAnEmail +from utils.sendAnEmail import sendAnEmail mailfrom = 'ukmonhelper2@ukmeteors.co.uk' @@ -19,7 +19,7 @@ print('usage: python reportBadCameras.py daysback') exit(0) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if datadir is None: print('define DATADIR first') exit(1) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index f7eeed01a..d79ddefff 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -8,14 +8,17 @@ import datetime import numpy import csv -import json import shutil import tempfile import boto3 +import glob +import tarfile from traj.pickleAnalyser import getVMagCodeAndStations from reports.CameraDetails import findSite, loadLocationDetails +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase, ObservationsDatabase + def processLocalFolder(trajdir, basedir): # load camera details @@ -39,48 +42,104 @@ def processLocalFolder(trajdir, basedir): return outstr -def getTrajPaths(trajdict): - trajpaths=[] - fullnames=[] - for traj in trajdict: - fullnames.append(trajdict[traj]['traj_file_path']) - pth, _ = os.path.split(trajdict[traj]['traj_file_path']) - trajpaths.append(pth) - return trajpaths, fullnames - - -def getListOfNewMatches(dir_path, tfile, prevtfile): - with open(os.path.join(dir_path, tfile), 'r') as inf: - trajs = json.load(inf) - with open(os.path.join(dir_path, prevtfile), 'r') as inf: - ptrajs = json.load(inf) - newtrajs = {k:v for k,v in trajs['trajectories'].items() if k not in ptrajs['trajectories']} - #print(len(newtrajs)) - _, newdirs = getTrajPaths(newtrajs) +def getListOfNewMatches(dir_path, db_path, rundate): + os.makedirs(db_path, exist_ok=True) + db_name = f'{rundate}_trajectories.db' + dailydb = TrajectoryDatabase(db_path=db_path, db_name=db_name, purge_records=True) + flist = glob.glob(os.path.join(dir_path, f'trajectories_{rundate}_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if dailydb.mergeTrajDatabase(fl): + os.remove(fl) + else: + print('error') + + if os.getenv('TESTMODE').lower() == 'true': + trajdir = 'matches/distrib/test' + else: + trajdir = 'matches/RMSCorrelate' + + cur = dailydb.dbhandle.execute('select traj_file_path from trajectories where status=1') + newtrajs = cur.fetchall() + + if len(newtrajs) > 0: + # now get a list of logically-deleted trajs from the current master DB + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + + # get the date range for the new trajectories + cur = dailydb.dbhandle.execute('select min(jdt_ref), max(jdt_ref) from trajectories where status=1') + vals = cur.fetchall() + jdt_beg = float(vals[0][0]) + jdt_end = float(vals[0][1]) + + # retrieve a list of logically-deleted trajectories from within that date range + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + # iterate over the delete list and update the daily db and new traj list accordingly + print('cleaning up deleted trajectories in dailydb') + for testtr in deltrajs: + if testtr in newtrajs: + sqlstr = f'update trajectories set status=0 where traj_file_path="{testtr[0]}"' + dailydb.dbhandle.execute(sqlstr) + dailydb.dbhandle.commit() + cur = dailydb.dbhandle.execute(f'select traj_id, status from trajectories where traj_file_path="{testtr[0]}"') + msg = cur.fetchone() + print(f'updated {msg}') + newtrajs.pop(newtrajs.index(testtr)) + + dailydb.closeTrajDatabase() + + newdirs = [] + for traj in newtrajs: + newdirs.append(os.path.join(trajdir, traj[0])) + return newdirs -def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): - prevdbname = 'prev_' + dbname - newdirs = getListOfNewMatches(dir_path, dbname, prevdbname) +def updatePairedDB(dir_path, db_path, rundate): + os.makedirs(db_path, exist_ok=True) + db_name = f'{rundate}_observations.db' + obsdb = ObservationsDatabase(db_path=db_path, db_name=db_name, purge_records=True) + flist = glob.glob(os.path.join(dir_path, f'observations_{rundate}_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if obsdb.mergeObsDatabase(fl): + os.remove(fl) + else: + print('error') + + cur = obsdb.dbhandle.execute('select count(*) from paired_obs where status=1') + obscount = cur.fetchall() + + return obscount + + +def findNewMatches(dir_path, out_path, repdtstr): + daily_path = os.path.join(os.path.split(dir_path)[0], 'dailydbs') + newdirs = getListOfNewMatches(dir_path, daily_path, rundate=repdtstr) # load camera details caminfo = loadLocationDetails() caminfo = caminfo[caminfo.active==1] - if repdtstr is not None: - repdt = datetime.datetime.strptime(repdtstr, '%Y%m%d') - else: - repdt = datetime.datetime.now() - datetime.timedelta(int(offset)) + repdt = datetime.datetime.strptime(repdtstr, '%Y%m%d') - os.makedirs(os.path.join(out_path, 'dailyreports'), exist_ok=True) + os.makedirs(out_path, exist_ok=True) # create filename. Allow for three reruns in a day - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_1.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_1.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_2.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_2.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_3.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_3.txt')) s3 = boto3.client('s3') srcbucket=os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] @@ -90,10 +149,11 @@ def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): trajdir = trajdir[trajdir.find('matches'):] trajpath, picklename = os.path.split(trajdir) localpick = os.path.join(tmpdir, picklename) - s3.download_file(srcbucket, trajdir, localpick) try: + s3.download_file(srcbucket, trajdir, localpick) bestvmag, shwr, stationids = getVMagCodeAndStations(localpick) except: + print(f'unable to find {trajdir}') bestvmag, shwr, stationids = 0,'',[''] stations=[] for statid in stationids: @@ -127,20 +187,34 @@ def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): outf.write('\n') # finally, create a "latest.txt" as well - latestlist = os.path.join(out_path, 'dailyreports', 'latest.txt') + latestlist = os.path.join(out_path, 'latest.txt') shutil.copy(matchlist, latestlist) return if __name__ == '__main__': - repdtstr = None - dbname = None - if len(sys.argv) > 4: - repdtstr = sys.argv[4] - if len(sys.argv) > 5: - dbname = sys.argv[5] - else: - dbname = 'processed_trajectories.json.bigserver' + + cand_db_dir = sys.argv[1] + daily_db_dir = sys.argv[2] + repdtstr = sys.argv[3] + + if repdtstr != datetime.datetime.now().strftime('%Y%m%d'): + print(f'running for a past date, extracting the container dbs to {cand_db_dir}') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + tarf = tarfile.open(os.path.join(datadir, 'distrib','containers',f'contdbs_{repdtstr}.tgz'), 'r') + members = [m for m in tarf.getmembers() if '.db' in m.name] + for m in members: + m.name = os.path.basename(m.name) + tarf.extractall(cand_db_dir, filter='fully_trusted') + tarf.close() - # arguments dblocation, datadir, days ago, rundate eg 20220524, full path to database - findNewMatches(sys.argv[1], sys.argv[2], sys.argv[3], repdtstr, dbname) + flist = glob.glob(os.path.join(cand_db_dir, 'trajectories_*.db')) + if len(flist) == 0: + print('no container databases to process, aborting') + else: + # arguments dblocation, datadir, days ago, rundate eg 20220524 + findNewMatches(cand_db_dir, daily_db_dir, repdtstr) + # update the daily database of paired observations + daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') + updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) + print('reportOfLatestMatches finished') diff --git a/archive/ukmon_pylib/reports/trackThumbnails.py b/archive/ukmon_pylib/reports/trackThumbnails.py index 0e012e1db..65837803b 100644 --- a/archive/ukmon_pylib/reports/trackThumbnails.py +++ b/archive/ukmon_pylib/reports/trackThumbnails.py @@ -7,7 +7,7 @@ import argparse import requests from tempfile import mkdtemp -from meteortools.utils import greatCircleDistance +from wmpl.Utils.Earth import greatCircleDistance RAD2DEG=57.2958 diff --git a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py index 793c3ee11..7dac55f6c 100644 --- a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py +++ b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py @@ -17,14 +17,14 @@ def test_createBarChart(): # pass year and datadir res = createBarChart(datadir=datadir, yr=yr) - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) os.remove(outf) # pass datadir only - will fail if we're not in 2023 res = createBarChart(datadir=datadir, yr=None) - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: @@ -35,7 +35,7 @@ def test_createBarChart(): # no params passed res = createBarChart() - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: diff --git a/archive/ukmon_pylib/tests/test_reports_various.py b/archive/ukmon_pylib/tests/test_reports_various.py index d9aeac7b4..9ddc3ba64 100644 --- a/archive/ukmon_pylib/tests/test_reports_various.py +++ b/archive/ukmon_pylib/tests/test_reports_various.py @@ -10,8 +10,9 @@ def test_createSummaryTable(): - createSummaryTable(curryr=None, datadir=datadir) - fname = os.path.join(datadir, 'summarytable.js') + yr=2023 + createSummaryTable(curryr=yr, datadir=datadir) + fname = os.path.join(datadir,'reports',str(yr), 'summarytable.js') assert os.path.isfile(fname) lis = open(fname, 'r').readlines() assert 'cell.innerHTML = ' in lis[9] diff --git a/archive/ukmon_pylib/traj/clusdetails-ee.txt b/archive/ukmon_pylib/traj/clusdetails-ee.txt deleted file mode 100644 index 1e17ed3f7..000000000 --- a/archive/ukmon_pylib/traj/clusdetails-ee.txt +++ /dev/null @@ -1,6 +0,0 @@ -trajsolver -sg-0d37a3b8ee1a3a1c6 -subnet-0c224d5642fb71023 -arn:aws:iam::822069317839:role/ecsTaskExecutionRole -/ecs/trajcont -trajcont diff --git a/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt new file mode 100644 index 000000000..d3be6424f --- /dev/null +++ b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt @@ -0,0 +1,6 @@ +trajsolvertest +sg-06fb7bbee3b4f5e94 +subnet-019eef41d5eaf419b +arn:aws:iam::183798037734:role/ecsTaskExecutionRole +/ecs/trajcontest +trajconttest diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index b213f4b45..0202a06fe 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -3,123 +3,98 @@ import os import sys import glob -import shutil import datetime - -from wmpl.Trajectory.CorrelateRMS import TrajectoryReduced, DatabaseJSON -#from wmpl.Trajectory.CorrelateRMS import MeteorObsRMS, PlateparDummy, MeteorPointRMS # noqa: F401 - - -# support class so i can add any paired obs objects -class dummyMeteorObsRMS(object): - def __init__(self, station_code, id): - self.station_code = station_code - self.id = id - - -# merge a distributed engine DB back into the master DB -def mergeDatabases(srcdir, srcdb, masterpth, masterfile, mastdb = None): - newdb = os.path.join(srcdir, srcdb) - mergedb = DatabaseJSON(newdb) - if mastdb is None: - mastdb = DatabaseJSON(masterfile) - mastdb.db_file_path = masterfile - # merge successful trajectories - for traj in mergedb.trajectories: - traj_obj = TrajectoryReduced(traj, json_dict = mergedb.trajectories[traj].__dict__) - traj_file_path = traj_obj.traj_file_path - traj_file_path = masterpth + '/' + traj_file_path[traj_file_path.find('trajectories'):] - traj_obj.traj_file_path = traj_file_path - mastdb.addTrajectory(traj_file_path, traj_obj=traj_obj) - # merge failed trajectories - for traj in mergedb.failed_trajectories: - traj_obj = TrajectoryReduced(traj, json_dict = mergedb.failed_trajectories[traj].__dict__) - traj_file_path = traj_obj.traj_file_path - traj_file_path = masterpth + '/' + traj_file_path[traj_file_path.find('trajectories'):] - traj_obj.traj_file_path = traj_file_path - mastdb.addTrajectory(traj_file_path, traj_obj=traj_obj, failed=True) - # merge paired obs data - for p in mergedb.paired_obs: - ids = mergedb.paired_obs[p] - for id in ids: - met_obs = dummyMeteorObsRMS(p, id) - mastdb.addPairedObservation(met_obs) - # save the master DB again - mastdb.save() - return mastdb - - -# utility to patch the database to have the right trajectory folder -def patchTrajDB(dbfile, targpath, oldstr='/home/ec2-user/data/RMSCorrelate'): - - with open(dbfile, 'r') as inf: - with open(os.path.join('/tmp/processed_trajectories.json'), 'w') as outf: - for lin in inf: - outf.write(lin.replace(oldstr, targpath)) - shutil.copyfile('/tmp/processed_trajectories.json', dbfile) - return - - -# utility to count how many of each type of observation was in a database -def countDataInDb(path_to_db): - mergedb = DatabaseJSON(path_to_db) - trajs = len(mergedb.trajectories) - failed = len(mergedb.failed_trajectories) - pairs = len(mergedb.paired_obs) - print(path_to_db, trajs, failed, pairs) - return - - -# utility to dump the detection dates (JDs) to a file -def dumpJDs(path_to_db, fname): - mergedb = DatabaseJSON(path_to_db) - with open(fname,'w') as outf: - outf.write('traj\n') - for t in mergedb.trajectories: - outf.write(f'{t}\n') - outf.write('failed\n') - for t in mergedb.failed_trajectories: - outf.write(f'{t}\n') - - -# utility to compare two databases and dump the JDs of any new events -def dumpNewEntries(db1, db2, fname): - olddb = DatabaseJSON(db1) - newdb = DatabaseJSON(db2) - newtraj = set(newdb.trajectories) - set(olddb.trajectories) - newfail = set(newdb.failed_trajectories) - set(olddb.failed_trajectories) - print(len(newtraj), len(newfail)) - with open(fname,'w') as outf: - outf.write('traj\n') - for t in newtraj: - outf.write(f'{t}\n') - outf.write('failed\n') - for t in newfail: - outf.write(f'{t}\n') +import logging + +from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase +from wmpl.Trajectory.CorrelateRMS import RMSDataHandle +from wmpl.Utils.TrajConversions import jd2Date + +log = logging.getLogger("consol_disttraj") + + +def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=False, matchstart=3): + """ + merge container databases into the master database, looking for and cleaning deleted trajectories + + arguments: + srcdir location of container databases + dbdir location of master databases + basedir location of raw data and trajectories + + keyword args: + ignore_missing default false: if true, create master DBs if not present + purge_records default false: if true, purge records from master DBs + matchstart default 3: range of days to scan for duplicate and deleted trajectories + + """ + + targdb = os.path.join(dbdir, 'observations.db') + if os.path.isfile(targdb) or ignore_missing: + obsdb = ObservationsDatabase(dbdir, purge_records=purge_records) + flist = glob.glob(os.path.join(srcdir, 'observations_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if obsdb.mergeObsDatabase(fl): + os.remove(fl) + obsdb.closeObsDatabase() + + targdb = os.path.join(dbdir, 'trajectories.db') + if os.path.isfile(targdb) or ignore_missing: + trajdb = TrajectoryDatabase(dbdir, purge_records=purge_records) + flist = glob.glob(os.path.join(srcdir, 'trajectories_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if trajdb.mergeTrajDatabase(fl): + os.remove(fl) + + # get the latest date in the database for use below + cur = trajdb.dbhandle.execute('select max(jdt_ref) from trajectories where status=1') + vals = cur.fetchall() + jdt_end = float(vals[0][0]) + trajdb.closeTrajDatabase() + + # find and remove duplicates using WMPL's built-in routine to clean up the trajectory DB + # NB: can only be run on the calcserver where trajectory folders are present + + # select range midday on the latest date to matchstart days earlier + dt_end = jd2Date(int(jdt_end) + 1, dt_obj=True, tzinfo=datetime.timezone.utc) + dt_beg = jd2Date(int(jdt_end) + 1 - matchstart, dt_obj=True, tzinfo=datetime.timezone.utc) + event_time_range = [dt_beg, dt_end] + + # capture RMSDataHandle log messages to console + log.setLevel(logging.DEBUG) + console_handler = logging.StreamHandler() + log.addHandler(console_handler) + + dh = RMSDataHandle(basedir, dt_range=event_time_range, db_dir=dbdir, output_dir=basedir,mcmode=1, archivemonths=0) + dh.updateTrajectoryDatabase(dt_range=event_time_range) + + # should never need to run this part as the candidates are all made in one place + targdb = os.path.join(dbdir, 'candidates.db') + if os.path.isfile(targdb): + canddb = CandidateDatabase(dbdir) + flist = glob.glob(os.path.join(srcdir, 'candidates_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if canddb.mergeCandDatabase(fl): + os.remove(fl) + canddb.closeCandDatabase() + + print('done') if __name__ == '__main__': if len(sys.argv) < 3: - print('usage: consolidateDistTraj folder_containing_srcdbs full_path_to_targb') + print('usage: consolidateDistTraj folder_containing_srcdbs targ_dbdir') exit(0) srcdir = sys.argv[1] - masterdb = sys.argv[2] - if len(sys.argv) > 3: - rundt = sys.argv[3] - else: - rundt = datetime.datetime.now().strftime('%Y%m%d') - # real path to the trajectories as per the master database - masterpth = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' - - flist = glob.glob1(srcdir, f'{rundt}*.json') - flist.sort() - mastdb = None - for fl in flist: - #countDataInDb(os.path.join(srcdir, fl)) - tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') - print(f'{tstamp} processing {fl}') - sys.stdout.flush() - mastdb = mergeDatabases(srcdir, fl, masterpth, masterdb, mastdb) - - print('patching path in mastdb') - patchTrajDB(masterdb, masterpth, oldstr='/home/ec2-user/prod/data/distrib') + dbdir = sys.argv[2] + basedir = os.path.dirname(os.path.normpath(dbdir)) + mergeDatabases(srcdir, dbdir, basedir) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 97555ca96..dfe4624aa 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -6,21 +6,8 @@ import os import sys import datetime -import json - -# read the task template to determine the paths to write any data to -# -def getTrajsolverPaths(): - templdir,_ = os.path.split(__file__) - with open(os.path.join(templdir, 'taskrunner.json'), 'r') as inf: - taskdets = json.load(inf) - taskenv = taskdets['overrides']['containerOverrides'][0]['environment'] - srcpath = [x for x in taskenv if x['name']=='SRCPATH'][0]['value'] # path from which trajsolver will consume candidates - outpath = [x for x in taskenv if x['name']=='OUTPATH'][0]['value'] # path to which trajsolver will publish trajectories - webpath = [x for x in taskenv if x['name']=='WEBPATH'][0]['value'] # web loc to which trajsolver will publish trajectories - - return srcpath, outpath, webpath +from traj.distributeCandidates import getTrajsolverPaths # make sure the local trajectories folder is synced with the master copy @@ -36,7 +23,7 @@ def refreshTrajectories(outf, matchstart, matchend, trajpath): return -# make sure the master copy is updated with any new locally updated trajectories +# make sure the shared bucket is updated with any new locally updated trajectories # def pushUpdatedTrajectoriesShared(outf, matchstart, matchend, targpath): for d in range(matchend, matchstart+1): @@ -46,15 +33,17 @@ def pushUpdatedTrajectoriesShared(outf, matchstart, matchend, targpath): ymd=thisdt.strftime('%Y%m%d') trajloc=f'trajectories/{yr}/{ym}/{ymd}' outf.write(f'if [ -d {trajloc} ] ; then \n') - outf.write(f'aws s3 sync {trajloc} {targpath}/matches/RMSCorrelate/{trajloc} --exclude "*" --include "*.pickle" --include "*report.txt" --quiet\n') - outf.write(f'aws s3 sync {trajloc}/plots {targpath}/matches/RMSCorrelate/{trajloc}/plots --quiet\n') + outf.write(f'aws s3 sync {trajloc} {targpath}/{trajloc} --exclude "*" --include "*.pickle" --include "*report.txt" --quiet\n') + outf.write(f'if [ -d {trajloc}/plots ] ; then \n') + outf.write(f'aws s3 sync {trajloc}/plots {targpath}/{trajloc}/plots --quiet\n') outf.write('fi\n') - outf.write(f'aws s3 sync trajectories/{yr}/plots {targpath}/matches/RMSCorrelate/trajectories/{yr}/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {targpath}/matches/RMSCorrelate/trajectories/{yr}/{ym}/plots --quiet\n') + outf.write('fi\n') + outf.write(f'aws s3 sync trajectories/{yr}/plots {targpath}/trajectories/{yr}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {targpath}/trajectories/{yr}/{ym}/plots --quiet\n') return -# make sure the master copy is updated with any new locally updated trajectories +# make sure the website is updated with any new locally updated trajectories # def pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath): for d in range(matchend, matchstart+1): @@ -63,16 +52,18 @@ def pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath): ym=thisdt.strftime('%Y%m') ymd=thisdt.strftime('%Y%m%d') trajloc=f'trajectories/{yr}/{ym}/{ymd}' - targloc=f'reports/{yr}/orbits/{ym}/{ymd}' + targloc=f'{yr}/orbits/{ym}/{ymd}' outf.write(f'if [ -d {trajloc} ] ; then \n') outf.write(f'aws s3 sync {trajloc} {webpath}/{targloc} --quiet\n') outf.write('fi\n') outf.write(f'aws s3 sync {trajloc}/plots {webpath}/{targloc}/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/plots {webpath}/reports/{yr}/orbits/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {webpath}/reports/{yr}/orbits/{ym}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/plots {webpath}/{yr}/orbits/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {webpath}/{yr}/orbits/{ym}/plots --quiet\n') return +# Create the density plots showing meteor showers +# def createDensityPlots(outf, calcdir, enddt, includeyear=True): yr = enddt.year ym = enddt.strftime('%Y%m') @@ -98,18 +89,24 @@ def createDensityPlots(outf, calcdir, enddt, includeyear=True): return -def SyncRawData(outf, matchstart, matchend, shbucket, calcdir): +# Sync the raw camera data from shared storage to the local disk +# +def SyncRawData(outf, matchstart, matchend, shbucket): # camera data - no need to replicate it for an historical date - outf.write(f'targdirs=$(aws s3 ls {shbucket}/matches/RMSCorrelate/ | egrep -v "traj|daily|test|plot|proce"|grep PRE | awk \'{{print $2}}\')\n') + outf.write(f'targdirs=$(aws s3 ls {shbucket}/ | egrep -v "traj|daily|test|plot|proce|dbs"|grep PRE | awk \'{{print $2}}\')\n') outf.write('for td in $targdirs ; do\n') for d in range(matchend+1, matchstart+2): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) trgdy=thisdt.strftime('%Y%m%d') - outf.write(f' aws s3 sync {shbucket}/matches/RMSCorrelate/$td {calcdir}/$td --exclude "*" --include "${{td:0:6}}_{trgdy}*" --quiet\n') + outf.write(f' aws s3 sync {shbucket}/$td ./$td --exclude "*" --include "${{td:0:6}}_{trgdy}*" --quiet\n') outf.write('done\n') return +# +# Get a list of images that are used by the solutions +# + def gatherUsedImageList(outf, matchstart, matchend, shbucket): for d in range(matchend, matchstart+1): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) @@ -117,67 +114,88 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): mth = thisdt.month dy = thisdt.day trajloc = f'trajectories/{yr}/{yr}{mth:02d}/{yr}{mth:02d}{dy:02d}' - out_dir = '/home/ec2-user/data/distrib' + out_dir = '~/data/distrib' outf.write(f'python -c "from traj.pickleAnalyser import getAllImages;getAllImages(\'{trajloc}\', \'{out_dir}\');"\n') - outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt"\n') + outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt" --quiet\n') outf.write(f'rm {out_dir}/consumed_*.txt\n') return -def createExecConsolSh(matchstart, matchend, execconsolsh): - shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! - _, outpath, _ = getTrajsolverPaths() +# +# Create the bash script that consolidates the generated data and makes sure the website and shared area are updated + +def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): + + istest = True if istest.lower()=='true' else False + print(f'istest is {istest}') + + srcpath, shbucket, webbucket = getTrajsolverPaths(istest=istest) + csuser = os.getenv('SERVERUSERID', default='ec2-user') + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) + includeyear = False + if enddt.day == 30: + includeyear = True with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - # outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') + outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execConsol start\n') + outf.write(f'aws s3 sync {srcpath}/ ~/data/distrib/canddbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --quiet\n') - outf.write('python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json\n') - + # must run this before consolidation as the process updates the database to remove any missing + # trajectories ie not on disk. outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') - refreshTrajectories(outf, matchstart, matchend, outpath) + refreshTrajectories(outf, matchstart, matchend, shbucket) + + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/canddbs/ {calcdir}/dbs/\n') + + outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') + outf.write('logger -s -t execConsol creating density plots\n') - createDensityPlots(outf, calcdir, enddt) + createDensityPlots(outf, calcdir, enddt, includeyear) outf.write('logger -s -t execConsol pushing data back to S3\n') pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket) pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) outf.write('logger -s -t execConsol getting the image list\n') gatherUsedImageList(outf, matchstart, matchend, shbucket) - #outf.write('unset AWS_PROFILE\n') + outf.write('logger -s -t execConsol done\n') return +# +# Create a bash script to replot the density charts if needed + + +def createExecReplotSh(matchstart, matchend, execconsolsh, istest=''): + + istest = True if istest.lower()=='true' else False -def createExecReplotSh(matchstart, matchend, execconsolsh): shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! - _, outpath, _ = getTrajsolverPaths() + csuser = os.getenv('SERVERUSERID', default='ec2-user') + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + _, outpath, _ = getTrajsolverPaths(istest=istest) + enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - #outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execReplot start\n') refreshTrajectories(outf, matchstart, matchend, outpath) createDensityPlots(outf, calcdir, enddt, False) gatherUsedImageList(outf, matchstart, matchend, shbucket) - #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execReplot done\n') return -def createDistribMatchingSh(matchstart, matchend, execmatchingsh): - shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') +def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): + csuser = os.getenv('SERVERUSERID', default='ec2-user') startdt = datetime.datetime.now() + datetime.timedelta(days=-matchstart) enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) @@ -186,66 +204,63 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh): enddtstr = enddt.strftime('%Y%m%d-080000') rundatestr = enddt.strftime('%Y%m%d') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' - srcpath, outpath, _ = getTrajsolverPaths() + _, outpath, webpath = getTrajsolverPaths(istest=istest) + + srcpath = os.getenv('UKMONSHAREDBUCKET') + '/matches/RMSCorrelate' with open(execmatchingsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - #outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') outf.write(f'cd {calcdir}\n') outf.write('df -h . \n') # fetch anything thats new from S3 + outf.write('logger -s -t execdistrib syncing latest trajectories from shared S3\n') + # in test, we seeded the test area with prod trajectories in April 2026 refreshTrajectories(outf, matchstart, matchend, outpath) outf.write('logger -s -t execdistrib syncing the raw data from shared S3\n') - outf.write(f'aws s3 cp {srcpath}/processed_trajectories.json {calcdir}/processed_trajectories.json --quiet\n') - outf.write(f'ls -ltr {calcdir}/*.json\n') - - SyncRawData(outf, matchstart, matchend, shbucket, calcdir) + SyncRawData(outf, matchstart, matchend, srcpath) outf.write('logger -s -t execdistrib starting correlator to update existing matches and create candidates\n') - outf.write(f'mkdir -p {calcdir}/candidates\n') - outf.write(f'rm {calcdir}/candidates/*.pickle >/dev/null 2>&1\n') - outf.write(f'time python -m wmpl.Trajectory.CorrelateRMS {calcdir} -i 1 -l -r \"({startdtstr},{enddtstr})\"\n') + outf.write('mkdir -p ./candidates/processed\n') + outf.write('rm ./candidates/*.pickle >/dev/null 2>&1\n') + outf.write(f'time python -m wmpl.Trajectory.CorrelateRMS . --dbdir ./dbs --logdir ./logs --mcmode 4 -l -r \"({startdtstr},{enddtstr})\"\n') # backup the raw candidates in case i need to reprocess some by hand - outf.write(f'mkdir -p {calcdir}/candidates/bkp\n') - outf.write(f'tar cvfz {calcdir}/candidates/bkp/{rundatestr}.tgz {calcdir}/candidates/*.pickle\n') - outf.write(f'find {calcdir}/candidates/bkp/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') + outf.write(f'tar czf ./candidates/processed/{rundatestr}.tgz ./candidates/*.pickle\n') + outf.write('find ./candidates/processed/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') + outf.write('find ./logs/ -mtime +28 -exec rm -f ' + '{} \\;\n') outf.write('logger -s -t execdistrib backing up the database to trajdb\n') - outf.write(f'cp {calcdir}/processed_trajectories.json {calcdir}/trajdb/processed_trajectories.json.{rundatestr}\n') - - outf.write('logger -s -t execdistrib Syncing the database back to shared S3\n') - outf.write(f'if [ -s {calcdir}/processed_trajectories.json ] ; then\n') - outf.write(f'aws s3 cp {calcdir}/processed_trajectories.json {srcpath}/processed_trajectories.json --quiet\n') - outf.write('else echo "bad database file" ; fi \n') + outf.write(f'tar cvzf ./trajdb/databases_{rundatestr}.tgz dbs/observations.db dbs/trajectories.db dbs/candidates.db\n') + outf.write('find ./trajdb/ -name "*" -mtime +14 -exec rm -f ' + '{} \\;\n') outf.write('logger -s -t execdistrib distributing candidates and launching containers\n') - outf.write(f'time python -m traj.distributeCandidates {rundatestr} {calcdir}/candidates {srcpath}\n') + outf.write(f'time python -m traj.distributeCandidates {rundatestr} ./candidates {istest}\n') # do this again to fetch todays results outf.write('logger -s -t execdistrib refetch latest trajectories\n') refreshTrajectories(outf, matchstart, matchend, outpath) - - outf.write('logger -s -t execdistrib and sync the back to S3 as well\n') - pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket) - pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) + + outf.write('logger -s -t execdistrib and sync back to S3 as well\n') + pushUpdatedTrajectoriesShared(outf, matchstart, matchend, outpath) + pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath) - #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execdistrib done\n') if __name__ == '__main__': if len(sys.argv) < 4: - print('Usage: createDistribMatchingSh day1 day2 outfile') + print('Usage: createDistribMatchingSh day1 day2 outfile optional_istest') exit(1) matchstart = int(sys.argv[1]) matchend = int(sys.argv[2]) outfname = sys.argv[3] + if len(sys.argv) > 4: + istest = True if sys.argv[4].lower()=='true' else False - createDistribMatchingSh(matchstart, matchend, outfname) + createDistribMatchingSh(matchstart, matchend, outfname, istest) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 3030c9b08..9255534e9 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -3,6 +3,8 @@ # Used when running the correlator in 'distributed' mode. # Pick up new groups of candidate trajectories and distribute them, # then wait for completion and gather the logs. + +# NB this runs on the calculation engine server # import json @@ -16,13 +18,30 @@ import pickle -def getClusterDetails(templdir): - sts = boto3.client('sts') - accid = sts.get_caller_identity()['Account'] - if accid == '183798037734': - clusdetails = os.path.join(templdir, 'clusdetails-mda.txt') +# read the task template to determine the paths to write any data to +# +def getTrajsolverPaths(istest=False): + templdir,_ = os.path.split(__file__) + if istest: + taskjson = 'taskrunner_test.json' + else: + taskjson = 'taskrunner.json' + with open(os.path.join(templdir, taskjson), 'r') as inf: + taskdets = json.load(inf) + taskenv = taskdets['overrides']['containerOverrides'][0]['environment'] + srcpath = [x for x in taskenv if x['name']=='SRCPATH'][0]['value'] # path from which trajsolver will consume candidates + outpath = [x for x in taskenv if x['name']=='OUTPATH'][0]['value'] # path to which trajsolver will publish trajectories + webpath = [x for x in taskenv if x['name']=='WEBPATH'][0]['value'] # web loc to which trajsolver will publish trajectories + + return srcpath, outpath, webpath + + +def getClusterDetails(istest=False): + templdir,_ = os.path.split(__file__) + if istest: + clusdetails = os.path.join(templdir, 'clusdetailstest-ukmda.txt') else: - clusdetails = os.path.join(templdir, 'clusdetails-mm.txt') + clusdetails = os.path.join(templdir, 'clusdetails-mda.txt') with open(clusdetails) as inf: lis = inf.readlines() clusname = lis[0].strip() @@ -34,12 +53,16 @@ def getClusterDetails(templdir): return [clusname, secgrp, subnet, exrolearn, loggrp, contname] -def createTaskTemplate(rundate, buckname, clusdets, spandays=3): +def createTaskTemplate(rundate, buckname, clusdets, spandays=3, istest=False): d1 = (rundate + datetime.timedelta(days = -(spandays-1))).strftime('%Y%m%d') d2 = (rundate + datetime.timedelta(days = 1)).strftime('%Y%m%d') templdir,_ = os.path.split(__file__) - templatefile = os.path.join(templdir, 'taskrunner.json') + if istest: + taskjson = 'taskrunner_test.json' + else: + taskjson = 'taskrunner.json' + templatefile = os.path.join(templdir, taskjson) clusname = clusdets[0] secgrp = clusdets[1] subnet = clusdets[2] @@ -65,19 +88,29 @@ def getDebugStatus(): return False -def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): +def distributeCandidates(rundate, srcdir, maxcount=20, istest=False): + + clusdets = getClusterDetails(istest=istest) + # the target for distribution is the source for the trajsolver + targdir, _, _ = getTrajsolverPaths(istest=istest) clusname = clusdets[0] + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] + ecsclient = boto3.client('ecs', region_name='eu-west-2') status = ecsclient.describe_clusters(clusters=[clusname]) if len(status['clusters']) == 0: print('cluster not running!') return False + print(clusdets, targdir) + print(f'Reading from {srcdir}') # obtain a list of picklefiles and sort by name - flist = glob.glob1(srcdir, '*.pickle') + flist = glob.glob('*.pickle', root_dir=srcdir) if len(flist) == 0: print('no candidates to process') return False @@ -85,12 +118,10 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): # work out how many buckets i need numcands = len(flist) + print(f'processing {numcands} candidates') numbucks = int(math.ceil(numcands/maxcount)) # create buckets - targdir = targdir[5:] - outbucket=targdir[:targdir.find('/')] - targdir = targdir[targdir.find('/')+1:] buckroot = os.path.join(targdir, rundate.strftime('%Y%m%d')) taskarns = [None] * numbucks @@ -101,14 +132,14 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): s3 = boto3.client('s3') for i in range(0, numbucks): - bucknames[i] = buckroot + f'_{i:02d}' + bucknames[i] = buckroot + f'_{i:03d}' filelist = flist[i::numbucks] for fli in filelist: src = os.path.join(srcdir, fli) dst = os.path.join(bucknames[i], fli) s3.upload_file(src, outbucket, dst) - taskjson = createTaskTemplate(rundate, bucknames[i], clusdets) + taskjson = createTaskTemplate(rundate, bucknames[i], clusdets, istest=istest) response = ecsclient.run_task(**taskjson) while len(response['tasks']) == 0: @@ -117,7 +148,7 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): taskarn = response['tasks'][0]['taskArn'] taskarns[i] = taskarn jsontempls[i] = taskjson - print(taskarn[51:]) + print(taskarn.split('/')[-1]) if isDbg is True and i == 3: break @@ -143,23 +174,31 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): return True -def monitorProgress(rundate): +def monitorProgress(rundatestr, istest=''): + + istest = False if (istest=='' or istest.lower() == 'false') else True client = boto3.client('ecs', region_name='eu-west-2') s3 = boto3.client('s3') - archbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - templdir,_ = os.path.split(__file__) - clusdets = getClusterDetails(templdir) + clusdets = getClusterDetails(istest=istest) + targdir, _, _ = getTrajsolverPaths(istest=istest) + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] # load the buckets, tasks and cluster name from the dump file - rundate = datetime.datetime.strptime(rundate, '%Y%m%d') - picklefile = os.path.join(datadir, 'distrib', rundate.strftime('%Y%m%d') + '.pickle') - rempickle = f"matches/distrib/{rundate.strftime('%Y%m%d')}.pickle" + rundate = datetime.datetime.strptime(rundatestr, '%Y%m%d') + if istest: + picklefile = os.path.join(datadir, 'distrib', 'test', rundatestr + '.pickle') + else: + picklefile = os.path.join(datadir, 'distrib', rundatestr + '.pickle') + rempickle = f"{targdir}/{rundatestr}.pickle" + os.makedirs(os.path.join(datadir, 'distrib', 'test'), exist_ok=True) try: - s3.download_file(archbucket, rempickle, picklefile) + s3.download_file(outbucket, rempickle, picklefile) except: - print('no containers to monitor') + print(f'no containers to monitor in {outbucket}/{rempickle}') return dumpdata = pickle.load(open(picklefile,'rb')) bucknames = dumpdata[0] @@ -190,59 +229,49 @@ def monitorProgress(rundate): thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) + taskid = tsk["arn"].split('/')[-1] try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {tsk["arn"][51:]} completed already') + print(f'{thisbuck}: task {taskid} completed already') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + for tsk in sts['tasks']: - print(f'checking {tsk["taskArn"][51:]}') + taskid = tsk["taskArn"].split('/')[-1] idx = taskarns.index(tsk['taskArn']) + _, thisbuck = os.path.split(bucknames[idx]) + print(f'{thisbuck}: checking {taskid} ') #print(tsk['taskArn'], tsk['lastStatus']) if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': # retry the task - thisjson = createTaskTemplate(rundate, bucknames[idx]) + thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] - print('task restarted') + print(f'{thisbuck}: task restarted') else: thisarn=taskarns.pop(idx) thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print('task completed') + print(f'{thisbuck}: task completed') # collect the logs from CloudWatch - realfname=None - logdir = os.path.join(datadir, '..', 'logs', 'distrib') - os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + if len(taskarns) > 99: sts = client.describe_tasks(cluster=clusname, tasks=taskarns[99:199]) for tsk in sts['failures']: @@ -252,87 +281,92 @@ def monitorProgress(rundate): thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) + taskid = tsk["arn"].split('/')[-1] try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {tsk["arn"][51:]} completed already') + print(f'{thisbuck}: task {taskid} completed already') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + for tsk in sts['tasks']: - print(f'checking {tsk["taskArn"][51:]}') + taskid = tsk["taskArn"].split('/')[-1] idx = taskarns.index(tsk['taskArn']) + _, thisbuck = os.path.split(bucknames[idx]) + print(f'{thisbuck} : checking {taskid} ') if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': # retry the task - thisjson = createTaskTemplate(rundate, bucknames[idx]) + thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] - print('task restarted') + print(f'{thisbuck}: task restarted') else: thisarn=taskarns.pop(idx) thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print('task completed') - - # collect the logs from CloudWatch - realfname=None - logdir = os.path.join(datadir, '..', 'logs', 'distrib') - os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + print(f'{thisbuck}: task completed') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + if taskcount > 0: # wait 60s before checking again - print('sleeping for 60s') - time.sleep(60.0) + print('sleeping for 30s') + time.sleep(30.0) return -def getMissedLogs(picklefile): +def getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir): + # collect the logs from CloudWatch + realfname=None + logdir = os.path.join(datadir, '..', 'logs', 'distrib') + os.makedirs(logdir, exist_ok=True) + tmpfname = os.path.join(logdir, f'{thisarn.split("/")[-1]}.log') + with open(tmpfname, 'w') as outf: + for events in getLogDetails(loggrp, thisarn.split("/")[-1], contname): + for evt in events: + evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) + msg = evt['message'] + outf.write(f'{evtdt} {msg}\n') + if msg[:10] == 'processing': + realfname = msg[11:].strip() + if realfname: + locname = os.path.join(logdir, f'{realfname}.log') + os.rename(tmpfname, locname) + remlog = f'{targdir}/logs/{realfname}.log' + s3.upload_file(Filename=locname, Bucket=outbucket, Key=remlog) + return + + +def getMissedLogs(picklefile, istest=False): + + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) s3 = boto3.client('s3') - archbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] + _, targdir, _ = getTrajsolverPaths(istest=istest) + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] + + clusdets = getClusterDetails(istest=istest) + contname = clusdets[5] + dumpdata = pickle.load(open(picklefile,'rb')) taskarns = dumpdata[1] - loggrp = '/ecs/trajcont' - contname = 'trajcont' - logdir = '/home/ec2-user/prod/logs/distrib' + loggrp = '/ecs/trajcont' for thisarn in taskarns: - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): @@ -341,39 +375,43 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): """ client = boto3.client('logs', region_name=region_name) + logstreamname = f'ecs/{contname}/{thisarn}' # first request - response = client.get_log_events( - logGroupName=loggrp, - logStreamName=f'ecs/{contname}/{thisarn}', - startFromHead=True) - yield response['events'] - - # second and later - while True: - prev_token = response['nextForwardToken'] + #print(f'looking for {logstreamname} in {loggrp}') + try: response = client.get_log_events( logGroupName=loggrp, - logStreamName=f'ecs/{contname}/{thisarn}', - nextToken=prev_token) - # same token then break - if response['nextForwardToken'] == prev_token: - break + logStreamName=logstreamname, + startFromHead=True) yield response['events'] + # second and later + while True: + prev_token = response['nextForwardToken'] + response = client.get_log_events( + logGroupName=loggrp, + logStreamName=logstreamname, + nextToken=prev_token) + # same token then break + if response['nextForwardToken'] == prev_token: + break + yield response['events'] + except Exception: + dtval = int(datetime.datetime.now().timestamp() * 1000) + msg = f'log for {thisarn} not available' + yield [{'timestamp': dtval, 'message': msg}] + if __name__ == '__main__': - if len(sys.argv) < 4: - rundt = datetime.datetime(2022,4,21) - srcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates' # hardcoded on calcserver - buck = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - targdir = f'{buck}/matches/distrib' + if len(sys.argv) < 3: + print('usage: python distributeCandidates yyyymmdd ./candidates istest\n') + print(' optional istest if True then test mode used') else: rundt = datetime.datetime.strptime(sys.argv[1], '%Y%m%d') srcdir = sys.argv[2] - targdir = sys.argv[3] - - templdir,_ = os.path.split(__file__) - clusdets = getClusterDetails(templdir) - print(clusdets) - distributeCandidates(rundt, srcdir, targdir, clusdets) + if len(sys.argv)>3: + istest = True if sys.argv[3].lower()=='true' else False + else: + istest = False + distributeCandidates(rundt, srcdir, istest=istest) diff --git a/archive/ukmon_pylib/traj/jsonDbMaintenance.py b/archive/ukmon_pylib/traj/jsonDbMaintenance.py deleted file mode 100644 index e5aa984c3..000000000 --- a/archive/ukmon_pylib/traj/jsonDbMaintenance.py +++ /dev/null @@ -1,72 +0,0 @@ -# small script to archive off old data from the JSON database -import datetime -import os -import sys -from dateutil.relativedelta import relativedelta - -from wmpl.Utils.TrajConversions import datetime2JD, jd2Date -from wmpl.Trajectory.CorrelateRMS import DatabaseJSON - -# Name of json file with the list of processed directories -JSON_DB_NAME = "processed_trajectories.json" - - -def archiveOldRecords(db, db_dir, older_than=3): - """ - Archive off old records to keep the database size down - - Keyword Arguments: - older_than: [int] number of months to keep, default 3 - """ - class DummyMetObs(): - def __init__(self, station, obs_id): - self.station_code = station - self.id = obs_id - - archdate = datetime.datetime.now(datetime.timezone.utc) - relativedelta(months=older_than) - archdate_jd = datetime2JD(archdate) - - arch_db_path = os.path.join(db_dir, 'archive', f'{archdate.strftime("%Y%m")}_{JSON_DB_NAME}') - os.makedirs(os.path.join(db_dir, 'archive'), exist_ok=True) - archdb = DatabaseJSON(arch_db_path) - - for traj in [t for t in db.trajectories if t < archdate_jd]: - if traj < archdate_jd: - archdb.addTrajectory(None, db.trajectories[traj], False) - db.removeTrajectory(db.trajectories[traj]) - - for traj in [t for t in db.failed_trajectories if t < archdate_jd]: - if traj < archdate_jd: - archdb.addTrajectory(None, db.failed_trajectories[traj], True) - db.removeTrajectory(db.failed_trajectories[traj]) - - for station in db.processed_dirs: - arch_processed = [dirname for dirname in db.processed_dirs[station] if - datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for dirname in arch_processed: - archdb.addProcessedDir(station, dirname) - db.processed_dirs[station].remove(dirname) - - for station in db.paired_obs: - arch_processed = [obs_id for obs_id in db.paired_obs[station] if - datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for obs_id in arch_processed: - archdb.addPairedObservation(DummyMetObs(station, obs_id)) - db.paired_obs[station].remove(obs_id) - - archdb.save() - db.save() - return - - -if __name__ == '__main__': - db_dir = sys.argv[1] - database_path = os.path.join(db_dir, JSON_DB_NAME) - db = DatabaseJSON(database_path) - soonest = jd2Date(min(min(list(db.trajectories.keys())), min(list(db.failed_trajectories.keys()))), dt_obj=True) - - nowdt = datetime.datetime.now() - mthsback = int((nowdt - soonest).days/30) - for i in range(mthsback, 2, -1): - print(f'archiving {i} months back') - archiveOldRecords(db, db_dir, i) diff --git a/archive/ukmon_pylib/traj/manualBulkReruns.py b/archive/ukmon_pylib/traj/manualBulkReruns.py index d94510575..98dd19b98 100644 --- a/archive/ukmon_pylib/traj/manualBulkReruns.py +++ b/archive/ukmon_pylib/traj/manualBulkReruns.py @@ -24,9 +24,10 @@ from wmpl.Utils.TrajConversions import geo2Cartesian from wmpl.Utils.TrajConversions import raDec2ECI from wmpl.Utils.Math import vectNorm, angleBetweenVectors, vectorFromPointDirectionAndAngle +from wmpl.Utils.TrajConversions import raDec2AltAz, altAz2RADec, datetime2JD +from wmpl.Utils.Earth import greatCircleDistance -from meteortools.utils import raDec2AltAz, altAz2RADec, datetime2JD, greatCircleDistance -from meteortools.fileformats import loadFTPDetectInfo, writeNewFTPFile +from utils.ftpDetectInfo import loadFTPDetectInfo, writeNewFTPFile # Dummy platepar structure needed by loadPlatePar @@ -199,7 +200,7 @@ def getOverlappingStations(datadir, stationid): checker = EventChecker(traj_constraints=trajcons) ppdir = os.path.join(datadir, 'consolidated','platepars') rp = loadPlatepar(ppdir, stationid) - ppfs = glob.glob1(ppdir, '*.json') + ppfs = glob.glob('*.json', root_dir=ppdir) overlaps=[] for ppf in ppfs: ppfname, _ = os.path.splitext(ppf) @@ -406,7 +407,7 @@ def prepDataForRange(startdt, enddt): # working out what has been used, then collecting the FTPs and platepars, # and finally removing the already-used meteors from the FTPs # - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) sdt = datetime.datetime.strptime(startdt, '%Y%m') edt = datetime.datetime.strptime(enddt, '%Y%m') smth = sdt.month @@ -430,7 +431,7 @@ def prepDataForRange(startdt, enddt): def moveInterestingCands(): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) pickledir = os.path.join(datadir, 'historic', 'candidates') goodcandr = os.path.join(datadir, 'historic', 'goodcands') os.makedirs(goodcandr, exist_ok=True) diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index ddfe6fbc6..16f1fd4b4 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -14,12 +14,10 @@ import datetime import json +from utils.convertSolLon import sollon2jd +from traj.ShowerAssociation import associateShower + from wmpl.Utils.TrajConversions import jd2Date -try: - from meteortools.utils import sollon2jd - from traj.ShowerAssociation import associateShower -except: - pass from wmpl.Utils.Math import mergeClosePoints, angleBetweenSphericalCoords from wmpl.Utils.Physics import calcMass from wmpl.Utils.Pickling import loadPickle @@ -81,7 +79,7 @@ def getBestView(picklename): except Exception: pass beststatid = statids[vmags.index(bestvmag)] - imgfn = glob.glob1(outdir, '*{}*.jpg'.format(beststatid)) + imgfn = glob.glob(f'*{beststatid}*.jpg', root_dir=outdir) if len(imgfn) > 0: bestimg = imgfn[0] elif os.path.isfile(os.path.join(outdir, 'jpgs.lst')): @@ -122,6 +120,8 @@ def getShowerDets(shwr): def createUFOOrbitFile(traj, outdir, amag, mass, shower_obj): + outdir = os.path.expanduser(outdir) + #print('Creating UFO Orbit style output file') orb = traj.orbit if shower_obj is None: @@ -378,6 +378,7 @@ def computeAbsoluteMagnitudes(traj, meteor_list): def draw3Dmap(traj, outdir): + outdir = os.path.expanduser(outdir) lats = [] lons = [] alts = [] @@ -493,6 +494,8 @@ def calcAdditionalValues(traj): def createAdditionalOutput(traj, outdir): + outdir = os.path.expanduser(outdir) + # calculate the values amag, vmag, mass, id, cod, shwrname, orb, shower_obj, lg, bg, vg, _ = calcAdditionalValues(traj) @@ -582,17 +585,20 @@ def getListOfImages(picklename): class TrajQualityParams(object): def __init__(self): - self. min_traj_points = 6 - self. min_qc = 5.0 - self. max_e = 1.5 - self. max_radiant_err = 2.0 - self. max_vg_err = 10.0 - self. max_begin_ht = 160 - self. min_end_ht = 20 - - -def getAllImages(dir_path, out_path): + self.min_traj_points = 4 + self.min_qc = 3.0 + self.max_e = 2.5 + self.max_radiant_err = 5.0 + self.max_vg_err = 20.0 + self.max_vg = 120.0 + self.max_begin_ht = 160 + self.min_end_ht = 0 + + +def getAllImages(dir_path, outdir): + outdir = os.path.expanduser(outdir) traj_quality_params = TrajQualityParams() + trajs = loadTrajectoryPickles(dir_path, traj_quality_params, verbose=True) imglist = [] for traj in trajs: @@ -604,6 +610,6 @@ def getAllImages(dir_path, out_path): except Exception: pass outfname = os.path.split(dir_path)[1] - with open(os.path.join(out_path, f'consumed_{outfname}.txt'), 'w') as outf: + with open(os.path.join(outdir, f'consumed_{outfname}.txt'), 'w') as outf: for img in imglist: outf.write(f'{img}\n') diff --git a/archive/ukmon_pylib/traj/taskrunner.json b/archive/ukmon_pylib/traj/taskrunner.json index 630b61384..be99acf10 100644 --- a/archive/ukmon_pylib/traj/taskrunner.json +++ b/archive/ukmon_pylib/traj/taskrunner.json @@ -29,6 +29,10 @@ { "name": "WEBPATH", "value": "s3://ukmda-website/reports" + }, + { + "name": "MAX_STNS", + "value": "9999" } ] } diff --git a/archive/ukmon_pylib/traj/taskrunner_test.json b/archive/ukmon_pylib/traj/taskrunner_test.json new file mode 100644 index 000000000..d834b6e2f --- /dev/null +++ b/archive/ukmon_pylib/traj/taskrunner_test.json @@ -0,0 +1,48 @@ +{ + "cluster": "trajsolvertest", + "count": 1, + "enableECSManagedTags": true, + "enableExecuteCommand": true, + "group": "trajsolvergrp", + "launchType": "FARGATE", + "networkConfiguration": { + "awsvpcConfiguration": { + "subnets":[], + "securityGroups": [], + "assignPublicIp": "ENABLED" + } + }, + "overrides": { + "containerOverrides": [ + { + "name": "trajconttest", + "command": [], + "environment": [ + { + "name": "SRCPATH", + "value": "s3://ukmda-shared/test/matches/distrib" + }, + { + "name": "OUTPATH", + "value": "s3://ukmda-shared/test/matches/RMSCorrelate" + }, + { + "name": "WEBPATH", + "value": "s3://ukmda-website/dummy" + }, + { + "name": "MAX_STNS", + "value": "9999" + } + ] + } + ], + "executionRoleArn": "", + "taskRoleArn": "" + }, + "platformVersion": "LATEST", + "propagateTags": "TASK_DEFINITION", + "referenceId": "", + "startedBy": "cli", + "taskDefinition": "trajsolvertest" +} diff --git a/archive/ukmon_pylib/utils/__init__.py b/archive/ukmon_pylib/utils/__init__.py new file mode 100644 index 000000000..1d590cf3f --- /dev/null +++ b/archive/ukmon_pylib/utils/__init__.py @@ -0,0 +1 @@ +# Copyright (C) 2018- Mark McIntyre diff --git a/archive/ukmon_pylib/utils/analyseGmnData.py b/archive/ukmon_pylib/utils/analyseGmnData.py new file mode 100644 index 000000000..749216f3c --- /dev/null +++ b/archive/ukmon_pylib/utils/analyseGmnData.py @@ -0,0 +1,105 @@ +# simple scripts to analyse the GMN data in python + +import pandas as pd +import os +import sys +import datetime + +from converters.gmnTxtToPandas import dirpath + + +def analyseAMonth(fulldf, yr, mth): + if mth != 0: + beg = datetime.datetime(yr, mth, 1) + if mth < 12: + end = datetime.datetime(yr, mth+1, 1) + else: + end = datetime.datetime(yr+1, 1, 1) + mthdf = fulldf[fulldf.utc_beg >= beg] + mthdf = mthdf[mthdf.utc_beg < end] + else: + mthdf = fulldf + tot_num = len(mthdf) + uktraj = mthdf[mthdf.stats.str.contains('UK')] + num_uk_traj = len(uktraj) + allstats = ','.join(uktraj.stats) + stats = list(set(allstats.split(','))) + ukstats = [x for x in stats if 'UK' in x] + otherstats = [x for x in stats if 'UK' not in x] + distinctuk = [x for x in ukstats if '_' not in x] + distinctother = [x for x in otherstats if '_' not in x] + ctrys = list(set([x[:2] for x in otherstats])) + otherctrydf = uktraj[uktraj.stats.str.contains('|'.join(ctrys))] + if mth == 0: + uktraj.to_csv('uk-trajectories.csv', index=False) + return len(distinctuk), len(distinctother), num_uk_traj, tot_num, len(otherctrydf) + + +def findDuplicatesById(yr, mth=None): + if mth: + datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') + else: + datafile = os.path.join(dirpath, 'parquet', f'gmn_{yr:04d}.parquet.snap') + df = pd.read_parquet(datafile) + df['dupe']=df.duplicated(subset=['id']) + dupeids = df[df.dupe].sort_values(by=['id']).id + duperows = df[df.id.isin(dupeids)] + print(duperows) + print(len(df)) + return duperows + + +def atleastOneStation(stats,statsnext): + if stats is None or statsnext is None: + return False + stats = stats.split(',') + statsnext = statsnext.split(',') + return any(i in statsnext for i in stats) + + +def findDuplicatesByStatAndJD(yr, mth=None, df=None): + if not df: + dirpath = '.' + if mth: + datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') + else: + datafile = os.path.join(dirpath, 'parquet', f'gmn_{yr:04d}.parquet.snap') + df = pd.read_parquet(datafile) + df['statsnext'] = df.stats.shift(-1) + df['jd_next'] = df.jd_beg.shift(-1) + df['Lat1_next'] = df.Lat1.shift(-1) + df['Lon1_next'] = df.Lon1.shift(-1) + nearjd = df.query('abs(jd_beg - jd_next) < (0.5/86400)') + nearlat = nearjd.query('abs(Lat1 - Lat1_next) < 1 and abs(Lon1 - Lon1_next) < 1') + samestats = nearlat.query('stats == statsnext') + nearlat['overlapstats'] = nearlat.apply(lambda row: atleastOneStation(row.stats, row.statsnext), axis=1) + commonstats = nearlat[nearlat.overlapstats] + print(f'there are {len(df)} trajectories in the period') + print(f'there are {len(nearjd)} ({round(100*len(nearjd)/len(df),2)}%) events within 1s') + print(f'there are {len(commonstats)} ({round(100*len(commonstats)/len(df),2)}%) with at least one common station') + print(f'there are {len(samestats)} ({round(100*len(samestats)/len(df),2)}%) with the same stations') + return nearjd, commonstats, samestats + + +if __name__ == '__main__': + yr = int(sys.argv[1]) + fulldf = None + data = [] + for mth in range(1,13): + print(f'processing month {mth}') + df = pd.read_parquet(f'parquet/monthly/gmn_{yr}{mth:02d}.parquet.snap') + cams, othercams, traj, totaltraj, othertraj = analyseAMonth(df, yr, mth) + data.append([mth, cams, othercams, traj, totaltraj, othertraj]) + fulldf = df if fulldf is None else pd.concat([fulldf, df]) + + result = pd.DataFrame(data, columns=['Mth','Camcount','OtherCams','TrajCount','TotalTraj','NonUKTraj']) + tmpres = result.set_index('Mth') + plot = tmpres.plot(title='UK Statistics') + fig = plot.get_figure() + fig.savefig(f'uk-analysis-{yr}.jpg') + + cams, othercams, traj, totaltraj, otherctry, = analyseAMonth(fulldf, yr, 0) + data.append([0, cams, othercams, traj, totaltraj, otherctry]) + + result = pd.DataFrame(data, columns=['Mth','Camcount','OtherCams','TrajCount','TotalTraj','NonUKTraj']) + result.to_csv('2025-stats.csv', index=False) diff --git a/archive/ukmon_pylib/analysis/compareBrightnessData.py b/archive/ukmon_pylib/utils/compareBrightnessData.py similarity index 91% rename from archive/ukmon_pylib/analysis/compareBrightnessData.py rename to archive/ukmon_pylib/utils/compareBrightnessData.py index 1ace683cb..663fb9af3 100644 --- a/archive/ukmon_pylib/analysis/compareBrightnessData.py +++ b/archive/ukmon_pylib/utils/compareBrightnessData.py @@ -10,9 +10,7 @@ def getBrightnessData(yyyymmdd, outdir=None): - prof = os.getenv('UKMPROFILE',default='ukmonshared') - sess = boto3.Session(profile_name=prof) - ddb = sess.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table('LiveBrightness') resp = table.query(KeyConditionExpression=Key('CaptureNight').eq(yyyymmdd)) df = pd.DataFrame(ddbjson.loads(resp['Items'])) diff --git a/archive/ukmon_pylib/utils/convertSolLon.py b/archive/ukmon_pylib/utils/convertSolLon.py new file mode 100644 index 000000000..31e4f762c --- /dev/null +++ b/archive/ukmon_pylib/utils/convertSolLon.py @@ -0,0 +1,41 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +import numpy as np +from wmpl.Utils.TrajConversions import date2JD + + +def sollon2jd(Year, Month, Long): + """ + Calculate the julian date corresponding to a solar longitude. + Because Solar Longitude is relative to the Spring equinox, the exact date + of a given LS varies from year to year. + + Parameters: + Year: [int] year you wish to calculate in. + Month: [int] month you wish to calculate in. + Long: [float] The solar longitude to convert. + + Returns: + [float] julian date + + Notes: + The function is only stable for date ranges from 1900-2100. + """ + + Long = np.radians(Long) + N = Year - 2000 + if abs(N) > 100: + print("Algorithm is not stable for years below 1900 or above 2100") + + JDM0 = 2451182.24736 + 365.25963575 * N + ApproxJD = date2JD(Year, Month, 15, 12, 0, 0) + DiffJD = ApproxJD-2451545 + + Dt = 1.94330 * np.sin(Long - 1.798135) + 0.01305272 * np.sin(2*Long + 2.634232) + 78.195268 + 58.13165 * Long - 0.0000089408 * DiffJD + + if abs(ApproxJD - (JDM0 + Dt))>50: + Dt = Dt + 365.2596 + + JD1 = JDM0 + Dt + + return JD1 diff --git a/archive/ukmon_pylib/utils/ftpDetectInfo.py b/archive/ukmon_pylib/utils/ftpDetectInfo.py new file mode 100644 index 000000000..0e78d1300 --- /dev/null +++ b/archive/ukmon_pylib/utils/ftpDetectInfo.py @@ -0,0 +1,506 @@ +# +# Load an FTPdetectInfo file - copied from RMS +# +# Copyright (C) 2018-2023 Mark McIntyre + +import os +import numpy as np +import configparser as crp +import json +import datetime + +from wmpl.Utils.TrajConversions import date2JD +from wmpl.Utils.Math import angleBetweenSphericalCoords + + +def filterFTPforSpecificTime(ftpfile, dtstr): + """ filter FTPdetect file for a specific event, by time, and copy it into a new file + + Arguments: + ftpfile - [string] full path to the ftpdetect file to filter + dtstr - [string] date/time of required event in yyyymmdd_hhmmss format + + Returns: + tuple containing + full name of the new file containing just matching events + the number of matching events + """ + meteor_list = loadFTPDetectInfo(ftpfile, time_offsets=None, join_broken_meteors=True, locdata=None) + refdt = datetime.datetime.strptime(dtstr, '%Y%m%d_%H%M%S') + #print(refdt) + new_met_list = [] + for met in meteor_list: + #print(met.ff_name) + dtpart = datetime.datetime.strptime(met.ff_name[10:25], '%Y%m%d_%H%M%S') + tdiff = (refdt - dtpart).seconds + #print(tdiff) + if abs(tdiff) < 21: + print('adding one entry') + new_met_list.append(met) + newname = writeNewFTPFile(ftpfile, new_met_list) + return newname, len(new_met_list) + + +def writeNewFTPFile(srcname, metlist): + """ creates a FTPDetect file from a list of MeteorObservation objects + + Arguments: + srcname - [string] full path to the original FTP file + metlist - list of MeteorObservation objects + + Returns: + the full path to the created file + + """ + outdir, fname = os.path.split(srcname) + newname = os.path.join(outdir, f'{fname}.old') + try: + os.rename(srcname, newname) + except: + pass + if os.path.isfile(srcname): + srcname = srcname[:-4] + '_new.txt' + with open(srcname, 'w') as ftpf: + _writeFTPHeader(ftpf, len(metlist), outdir, False) + metno = 1 + ffname = '' + for met in metlist: + if ffname == met.ff_name: + metno = metno + 1 + else: + metno = 1 + ffname = met.ff_name + _writeOneMeteor(ftpf, metno, met.station_id, met.time_data, len(met.frames), met.fps, met.frames, + np.degrees(met.ra_data), np.degrees(met.dec_data), + np.degrees(met.azim_data), np.degrees(met.elev_data), + None, met.mag_data, False, met.x_data, met.y_data, met.ff_name) + return srcname + + +def loadFTPDetectInfo(ftpdetectinfo_file_name, time_offsets=None, + join_broken_meteors=True, locdata=None): + """ Loads an FTPDEtect file into a list of MeteorObservation objects + + Arguments: + ftpdetectinfo_file_name: [str] Path to the FTPdetectinfo file. + stations: [dict] A dictionary where the keys are stations IDs, and values are lists of: + - latitude +N in radians + - longitude +E in radians + - height in meters + + + Keyword arguments: + time_offsets: [dict] (key, value) pairs of (stations_id, time_offset) for every station. None by + default. + join_broken_meteors: [bool] Join meteors broken across 2 FF files. + + + Return: + meteor_list: [list] A list of MeteorObservation objects filled with data from the FTPdetectinfo file. + + """ + stations={} + if locdata is None: + dirname, fname = os.path.split(ftpdetectinfo_file_name) + cfgfile = os.path.join(dirname, '.config') + cfg = crp.ConfigParser() + cfg.read(cfgfile) + try: + lat = float(cfg['System']['latitude'].split()[0]) + lon = float(cfg['System']['longitude'].split()[0]) + height = float(cfg['System']['elevation'].split()[0]) + except: + # try reading from platepars file + ppf = os.path.join(dirname, 'platepars_all_recalibrated.json') + if not os.path.isfile(ppf): + return [] + js = json.load(open(ppf, 'r')) + if len(js) < 10: + return [] + lat = js[list(js.keys())[0]]['lat'] + lon = js[list(js.keys())[0]]['lon'] + height = js[list(js.keys())[0]]['elev'] + statid= fname.split('_')[1] + else: + statid = locdata['station_code'] + lat = float(locdata['lat']) + lon = float(locdata['lon']) + height = float(locdata['elev']) + + stations[statid] = [np.radians(lat), np.radians(lon), height*1000] + meteor_list = [] + + with open(ftpdetectinfo_file_name) as f: + # Skip the header + for i in range(11): + next(f) + + current_meteor = None + + bin_name = False + cal_name = False + meteor_header = False + + for line in f: + # Skip comments + if line.startswith("#"): + continue + + line = line.replace('\n', '').replace('\r', '') + + # Skip the line if it is empty + if not line: + continue + + if '-----' in line: + # Mark that the next line is the bin name + bin_name = True + + # If the separator is read in, save the current meteor + if current_meteor is not None: + current_meteor._finish() + meteor_list.append(current_meteor) + continue + + if bin_name: + bin_name = False + + # Mark that the next line is the calibration file name + cal_name = True + + # Save the name of the FF file + ff_name = line + + # Extract the reference time from the FF bin file name + line = line.split('_') + + # Count the number of string segments, and determine if it the old or new CAMS format + if len(line) == 6: + sc = 1 + else: + sc = 0 + + ff_date = line[1 + sc] + ff_time = line[2 + sc] + milliseconds = line[3 + sc] + + year = ff_date[:4] + month = ff_date[4:6] + day = ff_date[6:8] + + hour = ff_time[:2] + minute = ff_time[2:4] + seconds = ff_time[4:6] + + year, month, day, hour, minute, seconds, milliseconds = map(int, [year, month, day, hour, + minute, seconds, milliseconds]) + + # Calculate the reference JD time + jdt_ref = date2JD(year, month, day, hour, minute, seconds, milliseconds) + continue + + if cal_name: + cal_name = False + # Mark that the next line is the meteor header + meteor_header = True + continue + + if meteor_header: + meteor_header = False + line = line.split() + + # Get the station ID and the FPS from the meteor header + station_id = line[0].strip() + fps = float(line[3]) + + # Try converting station ID to integer + try: + station_id = int(station_id) + except: + pass + + # If the time offsets were given, apply the correction to the JD + if time_offsets is not None: + if station_id in time_offsets: + print('Applying time offset for station {:s} of {:.2f} s'.format(str(station_id), + time_offsets[station_id])) + + jdt_ref += time_offsets[station_id]/86400.0 + else: + print('Time offset for given station not found!') + + # Get the station data + if station_id in stations: + lat, lon, height = stations[station_id] + else: + print('ERROR! No info for station ', station_id, ' found in CameraSites.txt file!') + print('Exiting...') + break + # Init a new meteor observation + current_meteor = MeteorObservation(jdt_ref, station_id, lat, lon, height, fps, + ff_name=ff_name) + continue + + # Read in the meteor observation point + if (current_meteor is not None) and (not bin_name) and (not cal_name) and (not meteor_header): + + line = line.replace('\n', '').split() + + # Read in the meteor frame, RA and Dec + frame_n = float(line[0]) + x = float(line[1]) + y = float(line[2]) + ra = float(line[3]) + dec = float(line[4]) + azim = float(line[5]) + elev = float(line[6]) + + # Read the visual magnitude, if present + if len(line) > 8: + mag = line[8] + if mag == 'inf': + mag = None + else: + mag = float(mag) + else: + mag = None + + # Add the measurement point to the current meteor + current_meteor._addPoint(frame_n, x, y, azim, elev, ra, dec, mag) + + # Add the last meteor the the meteor list + if current_meteor is not None: + current_meteor._finish() + meteor_list.append(current_meteor) + + # Concatenate observations across different FF files ### + if join_broken_meteors: + + # Go through all meteors and compare the next observation + merged_indices = [] + for i in range(len(meteor_list)): + + # If the next observation was merged, skip it + if (i + 1) in merged_indices: + continue + + # Get the current meteor observation + met1 = meteor_list[i] + + if i >= (len(meteor_list) - 1): + break + + # Get the next meteor observation + met2 = meteor_list[i + 1] + + # Compare only same station observations + if met1.station_id != met2.station_id: + continue + + # Extract frame number + met1_frame_no = int(met1.ff_name.split("_")[-1].split('.')[0]) + met2_frame_no = int(met2.ff_name.split("_")[-1].split('.')[0]) + + # Skip if the next FF is not exactly 256 frames later + if met2_frame_no != (met1_frame_no + 256): + continue + + + # Check for frame continouty + if (met1.frames[-1] < 254) or (met2.frames[0] > 2): + continue + + # Check if the next frame is close to the predicted position + + # Compute angular distance between the last 2 points on the first FF + ang_dist = angleBetweenSphericalCoords(met1.dec_data[-2], met1.ra_data[-2], met1.dec_data[-1], + met1.ra_data[-1]) + + # Compute frame difference between the last frame on the 1st FF and the first frame on the 2nd FF + df = met2.frames[0] + (256 - met1.frames[-1]) + + # Skip the pair if the angular distance between the last and first frames is 2x larger than the + # frame difference times the expected separation + ang_dist_between = angleBetweenSphericalCoords(met1.dec_data[-1], met1.ra_data[-1], + met2.dec_data[0], met2.ra_data[0]) + + if ang_dist_between > 2*df*ang_dist: + continue + + # If all checks have passed, merge observations ### + # Recompute the frames + frames = 256.0 + met2.frames + + # Recompute the time data + time_data = frames/met1.fps + + # Add the observations to first meteor object + met1.frames = np.append(met1.frames, frames) + met1.time_data = np.append(met1.time_data, time_data) + met1.x_data = np.append(met1.x_data, met2.x_data) + met1.y_data = np.append(met1.y_data, met2.y_data) + met1.azim_data = np.append(met1.azim_data, met2.azim_data) + met1.elev_data = np.append(met1.elev_data, met2.elev_data) + met1.ra_data = np.append(met1.ra_data, met2.ra_data) + met1.dec_data = np.append(met1.dec_data, met2.dec_data) + met1.mag_data = np.append(met1.mag_data, met2.mag_data) + + # Merge the FF file name and create a list + if (met1.ff_name is not None) and (met2.ff_name is not None): + met1.ff_name = met1.ff_name + ',' + met2.ff_name + + # Sort all observations by time + met1._finish() + + # Indicate that the next observation is to be skipped + merged_indices.append(i + 1) + + # Removed merged meteors from the list + meteor_list = [element for i, element in enumerate(meteor_list) if i not in merged_indices] + + return meteor_list + + +class MeteorObservation(object): + """ Container for meteor observations. + """ + def __init__(self, jdt_ref, station_id, latitude, longitude, height, fps, ff_name=None): + """ Construct the MeteorObservation object. + Arguments: + jdt_ref: [float] Reference Julian date when the relative time is t = 0s. + station_id: [str] Station ID. + latitude: [float] Latitude +N in radians. + longitude: [float] Longitude +E in radians. + height: [float] Elevation above sea level (MSL) in meters. + fps: [float] Frames per second. + + Keyword arguments: + ff_name: [str] Name of the originating FF file. + """ + self.jdt_ref = jdt_ref + self.station_id = station_id + self.latitude = latitude + self.longitude = longitude + self.height = height + self.fps = fps + self.ff_name = ff_name + self.frames = [] + self.time_data = [] + self.x_data = [] + self.y_data = [] + self.azim_data = [] + self.elev_data = [] + self.ra_data = [] + self.dec_data = [] + self.mag_data = [] + self.abs_mag_data = [] + + def _addPoint(self, frame_n, x, y, azim, elev, ra, dec, mag): + """ Adds the measurement point to the meteor. + + Arguments: + frame_n: [flaot] Frame number from the reference time. + x: [float] X image coordinate. + y: [float] X image coordinate. + azim: [float] Azimuth, J2000 in degrees. + elev: [float] Elevation angle, J2000 in degrees. + ra: [float] Right ascension, J2000 in degrees. + dec: [float] Declination, J2000 in degrees. + mag: [float] Visual magnitude. + """ + self.frames.append(frame_n) + + # Calculate the time in seconds w.r.t. to the reference JD + point_time = float(frame_n)/self.fps + self.time_data.append(point_time) + self.x_data.append(x) + self.y_data.append(y) + + # Angular coordinates converted to radians + self.azim_data.append(np.radians(azim)) + self.elev_data.append(np.radians(elev)) + self.ra_data.append(np.radians(ra)) + self.dec_data.append(np.radians(dec)) + self.mag_data.append(mag) + + def _finish(self): + """ When the initialization is done, convert data lists to numpy arrays. """ + self.frames = np.array(self.frames) + self.time_data = np.array(self.time_data) + self.x_data = np.array(self.x_data) + self.y_data = np.array(self.y_data) + self.azim_data = np.array(self.azim_data) + self.elev_data = np.array(self.elev_data) + self.ra_data = np.array(self.ra_data) + self.dec_data = np.array(self.dec_data) + self.mag_data = np.array(self.mag_data) + # Sort by frame + temp_arr = np.c_[self.frames, self.time_data, self.x_data, self.y_data, self.azim_data, + self.elev_data, self.ra_data, self.dec_data, self.mag_data] + temp_arr = temp_arr[np.argsort(temp_arr[:, 0])] + self.frames, self.time_data, self.x_data, self.y_data, self.azim_data, self.elev_data, self.ra_data, \ + self.dec_data, self.mag_data = temp_arr.T + + +def _writeFTPHeader(ftpf, metcount, fldr, ufo=True): + """ + Internal function to create the header of the FTPDetect file + """ + l1 = 'Meteor Count = {:06d}\n'.format(metcount) + ftpf.write(l1) + ftpf.write('-----------------------------------------------------\n') + if ufo is True: + ftpf.write('Processed with UFOAnalyser\n') + else: + ftpf.write('Processed with RMS 1.0\n') + ftpf.write('-----------------------------------------------------\n') + l1 = 'FF folder = {:s}\n'.format(fldr) + ftpf.write(l1) + l1 = 'CAL folder = {:s}\n'.format(fldr) + ftpf.write(l1) + ftpf.write('-----------------------------------------------------\n') + ftpf.write('FF file processed\n') + ftpf.write('CAL file processed\n') + ftpf.write('Cam# Meteor# #Segments fps hnr mle bin Pix/fm Rho Phi\n') + ftpf.write('Per segment: Frame# Col Row RA Dec Azim Elev Inten Mag\n') + + +def _writeOneMeteor(ftpf, metno, sta, evttime, fcount, fps, fno, ra, dec, az, alt, b, mag, + ufo=True, x=None, y=None, ffname = None): + """ Internal function to write one meteor event into the file in FTPDetectInfo style + """ + ftpf.write('-------------------------------------------------------\n') + if ffname is None: + ms = '{:03d}'.format(int(evttime.microsecond / 1000)) + + fname = 'FF_' + sta + '_' + evttime.strftime('%Y%m%d_%H%M%S_') + ms + '_0000000.fits\n' + else: + fname = ffname + '\n' + ftpf.write(fname) + + if ufo is True: + ftpf.write('UFO UKMON DATA Recalibrated on: ') + else: + ftpf.write('RMS data reprocessed on: ') + ftpf.write(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f UTC\n')) + li = f'{sta} {metno:04d} {fcount:04d} {fps:04.2f} 000.0 000.0 00.0 000.0 0000.0 0000.0\n' + ftpf.write(li) + + for i in range(len(fno)): + # 204.4909 0422.57 0353.46 262.3574 +16.6355 267.7148 +23.0996 000120 3.41 + bri = 0 + if b is not None: + bri = int(b[i]) + if ufo is True: + # UFO is timestamped as at the first detection + thisfn = fno[i] - fno[0] + thisx = 0 + thisy = 0 + else: + thisfn = fno[i] + thisx = x[i] + thisy = y[i] + li = f'{thisfn:.4f} {thisx:07.2f} {thisy:07.2f} ' + li = li + f'{ra[i]:8.4f} {dec[i]:+7.4f} {az[i]:8.4f} ' + li = li + f'{alt[i]:+7.4f} {bri:06d} {mag[i]:.02f}\n' + ftpf.write(li) diff --git a/archive/ukmon_pylib/utils/getActiveShowers.py b/archive/ukmon_pylib/utils/getActiveShowers.py new file mode 100644 index 000000000..74d03b4cc --- /dev/null +++ b/archive/ukmon_pylib/utils/getActiveShowers.py @@ -0,0 +1,139 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +# simple script to get the active shower list from the IMO working list + +from utils.imoWorkingShowerList import IMOshowerList as iwsl +import datetime +import numpy as np +import os + + +def getActiveShowers(targdate, retlist=False, inclMinor=False): + """ + Return a list of showers active at the specified date + + Arguments: + targdate: [str] Date in YYYYMMDD format + + Keyword Arguments: + retlist: [bool] return a list, or print to console. Default False=print + inclMinor: [bool] include minor showers or only return major showers + + Returns: + If retlist is true, returns a python list of shower short-codes eg ['PER','LYR'] + + """ + sl = iwsl() + testdate = datetime.datetime.strptime(targdate, '%Y%m%d') + listofshowers=sl.getActiveShowers(testdate, True, inclMinor=inclMinor) + if retlist is False: + for shwr in listofshowers: + print(shwr) + else: + return listofshowers + + +def getActiveShowersStr(targdatestr): + """ + Prints a comma-separated list of showers active at the specified date + + Arguments: + targdate: [str] Date in YYYYMMDD format + + Returns: + nothing + + """ + shwrs = getActiveShowers(targdatestr, retlist=True) + shwrs.append('spo') + for s in shwrs: + print(s) + + +def getShowerDets(shwr, stringFmt=False, dataPth=None): + """ Get details of a shower + + Arguments: + shwr: [string] three-letter shower code eg PER + Keyword Arguments: + stringFmt [bool] default False, return a string rather than a list + dataPth [string] path to the datafiles. Default None means data read from internal files. + + Returns: + (id, full name, peak solar longitude, peak date mm-dd) + """ + sl = iwsl() + mtch = sl.getShowerByCode(shwr, useFull=True) + if len(mtch) > 0 and mtch['@id'] is not None: + id = int(mtch['@id']) + nam = mtch['name'] + pkdtstr = mtch['peak'] + dt = datetime.datetime.now() + yr = dt.year + pkdt = datetime.datetime.strptime(f'{yr} {pkdtstr}','%Y %b %d') + dtstr = pkdt.strftime('%m-%d') + pksollong = mtch['pksollon'] + else: + id, nam, pksollong, dtstr = 0, 'Unknown', 0, 'Unknown' + if stringFmt: + return f"{pksollong},{dtstr},{nam},{shwr}" + else: + return id, nam, pksollong, dtstr + + +def getShowerPeak(shwr): + """ Get date of a shower peak in MM-DD format + + Arguments: + shwr: [string] three-letter shower code eg PER + + Returns: + peak date mm-dd + """ + _, _, _, pk = getShowerDets(shwr) + return pk + + +def numpifyShowerData(): + """Refresh the numpy versions of the shower data files """ + srcdir = os.getenv('SRC', default=os.path.expanduser('~/prod')) + abs_path = os.getenv('WMPL_LOC', default=os.path.expanduser('~/src/WesternMeteorPyLib')) + + iau_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'streamfulldata.csv') + iau_shower_table_npy = os.path.join(abs_path, 'wmpl', 'share', 'streamfulldata.npy') + iau_shower_list = np.loadtxt(iau_shower_table_file, delimiter="|", usecols=range(20), dtype=str) + np.save(iau_shower_table_npy, iau_shower_list) + + iau_shower_table_npy = os.path.join(srcdir, 'share', 'streamfulldata.npy') + np.save(iau_shower_table_npy, iau_shower_list) + + gmn_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'gmn_shower_table_20230518.txt') + gmn_shower_table_npy = os.path.join(abs_path, 'wmpl', 'share', 'gmn_shower_table_20230518.npy') + gmn_shower_list = _loadGMNShowerTable(*os.path.split(gmn_shower_table_file)) + np.save(gmn_shower_table_npy, gmn_shower_list) + + gmn_shower_table_npy = os.path.join(srcdir, 'share', 'gmn_shower_table_20230518.npy') + np.save(gmn_shower_table_npy, gmn_shower_list) + + +def _loadGMNShowerTable(dir_path, file_name): + gmn_shower_list = [] + with open(os.path.join(dir_path, file_name), encoding='cp1252') as f: + for line in f: + if line.startswith('#'): + continue + line = line.strip() + line = line.replace('\n', '').replace('\r', '') + if not line: + continue + la_sun, L_g, B_g, v_g, dispersion, IAU_no, IAU_code = line.split() + gmn_shower_list.append([ + np.radians(float(la_sun)), + np.radians(float(L_g)), + np.radians(float(B_g)), + 1000*float(v_g), + np.radians(float(dispersion)), + int(IAU_no)] + + ) + return np.array(gmn_shower_list) diff --git a/archive/ukmon_pylib/utils/getRiseSet.py b/archive/ukmon_pylib/utils/getRiseSet.py new file mode 100644 index 000000000..081eeed98 --- /dev/null +++ b/archive/ukmon_pylib/utils/getRiseSet.py @@ -0,0 +1,32 @@ +# Copyright (C) 2018- Mark McIntyre +# +import ephem + + +def getNextRiseSet(lati, longi, elev, fordate=None): + """ Calculate the next rise and set times for a given lat, long, elev + + Paramters: + lati: [float] latitude in degrees + longi: [float] longitude in degrees (+E) + elev: [float] latitude in metres + fordate:[datetime] date to calculate for, today if none + + Returns: + rise, set: [date tuple] next rise and set as datetimes + + Note that set may be earlier than rise, if you're invoking the function during daytime. + + """ + obs = ephem.Observer() + obs.lat = float(lati) / 57.3 # convert to radians, close enough for this + obs.lon = float(longi) / 57.3 + obs.elev = float(elev) + obs.horizon = -6.0 / 57.3 # degrees below horizon for darkness + if fordate is not None: + obs.date = fordate + + sun = ephem.Sun() + rise = obs.next_rising(sun).datetime() + set = obs.next_setting(sun).datetime() + return rise, set diff --git a/archive/ukmon_pylib/utils/getUncalImages.py b/archive/ukmon_pylib/utils/getUncalImages.py new file mode 100644 index 000000000..381c96a7c --- /dev/null +++ b/archive/ukmon_pylib/utils/getUncalImages.py @@ -0,0 +1,25 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# + +import os +import datetime + + +def getUncalibratedImageList(dtstr=None): + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + now = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y%m%d') + if dtstr is not None and dtstr != now: + logfile = os.path.join(datadir, '..', 'logs', f'matchJob.log-{dtstr}') + else: + logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') + if os.path.isfile(logfile): + flines = open(logfile, 'r').readlines() + uncal = [f for f in flines if 'not recalibrated' in f] + imglist = [x.split('Skipping ')[1].split(',')[0] for x in uncal] + with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: + for li in imglist: + outf.write(f'{li}\n') + else: + print('file not found', logfile) + imglist=[] + return imglist diff --git a/archive/ukmon_pylib/utils/imoWorkingShowerList.py b/archive/ukmon_pylib/utils/imoWorkingShowerList.py new file mode 100644 index 000000000..abc76760a --- /dev/null +++ b/archive/ukmon_pylib/utils/imoWorkingShowerList.py @@ -0,0 +1,223 @@ +# +# python module to read the IMO Working Shower short List +# +# Copyright (C) 2018-2023 Mark McIntyre + +import xmltodict +import datetime +import os +import numpy as np +import copy + +from wmpl.Utils.TrajConversions import jd2Date +from utils.convertSolLon import sollon2jd + +try: + # imported from $SRC/share + from majorminor import majorlist, minorlist +except Exception: + c = ['QUA', 'LYR', 'ETA', 'CAP', 'SDA', 'PER', 'AUR', 'ORI', 'NTA', 'STA', 'LEO', 'GEM', 'URS'] + minorlist = ['SPE','OCT','DRA','EGE','MON','HYD','COM','NOO'] + + +class IMOshowerList: + """ + Class that loads and parses the IMO Working Shower list, or if needed, the unconfirmed list. + Not all known showers are in the IMO working list. If a shower is not in the Working List then + this library will reference the full shower list curated by Peter Jenniskens which contains + debated and unconfirmed showers. + + """ + def __init__(self, fname=None, fullstreamname=None): + if fname is None: + srcdir = os.getenv('SRC', default=os.path.expanduser('~/prod')) + fname = os.path.join(srcdir, 'share', 'IMO_Working_Meteor_Shower_List.xml') + if not os.path.isfile(fname): + print('unable to open data file') + return + + tmplist = xmltodict.parse(open(fname, 'rb').read()) + self.showerlist = tmplist['meteor_shower_list']['shower'] + if fullstreamname is None: + fullstreamname = os.path.join(srcdir, 'share', 'streamfulldata.npy') + self.fullstreamdata = np.load(fullstreamname) + #print('initialised') + + + def getShowerByCode(self, iaucode, useFull=False): + ds = {'@id':None, 'IAU_code':None,'start':None, 'end':None, + 'peak':None, 'r':None, 'name':None, 'V':None, 'ZHR':None, 'RA':None, 'DE':None, 'pksollon': None} + ds2 = {'@id':None, 'IAU_code':None,'start':None, 'end':None, + 'peak':None, 'r':None, 'name':None, 'V':None, 'ZHR':None, 'RA':None, 'DE':None, 'pksollon': None} + for shower in self.showerlist: + if shower['IAU_code'] == iaucode: + ds = shower + if ds['@id'] is None: + ds['@id'] = -1 + pksollong = -1 + #print(ds) + subset = self.fullstreamdata[np.where(self.fullstreamdata[:,3]==iaucode)] + if subset is not None and len(subset) > 0: + mtch = [sh for sh in subset if int(sh[6]) > -1] + if len(mtch) > 0: + ds2 = copy.deepcopy(ds) + ds2['IAU_code'] = mtch[-1][3].strip() + ds2['name'] = mtch[-1][4].strip() + ds2['V'] = mtch[-1][12] + ds2['@id'] = mtch[-1][1] + ds2['RA'] = mtch[-1][8] + ds2['DE'] = mtch[-1][9] + + pksollong = float(mtch[-1][7]) + dt = datetime.datetime.now() + yr = dt.year + mth = dt.month + jd = sollon2jd(yr, mth, pksollong) + pkdt = jd2Date(jd, dt_obj=True) + ds2['peak'] = pkdt.strftime('%h %d') + # start/end pop idx, ZHR not available in the IAU data + ds2['start'] = (pkdt + datetime.timedelta(days=-2)).strftime('%h %d') + ds2['end'] = (pkdt + datetime.timedelta(days=2)).strftime('%h %d') + ds2['pksollon'] = pksollong + #print(ds2) + else: + # okay so its poor quality but lets try it anyway + mtch = subset + ds2 = copy.deepcopy(ds) + ds2['IAU_code'] = mtch[-1][3].strip() + ds2['name'] = mtch[-1][4].strip() + ds2['V'] = mtch[-1][12] + ds2['@id'] = mtch[-1][1] + ds2['RA'] = mtch[-1][8] + ds2['DE'] = mtch[-1][9] + + pksollong = float(mtch[-1][7]) + dt = datetime.datetime.now() + yr = dt.year + mth = dt.month + jd = sollon2jd(yr, mth, pksollong) + pkdt = jd2Date(jd, dt_obj=True) + ds2['peak'] = pkdt.strftime('%h %d') + # start/end pop idx, ZHR not available in the IAU data + ds2['start'] = (pkdt + datetime.timedelta(days=-2)).strftime('%h %d') + ds2['end'] = (pkdt + datetime.timedelta(days=2)).strftime('%h %d') + ds2['pksollon'] = pksollong + if useFull is False: + if 'pksollon' not in ds: + ds['pksollon'] = ds2['pksollon'] + elif ds['pksollon'] is None: + ds['pksollon'] = ds2['pksollon'] + if ds['peak'] is None: + ds['peak'] = ds2['peak'] + ds['@id'] = ds2['@id'] + return ds + else: + ds2['ZHR'] = ds['ZHR'] + ds2['r'] = ds['r'] + return ds2 + + def getStart(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + if shower['start'] is not None: + startdate = datetime.datetime.strptime(shower['start'], '%b %d') + else: + startdate = datetime.datetime.strptime(shower['peak'], '%b %d') + datetime.timedelta(days=-3) + if iaucode == 'QUA' and mth !=12: + # quadrantids straddle yearend + now = now - 1 + startdate = startdate.replace(year=now) + return startdate + + def getEnd(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + #print(shower) + if shower['end'] is not None: + enddate = datetime.datetime.strptime(shower['end'], '%b %d') + else: + enddate = datetime.datetime.strptime(shower['peak'], '%b %d') + datetime.timedelta(days=3) + if iaucode == 'QUA' and mth == 12: + # quadrantids straddle yearend + now = now + 1 + enddate = enddate.replace(year=now) + return enddate + + def getPeak(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + enddate = datetime.datetime.strptime(shower['peak'], '%b %d') + if iaucode == 'QUA' and mth == 12: + # quadrantids straddle yearend + now = now + 1 + enddate = enddate.replace(year=now) + return enddate + + def getRvalue(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['r'] + + def getName(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['name'] + + def getVelocity(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['V'] + + def getZHR(self, iaucode): + shower = self.getShowerByCode(iaucode) + zhr = shower['ZHR'] + if zhr is None: + return -1 + else: + return int(zhr) + + def getRaDec(self, iaucode): + shower = self.getShowerByCode(iaucode) + return float(shower['RA']), float(shower['DE']) + + def getActiveShowers(self, datetotest, majorOnly=False, inclMinor=False): + activelist = [] + for shower in self.showerlist: + shwname = shower['IAU_code'] + if shwname == 'ANT': #skip the anthelion source, its not a real shower + continue + start = self.getStart(shwname, datetotest.strftime('%Y%m%d')) + #print(shwname, start,shower) + end = self.getEnd(shwname, datetotest.strftime('%Y%m%d')) + datetime.timedelta(days=3) + if datetotest > start and datetotest < end: + if majorOnly is False or (majorOnly is True and shwname in majorlist): + activelist.append(shwname) + elif inclMinor is True and shwname in minorlist: + activelist.append(shwname) + return activelist + + def getMajorShowers(self, includeSpo=False, stringFmt=False): + majlist = majorlist + if includeSpo is True: + majlist.append('spo') + if stringFmt is True: + tmplist = '' + for shwr in majlist: + tmplist = tmplist + shwr + ' ' + majlist = tmplist + return majlist diff --git a/archive/ukmon_pylib/utils/kmlHandlers.py b/archive/ukmon_pylib/utils/kmlHandlers.py new file mode 100644 index 000000000..60f958b24 --- /dev/null +++ b/archive/ukmon_pylib/utils/kmlHandlers.py @@ -0,0 +1,149 @@ +import xmltodict +from shapely.geometry import Polygon +import csv +import simplekml +import numpy as np +import pandas as pd +import os + + +def readCameraKML(kmlFilename, return_poly=False): + """ Load a KML file and return either a list of lats and longs, or a Shapely polygon + + Arguments: + kmlFilename: [string] full path to the KML file to consume + return_poly: [bool] return a Shapely polygon? Default False + + + Returns: + if return_poly is false, returns a tuple of (cameraname, lats, longs) where lats and longs are lists of the + latitudes and longitudes in the KML file. + + If return_poly is true, returns a tuple of (cameranamem, shapely Polygon) + """ + + with open(kmlFilename) as fd: + x = xmltodict.parse(fd.read()) + cname = x['kml']['Folder']['name'] + coords = x['kml']['Folder']['Placemark']['MultiGeometry']['Polygon']['outerBoundaryIs']['LinearRing']['coordinates'] + coords = coords.split('\n') + if return_poly is False: + lats = [] + lngs = [] + for lin in coords: + s = lin.split(',') + lngs.append(float(s[0])) + lats.append(float(s[1])) + return cname, lats, lngs + else: + ptsarr=[] + for lin in coords: + s = lin.split(',') + ptsarr.append((float(s[0]), float(s[1]))) + polyg = Polygon(ptsarr) + return cname, polyg + + +def trackCsvtoKML(trackcsvfile, trackdata=None, saveOutput=True, outdir=None): + """ + Either reads a CSV file containing lat, long, height of an event and + creates a 3d KML file from it or, if trackdata is populated, converts a Pandas dataframe containing + the same data. Output is written to disk unless saveOutput is false. + + Arguments: + trackcsvfile: [string] full path to the file to read from + trackdata: [array] pandas dataframe containing the data. Default None + saveOutput: [bool] write the KML file to disk. Default true + outdir: [string] where to save the file. Default same folder as the source file + + Returns: + the KML file as a tuple + """ + kml=simplekml.Kml() + kml.document.name = trackcsvfile + if trackdata is None: + inputfile = csv.reader(open(trackcsvfile)) + for row in inputfile: + #columns are lat, long, height, times + kml.newpoint(name='', coords=[(row[1], row[0], row[2])]) + else: + for i,r in trackdata.iterrows(): + kml.newpoint(name=f'{r[3]:.5f}', coords=[(r[1], r[0], r[2])], extrude=1, altitudemode='absolute') + if 'csv' in trackcsvfile: + outname = trackcsvfile.replace('.csv','.kml') + else: + outname = f'{trackcsvfile}.kml' + if saveOutput: + if outdir is None: + outdir, _ = os.path.split(trackcsvfile) + os.makedirs(outdir, exist_ok=True) + outname = os.path.join(outdir, outname) + kml.save(outname) + return kml + + +def trackKMLtoCsv(kmlfile, kmldata = None, saveOutput=True, outdir=None): + """ convert a track KML retrieved by ukmondb.trajectoryKML to a 3 dimensional CSV file + containing coordinates of points on the trajectory. + + Arguments: + kmlfile [string] full path to the KML file to read from. + kmldata [kml] the kml data if available. + saveOutput [bool] Default True, save the output to file. + outdir [string] where to save to if saveOutput is True. + + Note: if kmldata is supplied, then kmlfile is ignored. + + Returns: + a Pandas dataframe containing the lat, long, alt and time of each + point on the trajectory, sorted by time. + + """ + with open(kmlfile) as fd: + x = xmltodict.parse(fd.read()) + placemarks=x['kml']['Document']['Placemark'] + lats = [] + lons = [] + alts = [] + tims = [] + for pm in placemarks: + tims.append(float(pm['name'])) + coords = pm['Point']['coordinates'].split(',') + lons.append(float(coords[0])) + lats.append(float(coords[1])) + alts.append(float(coords[2])) + df = pd.DataFrame({"lats": lats, "lons": lons, "alts": alts, "times": tims}) + df = df.sort_values(by=['times', 'lats']) + if saveOutput: + fname = kmlfile + if outdir is None: + outdir, fname = os.path.split(kmlfile) + outf = os.path.join(outdir, fname).replace('.kml', '.csv').replace('.KML','.csv') + df.to_csv(outf, index=False) + return df + + +def getTrackDetails(traj): + """ Get track details from a WMPL trajectory object + + Arguments: + traj: a WMPL trajectory object containing observations + + Returns: + a Pandas dataframe containing the lat, long, alt and time of each point on the trajectory, sorted by time. + """ + lats = [] + lons = [] + alts = [] + lens = [] + # Go through observation from all stations + for obs in traj.observations: + # Go through all observed points + for i in range(obs.kmeas): + lats.append(np.degrees(obs.model_lat[i])) + lons.append(np.degrees(obs.model_lon[i])) + alts.append(obs.model_ht[i]) + lens.append(obs.time_data[i]) + df = pd.DataFrame({"lats": lats, "lons": lons, "alts": alts, "times": lens}) + df = df.sort_values(by=['times', 'lats']) + return df diff --git a/archive/ukmon_pylib/analysis/meteorFlux.py b/archive/ukmon_pylib/utils/meteorFlux.py similarity index 100% rename from archive/ukmon_pylib/analysis/meteorFlux.py rename to archive/ukmon_pylib/utils/meteorFlux.py diff --git a/archive/ukmon_pylib/utils/sendAnEmail.py b/archive/ukmon_pylib/utils/sendAnEmail.py new file mode 100644 index 000000000..a7d215fba --- /dev/null +++ b/archive/ukmon_pylib/utils/sendAnEmail.py @@ -0,0 +1,192 @@ +# Copyright (C) 2018-2023 Mark McIntyre +import os +import platform +import base64 +import email +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart + +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from google_auth_oauthlib.flow import InstalledAppFlow +from googleapiclient.discovery import build + +SCOPES =['https://mail.google.com/'] + + +def _getGmailCreds(tokfile=None, crdfile=None): + creds = None + # The token stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first + # time. + if tokfile is None: + tokfile = '~/.ssh/gmailtoken.json' + if crdfile is None: + crdfile = '~/.ssh/gmailcreds.json' + tokfile = os.path.expanduser(tokfile) + crdfile = os.path.expanduser(crdfile) + if os.path.exists(tokfile): + creds = Credentials.from_authorized_user_file(tokfile, SCOPES) + + # If there are no (valid) credentials available, ask the user to log in interactively. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + if not os.path.isfile(crdfile): + print(f'to use this function you must have stored your Google OAUTH2 secret json file in {crdfile}') + print('To set up OAUTH2 go to your google cloud console, select APIs then Credentials, and add an OAUTH2 desktop client ID.') + return None + flow = InstalledAppFlow.from_client_secrets_file(crdfile, SCOPES) + creds = flow.run_local_server(port=0) + # Save the credentials for the next run + with open(tokfile, 'w') as token: + token.write(creds.to_json()) + return creds + + +def _refreshCreds(tokfile=None, crdfile=None): + creds = None + if tokfile is None: + tokfile = '~/.ssh/gmailtoken.json' + if crdfile is None: + crdfile = '~/.ssh/gmailcreds.json' + tokfile = os.path.expanduser(tokfile) + crdfile = os.path.expanduser(crdfile) + if os.path.exists(tokfile): + creds = Credentials.from_authorized_user_file(tokfile, SCOPES) + + # If there are no (valid) credentials available, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + try: + creds.refresh(Request()) + except: + flow = InstalledAppFlow.from_client_secrets_file(crdfile, SCOPES) + creds = flow.run_local_server(port=0) + # Save the credentials for the next run + if creds.valid: + with open(tokfile, 'w') as token: + token.write(creds.to_json()) + else: + print('credentials not valid, try again') + return creds + + +def _create_message(sender, to, subject, message_text, msg_text_html=None): + if msg_text_html is None: + msg = MIMEText(message_text) + else: + msg = MIMEMultipart('alternative') + msg.attach(MIMEText(message_text, 'plain')) + msg.attach(MIMEText(msg_text_html, 'html')) # html must be the last part + msg['to'] = to + msg['from'] = sender + msg['subject'] = subject + return {'raw': base64.urlsafe_b64encode(msg.as_string().encode('utf-8')).decode('utf-8')} + + + +def sendAnEmail(mailrecip, message, subject, mailfrom, files=None, tokfile=None, crdfile=None, msg_html=None): + """ sends an email using gmail. + + Arguments: + mailrecip: [string] email address of recipient. + message: [string] the message to send. + subject: [string] Subject line. + mailfrom: [string] email address of sender. + Keyword Args: + files: [list] list of files to attach, not currently implemented. + tokfile: [string] full path to token file, if not ~/.ssh/gmailtoken.json + crdfile: [string] full path to credentials file, if not ~/.ssh/gmailcreds.json + msg_html: [string] HTML version of the message body, if any + + Returns: + Nothing, though a message is printed onscreen. + + Notes: + You must have gmail OAUTH2 set up. The gmail credentials default to gmailtoken.json and + gmailcreds.json in the $HOME/.ssh folder. + """ + + if subject is None: + subject = platform.uname()[1] + + # email a summary to the mailrecip + creds = _getGmailCreds(tokfile, crdfile) + if not creds: + return + service = build('gmail', 'v1', credentials=creds) + + subj = subject + mailmsg = _create_message(mailfrom, mailrecip, subj, message, msg_text_html=msg_html) + + try: + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + print('Message Id: %s' % retval['id']) + except: + print('An error occurred sending the message') + + +def forwardAnEmail(reciplist, msgid=None, tokfile=None, crdfile=None): + """ Forward email from a gmail account to a list of recipients + + Arguments: + recplist: [list of strings] list of addresses to forward to + + Keyword arguments: + msgid: [string] gmail message id. If None, then all unread mail is forwarded + tokfile: [string] full path to a gmail oauth2 json token file + crdfile: [string] full path to a gmail oauth2 json credentials file for initial authorization + + To obtain the oauth2 credentials file, go to the google cloud console, select APIs and enable gmail. Then + go to Credentials and create an OAUTH2 Client ID for Desktop and download the JSON credentials file. + Upon first run, you'll be taken to the gmail authorisation screen and the token file will be created. + Subsequent runs will use the token file. The creds file will be used to reauthorise periodically. + """ + creds = _getGmailCreds(tokfile, crdfile) + if not creds: + return + service = build('gmail', 'v1', credentials=creds) + userid = 'me' + if msgid is None: + try: + msglist = (service.users().messages().list(userId=userid, labelIds=['INBOX','UNREAD']).execute()) + for msg in msglist['messages']: + msgid = msg['id'] + # retrieve raw mail + message = (service.users().messages().get(userId='me', id=msgid, format='raw').execute()) + # decode it from base64 + decmsg = base64.urlsafe_b64decode(message['raw']) + newmsg = email.message_from_bytes(decmsg) + newmsg.add_header('Reply-To',newmsg['from']) + newmsg.replace_header('Subject','Fwd: ' + newmsg['subject']) + # Send it + for recip in reciplist: + newmsg.replace_header('To', recip) + mailmsg = {'raw': base64.urlsafe_b64encode(newmsg.as_string().encode('utf-8')).decode('utf-8')} + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + # This will mark the messagea as read + service.users().messages().modify(userId=userid, id=msgid, body={'removeLabelIds': ['UNREAD']}).execute() + print(retval) + except Exception: + print('Nothing to forward') + else: + try: + message = (service.users().messages().get(userId='me', id=msgid, format='raw').execute()) + # decode it from base64 + decmsg = base64.urlsafe_b64decode(message['raw']) + newmsg = email.message_from_bytes(decmsg) + newmsg.add_header('Reply-To',newmsg['from']) + newmsg.replace_header('Subject','Fwd: ' + newmsg['subject']) + # Send it + for recip in reciplist: + newmsg.replace_header('To', recip) + mailmsg = {'raw': base64.urlsafe_b64encode(newmsg.as_string().encode('utf-8')).decode('utf-8')} + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + # This will mark the messagea as read + service.users().messages().modify(userId=userid, id=msgid, body={'removeLabelIds': ['UNREAD']}).execute() + print(retval) + except Exception: + print('Nothing to forward') + return diff --git a/archive/usermgmt/addAdminUser.sh b/archive/usermgmt/addAdminUser.sh deleted file mode 100644 index eb02ec10c..000000000 --- a/archive/usermgmt/addAdminUser.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# simple script to create a new user-maintenance account -# and save the access key locally. Can only be used by an account with IAM Admin permissions. -$username = $args[0] -aws iam create-user --user-name $username --profile ukmda_admin -aws iam add-user-to-group --user-name $username --user-group Administrators --profile ukmda_admin diff --git a/archive/usermgmt/addFBApiKey.sh b/archive/usermgmt/addFBApiKey.sh deleted file mode 100644 index eb957f893..000000000 --- a/archive/usermgmt/addFBApiKey.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# copyright Mark McIntyre, 2024- - -# script to add an api key for a user - -$username = $1 -aws apigateway create-api-key --name "$username" --region eu-west-1 --description "key for $username" --enabled --profile ukmda_admin \ No newline at end of file diff --git a/archive/usermgmt/addSftpUser.sh b/archive/usermgmt/addSftpUser.sh deleted file mode 100644 index 82fdcbd26..000000000 --- a/archive/usermgmt/addSftpUser.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# bash script to create new sftp user -# -userid=$1 -shortid=$2 -updatemode=$3 -oldloc=$4 - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -cd $here - -if [ -f ../config.ini ] ; then - source ~/dev/config.ini - keydir=/home/ec2-user/dev/keymgmt -else - source ~/prod/config.ini - keydir=/home/ec2-user/keymgmt - -fi -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -cd $keydir - -logger -s -t addSftpUser "adding user $userid $shortid $oldloc" - -# wait for the file uploads to complete. For some reason paramiko does this asynchronously -counter=0 -# all-lowercase versions of the names -userid="${userid,,}" -shortid_l="${shortid,,}" - -ls $keydir/$shortid_l.key > /dev/null 2>&1 -while [[ $? -ne 0 && $counter -ne 5 ]] ; do - sleep 2 - logger -s -t addSftpUser "waiting for raw key for $shortid" - ls $keydir/keys/$shortid_l.key > /dev/null 2>&1 - counter=$((counter + 1)) -done -if [ ! -f $keydir/keys/$shortid_l.key ] ; then - logger -s -t addSftpUser "missing keyfile for $shortid" - exit 1 -fi - -# add a unix user and set their homedir to /var/sftp/userid -grep -w $userid /etc/passwd -if [ $? -eq 1 ] ; then - dt=$(date +%Y-%m-%d) - logger -s -t addSftpUser "Creating unix user $userid" - sudo useradd --system --shell /usr/sbin/nologin --groups sftp --home /var/sftp/$userid --comment '${dt}' $userid - sudo mkdir /var/sftp/$userid - sudo chown root:sftp /var/sftp/$userid - sudo chmod 751 /var/sftp/$userid - # create the .ssh folder, platepar folder and empty client copy of the ini file - sudo mkdir /var/sftp/$userid/.ssh - sudo mkdir /var/sftp/$userid/platepar - sudo touch /var/sftp/$userid/ukmon.ini.client - # make these three writeable by the client - sudo chown $userid:$userid /var/sftp/$userid/platepar /var/sftp/$userid/.ssh /var/sftp/$userid/ukmon.ini.client -else - logger -s -t addSftpUser "Unix user $userid already exists" -fi -# copy the public key to the right place -logger -s -t addSftpUser "Applying the public key $userid" -dos2unix $keydir/sshkeys/$userid.pub -sudo cp $keydir/sshkeys/$userid.pub /var/sftp/$userid/.ssh/authorized_keys -sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys -sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys - -# add the key to logupload's authorized_keys file -logger -s -t addSftpUser "Adding the key to loguploads authorized_keys" -awk 1 $keydir/sshkeys/*.pub > /tmp/logul.pub -sudo cp /tmp/logul.pub /var/sftp/logupload/.ssh/authorized_keys -sudo chown logupload:logupload /var/sftp/logupload/.ssh/authorized_keys -rm /tmp/logul.pub - -# copy the ini and aws key files -logger -s -t addSftpUser "Copying the ini file and aws keyfile" -sudo cp $keydir/inifs/$userid.ini /var/sftp/$userid/ukmon.ini -sudo cp $keydir/keys/$shortid_l.key /var/sftp/$userid/live.key -sudo cp $keydir/csvkeys/${shortid}.csv /var/sftp/$userid/$userid.csv -sudo dos2unix /var/sftp/$userid/live.key -sudo dos2unix /var/sftp/$userid/ukmon.ini -sudo dos2unix /var/sftp/$userid/$userid.csv -sudo chown $userid:$userid /var/sftp/$userid/$userid.csv /var/sftp/$userid/ukmon.ini /var/sftp/$userid/live.key - -# if we are moving a station, update the old ini file -# so that the next run points to the new loc -if [ ! -z $oldloc ] ; then - logger -s -t addSftpUser "Moving $oldloc to $userid" - sudo cp /var/sftp/$oldloc/ukmon.ini /var/sftp/$oldloc/ukmon.ini.bkp - sudo cp $keydir/inifs/$userid.ini /var/sftp/$oldloc/ukmon.ini - # and copy the ssh authorized keys file - sudo cp $oldloc/.ssh/authorized_keys /var/sftp/$userid/.ssh/ - sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys - sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys -fi - -logger -s -t addSftpUser "Finished" - -# run these as root to create the structure -# groupadd sftp -# mkdir -p /var/sftp -# chown root:root /var/sftp -# chmod 751 /var/sftp - -# Add this to /etc/ssh/sshd_config -# Match group sftp -# ChrootDirectory /var/sftp/%u -# AllowTCPForwarding no -# X11Forwarding no -# ForceCommand internal-sftp -# -# and then reload sshd -# service sshd reload diff --git a/archive/usermgmt/camTableMaintenance.py b/archive/usermgmt/camTableMaintenance.py deleted file mode 100644 index 30e27b54b..000000000 --- a/archive/usermgmt/camTableMaintenance.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Create and access dynamodb tables containing camera upload timings etc -# -# Copyright (C) 2018-2023 Mark McIntyre - -import boto3 -import os - -from stationMaint2 import addRow, loadLocationDetails - - -def addCreatedDate(): - bucket = 'ukmda-shared' - s3 = boto3.client('s3') - camdets = loadLocationDetails() - for _, cam in camdets.iterrows(): - res = s3.list_objects_v2(Bucket=bucket,Prefix=f'matches/RMSCorrelate/{cam["stationid"]}/', Delimiter='/') - if 'CommonPrefixes' in res: - earliest_fldr = os.path.split(res['CommonPrefixes'][0]['Prefix'][:-1])[1] - created = earliest_fldr.split('_')[1] - else: - created = '' - newdata = {'stationid': cam['stationid'], 'site': cam['site'], 'humanName':cam['humanName'], 'eMail': cam['eMail'], - 'direction': cam['direction'], 'camtype': cam['camtype'], 'active': cam['active'], - 'created': created, 'oldcode': cam['stationid']} - print(newdata) - addRow(newdata) - return - - -# remove a row from the table keyed on stationid adn datestamp in yyyymmdd_hhmmss format -def deleteRow(stationid, ddb=None): - if not ddb: - ddb = boto3.resource('dynamodb', region_name='eu-west-2') - table = ddb.Table('camdetails') - try: - table.delete_item(Key={'stationid': stationid}) - except Exception: - pass - return - - -def dumpCamTable(outdir, statdets=None, ddb=None, exportmindets=False): - if statdets is None: - statdets = loadLocationDetails(ddb=ddb) - statdets = statdets[statdets.active==1] - if exportmindets: - statdets = statdets[['stationid', 'eMail']] - statdets.to_csv(os.path.join(outdir,'camtable.csv'), index=False) diff --git a/archive/usermgmt/testkeys.py b/archive/usermgmt/testkeys.py deleted file mode 100644 index 398d9beed..000000000 --- a/archive/usermgmt/testkeys.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2018-2023 Mark McIntyre -# test that the keys provide read and write access - -import boto3 -import os -import sys - - -def readKeyFile(filename): - if not os.path.isfile(filename): - print('credentials file missing, cannot continue') - exit(1) - with open(filename, 'r') as fin: - lis = fin.readlines() - vals = {} - for li in lis: - if li[0]=='#': - continue - if '=' in li: - valstr = li.split(' ')[1] - data = valstr.split('=') - val = data[1].strip() - if val[0]=='"': - val = val[1:len(val)-1] - vals[data[0]] = val - if 'S3FOLDER' not in vals and 'CAMLOC' in vals: - vals['S3FOLDER'] = f'archive/{vals["CAMLOC"]}' - if 'S3FOLDER' in vals and vals['S3FOLDER'][-1] == '/': - vals['S3FOLDER'] = vals['S3FOLDER'][:-1] - if 'ARCHBUCKET' not in vals: - vals['ARCHBUCKET'] = 'ukmda-shared' - if 'LIVEBUCKET' not in vals: - vals['LIVEBUCKET'] = 'ukmda-live' - if 'WEBBUCKET' not in vals: - vals['WEBBUCKET'] = 'ukmda-website' - if 'ARCHREGION' not in vals: - vals['ARCHREGION'] = 'eu-west-2' - if 'LIVEREGION' not in vals: - vals['LIVEREGION'] = 'eu-west-1' - if 'MATCHDIR' not in vals: - vals['MATCHDIR'] = 'matches/RMSCorrelate' - #print(vals) - return vals - - -def uploadTest(keyfile): - myloc = os.path.split(os.path.abspath(__file__))[0] - filename = os.path.join(myloc, 'keys', keyfile) - keys = readKeyFile(filename) - target = keys['ARCHBUCKET'] - archreg = keys['ARCHREGION'] - livereg = keys['ARCHREGION'] - awskey = keys['AWS_ACCESS_KEY_ID'] - awssec = keys['AWS_SECRET_ACCESS_KEY'] - - with open('test.txt', 'w') as f: - f.write('test') - try: - conn = boto3.Session(aws_access_key_id=awskey, aws_secret_access_key=awssec, region_name=archreg) - s3 = conn.resource('s3') - target='ukmda-shared' - s3.meta.client.upload_file('test.txt', target, 'archive/test.txt') - key = {'Objects': []} - key['Objects'] = [{'Key': 'archive/test.txt'}] - s3.meta.client.delete_objects(Bucket=target, Delete=key) - print(f'{keyfile} archive test successful') - except Exception: - print(f'{keyfile} archive test FAILED') - - try: - conn = boto3.Session(aws_access_key_id=awskey, aws_secret_access_key=awssec, region_name=livereg) - s3 = conn.resource('s3') - target = 'ukmda-live' - s3.meta.client.upload_file('test.txt', target, 'test.txt') - key = {'Objects': []} - key['Objects'] = [{'Key': 'test.txt'}] - s3.meta.client.delete_objects(Bucket=target, Delete=key) - print(f'{keyfile} live test successful') - except Exception: - print(f'{keyfile} live test FAILED') - - os.remove('test.txt') - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print("usage: python testKeys.py keyname\n") - exit(1) - else: - k = sys.argv[1] - uploadTest(k) diff --git a/archive/usermgmt/updateAwsKey.sh b/archive/usermgmt/updateAwsKey.sh deleted file mode 100644 index ff508560d..000000000 --- a/archive/usermgmt/updateAwsKey.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# bash script to update the keyfile to remove plaintext AWS security details -# - -targ=$1 -force=$2 - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -if [ -f ../config.ini ] ; then - source ~/dev/config.ini - keydir=/home/ec2-user/dev/keymgmt -else - source ~/prod/config.ini - keydir=/home/ec2-user/keymgmt - -fi -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -export AWS_PROFILE=ukmonshared -python -m maintenance.getUserAndKeyInfo update $targ $force -export AWS_PROFILE= \ No newline at end of file diff --git a/archive/usermgmt/updateKeysForSplit.py b/archive/usermgmt/updateKeysForSplit.py deleted file mode 100644 index f2025a0df..000000000 --- a/archive/usermgmt/updateKeysForSplit.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import boto3 -import json -import sys -import shutil - - -def createKeyFile(livekey, archkey, location): - archbucket = os.getenv('SRCBUCKET', default='ukmda-shared') - livebucket = os.getenv('LIVEBUCKET', default='ukmda-live') - webbucket = os.getenv('WEBSITEBUCKET', default='ukmda-website') - outf = 'keys/' + location.lower() + '.key' - if archkey is None: - return outf - with open(outf, 'w') as ouf: - ouf.write(f"export AWS_ACCESS_KEY_ID={archkey['AccessKey']['AccessKeyId']}\n") - ouf.write(f"export AWS_SECRET_ACCESS_KEY={archkey['AccessKey']['SecretAccessKey']}\n") - ouf.write(f"export LIVE_ACCESS_KEY_ID={livekey['AccessKey']['AccessKeyId']}\n") - ouf.write(f"export LIVE_SECRET_ACCESS_KEY={livekey['AccessKey']['SecretAccessKey']}\n") - ouf.write('export AWS_DEFAULT_REGION=eu-west-1\n') - ouf.write(f'export CAMLOC="{location}"\n') - ouf.write(f'export S3FOLDER="archive/{location}/"\n') - ouf.write(f'export ARCHBUCKET={archbucket}\n') - ouf.write(f'export LIVEBUCKET={livebucket}\n') - ouf.write(f'export WEBBUCKET={webbucket}\n') - ouf.write('export ARCHREGION=eu-west-2\n') - ouf.write('export LIVEREGION=eu-west-1\n') - ouf.write('export MATCHDIR=matches/RMSCorrelate\n') - os.chmod(outf, 0o666) - return outf - - -def readOldKeyFile(camid): - keyf= f'/var/sftp/{camid}/live.key' - if not os.path.isfile(keyf): - print(f'invalid camid {camid}') - exit() - flines = open(keyf, 'r').readlines() - camloc = None - for li in flines: - if 'AWS_ACCESS_KEY_ID' in li: - oldkey = li.split('=')[1].strip() - elif 'AWS_SECRET_ACCESS_KEY' in li: - oldsec= li.split('=')[1].strip() - elif 'CAMLOC' in li: - camloc= li.split('=')[1].strip().strip('"') - elif 'LIVE_ACCESS_KEY_ID' in li: - print('already processed') - exit() - if camloc is None: - print('malformed key file - no camloc') - exit() - return {'AccessKey': {'AccessKeyId': oldkey, 'SecretAccessKey': oldsec}}, camloc - - -def createNewUser(location): - archuserdets = 'users/' + location + '_arch.txt' - archkeyf = 'rawkeys/arch/' + location + '.key' - archprof = os.getenv('ARCH_PROFILE', default='ukmda_admin') - archconn = boto3.Session(profile_name=archprof) - iamc = archconn.client('iam') - sts = archconn.client('sts') - acct = sts.get_caller_identity()['Account'] - policyarn = 'arn:aws:iam::' + acct + ':policy/UkmonLive' - policyarn2 = 'arn:aws:iam::' + acct + ':policy/UKMDA-shared' - try: - _ = iamc.get_user(UserName=location) - print('location exists, not adding it') - archkey = None - except Exception: - print('new location') - usr = iamc.create_user(UserName=location) - with open(archuserdets, 'w') as outf: - outf.write(str(usr)) - archkey = iamc.create_access_key(UserName=location) - with open(archkeyf, 'w') as outf: - outf.write(json.dumps(archkey, indent=4, sort_keys=True, default=str)) - with open(os.path.join('csvkeys', location + '_arch.csv'),'w') as outf: - outf.write('Access key ID,Secret access key\n') - outf.write('{},{}\n'.format(archkey['AccessKey']['AccessKeyId'], archkey['AccessKey']['SecretAccessKey'])) - _ = iamc.attach_user_policy(UserName=location, PolicyArn=policyarn) - _ = iamc.attach_user_policy(UserName=location, PolicyArn=policyarn2) - os.chmod(archuserdets, 0o666) - os.chmod(archkeyf, 0o666) - return archkey - - -def copyKeyFileToTarget(newkeyfile, camid): - targkeyf = f'/var/sftp/{camid}/live.key' - bkpkeyf = newkeyfile + '.bkp' - if not os.path.isfile(bkpkeyf): - shutil.copyfile(targkeyf, bkpkeyf) - shutil.copyfile(newkeyfile, targkeyf) - return - - -if __name__ == '__main__': - os.chdir('/home/ec2-user/keymgmt') - os.makedirs('users/', exist_ok=True) - os.makedirs('csvkeys/', exist_ok=True) - os.makedirs('rawkeys/arch/', exist_ok=True) - os.makedirs('keys/', exist_ok=True) - - camid = sys.argv[1] - print(f'processing {camid}... ', end='') - sys.stdout.flush() - oldkey, location = readOldKeyFile(camid) - archkey = createNewUser(location) - newkeyfile = createKeyFile(oldkey, archkey, location) - copyKeyFileToTarget(newkeyfile, camid) diff --git a/archive/utils/README.md b/archive/utils/README.md index e179ca9f5..d1317bbe9 100644 --- a/archive/utils/README.md +++ b/archive/utils/README.md @@ -3,27 +3,36 @@ This folder contains some utility functions useful for managing the environment. ## Used by the Batch +* cleanupDeletedTrajs.sh - checks for and removes any logically-deleted trajectories eg duplicates +* getCostmetrics.sh - extract costs data from AWS and push to the website +* loadMatchCsvMDB.sh - loads the match data into MariaDB +* loadSingleCsvMDB.sh - loads the single-station data into MariaDB +* loadBrightCsvMDB.sh - loads brightness data into MariaDB * clearSpace.sh - used by the batch to delete old logs etc -* clearCaches.sh - clears the in-memory cache. Not used much. +* clearCaches.sh - clears the in-memory cache before and after running memory-intensive jobs. + +## Other routine maintenance +* checkAndRollKeys.sh - rolls station AWS Key/Secret pairs periodically to avoid stale keys +* statsToMqtt.sh - posts server space/memory etc statistics to MQ +* userAudit.sh - performs a user audit and emails a report ## User tools to maintain data * deleteOrbit.sh - remove an orbit from the database and website * updateFireballFlag.sh - marks or unmarks a match as a Fireball * updateFireballImage.sh - updates the image shown for a fireball -* stopstart-calcengine.sh - stops/starts the calculation engine server +* stopstartCalcengine.sh - stops/starts the calculation engine server ## Used by the deployment tool * makeConfig.sh - used by the deployment process to make the config file ## Used in case one of the Lambdas fails -* rerunconsolidateFTPdetect.sh - reruns the lambda -* rerunFTPtoUkMONlambra.sh - reruns the lambda -* rerun GetExtraFiles.sh - reruns the lambda +* rerunFTPtoUkMONlambra.sh - reruns FTPtoUKMON lambda +* rerunGetExtraFiles.sh - reruns GetExtraFiles lambda * cftpd_templ.json - template used by the above ## The following are work in progress -* monthlyClearDown.sh - should be run monthly to clear down old data * createTestDataSet.sh - work in progress +* getGmnData.sh - pulls GMN datasets from the GMN server ## Copyright All code Copyright (C) 2018-2023 Mark McIntyre diff --git a/archive/utils/checkCalcServerRunTime.sh b/archive/utils/checkCalcServerRunTime.sh deleted file mode 100644 index 8380d69f3..000000000 --- a/archive/utils/checkCalcServerRunTime.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# script to extract the start/stop times of the Calculation Engine server from the logs -# I have a spreadsheet that consumes this and works out the cost for various models of server (c8g.2xlarge, c8g.4xlarge etc) - -cd ~/prod/logs - -csl=/tmp/calcserver_timings.txt -csl2=/tmp/csl.txt -[ -f $csl ] && rm $csl -touch $csl - -ls -1tr nightly*.gz | while read i -do - mfz=matchJob${i:10:50} - gunzip $i - gunzip $mfz - nf=$(basename $i .gz) - mf=$(basename $mfz .gz) - egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl - gzip $nf - gzip $mf -done -ls -1tr nightlyJob* | grep -v gz | while read nf -do - mf=matchJob${nf:10:50} - egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl -done -[ -f $csl2 ] && rm $csl2 -touch $csl2 -cat $csl | while read i -do - parta=$(echo $i | awk -F ">" '{print $2}') - partb=$(echo $parta | awk -F " " '{printf("%s, %s, %s\n",$1,$2,$3)}') - echo $partb >> $csl2 -done -[ -f $csl ] && rm $csl -python << EOD -lis = open('$csl2','r').readlines() -data = [x.strip().split(',') for x in lis] -currdt = None -results = [] -res = [] -col = 0 -with open('/tmp/computeservertimes.csv','w') as outf: - for i in range(len(data)): - mth = data[i][0] - dy = data[i][1] - tim = data[i][2] - dtstr = f'{mth} {dy}' - if currdt is None: - currdt = dtstr - if currdt != dtstr: - outf.write(','.join(res) + '\n') - res = [] - col = 0 - if col == 0: - res.append(dtstr) - res.append(tim) - else: - res.append(tim) - col += 1 - currdt = dtstr - outf.write(','.join(res) + '\n') -EOD -[ -f $csl2 ] && rm $csl2 diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh new file mode 100644 index 000000000..70774dd1d --- /dev/null +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre +# +# this script reads a list of logically-deleted trajectories from the sqlite database +# and moves the corresponding pickle and report data to a backup area +# These trajectories are generally duplicated events + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $here/../config.ini >/dev/null 2>&1 +conda activate $HOME/miniconda3/envs/${WMPL_ENV} + +cd ${DATADIR}/distrib + +startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') +jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") + +logger -s -t cleanupDeletedTrajs "starting: checking main storage and website" +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + + # check if there are two trajectories with the same folder - we don't want to delete the active one + cnt=$(sqlite3 $DATADIR/distrib/trajectories.db "select count(traj_id) from trajectories where traj_file_path='$traj';") + moved=0 + if [ $cnt == 1 ] ; then + trajdir=$(dirname $traj) + srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir + trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir + aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet + moved=1 + done + yr=${trajdir:13:4} + trajpth=$(basename $trajdir) + webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth + aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv $webloc/$fname $newloc/$fname --quiet + moved=1 + done + [ $moved == 1 ] && echo "moved $trajpth" + else + echo skipping $traj + fi +done + +logger -s -t cleanupDeletedTrajs "checking raw fullcsv data files" +yr=${startdt:0:4} +csvloc=matches/${yr}/fullcsv +newloc=matches/duplicates/csvs +python -c "from maintenance.dataMaintenance import removeDeletedTrajCsv;removeDeletedTrajCsv('$csvloc', '$newloc')" + +# cater for year-end +newyr=$(date +%Y) +if [ $newyr != $yr ] ; then + csvloc=matches/${newyr}/fullcsv + newloc=matches/duplicates/csvs + python -c "from maintenance.dataMaintenance import removeDeletedTrajCsv;removeDeletedTrajCsv('$csvloc', '$newloc')" + +fi + +logger -s -t cleanupDeletedTrajs "checking consolidated matches" +lasttraj=$(sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | tail -1) +trajdir=$(dirname $lasttraj) +yr=${trajdir:13:4} +trajpth=$(basename $trajdir) +matchcsv=$DATADIR/matched/matches-full-${yr}.csv +grep $trajpth $matchcsv > /dev/null +if [ $? == 0 ] ; then + python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$matchcsv')" + python -m converters.toParquet $matchcsv + aws s3 sync $DATADIR/matched/ $UKMONSHAREDBUCKET/matches/matched/ --include "*" --exclude "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + aws s3 sync $DATADIR/matched/ $UKMONSHAREDBUCKET/matches/matchedpq/ --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + aws s3 sync $DATADIR/matched/ $WEBSITEBUCKET/browse/parquet/ --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + + srchcsv=$DATADIR/searchidx/${yr}-allevents.csv + python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$srchcsv')" + aws s3 sync $DATADIR/searchidx/ $WEBSITEBUCKET/search/indexes/ --exclude "*" --include "*allevents.csv" --quiet +fi + +# no need to check the SQL database in mariadb, as it is populated from the CSV file that +# we cleaned up above +logger -s -t cleanupDeletedTrajs "finished cleanupDeletedTrajs" diff --git a/archive/utils/getCostMetrics.sh b/archive/utils/getCostMetrics.sh index 699bb2577..99e7fc8a7 100644 --- a/archive/utils/getCostMetrics.sh +++ b/archive/utils/getCostMetrics.sh @@ -26,4 +26,4 @@ python $PYLIB/metrics/costMetrics.py . eu-west-1 $thismth #export AWS_PROFILE=realukms #python $PYLIB/metrics/costMetrics.py . eu-west-1 $thismth -export AWS_PROFILE= \ No newline at end of file +export AWS_PROFILE= diff --git a/archive/utils/liveevent.template b/archive/utils/liveevent.template deleted file mode 100644 index 02af78a63..000000000 --- a/archive/utils/liveevent.template +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Records": [ - { - "s3": { - "bucket": { - "name": "ukmda-live", - "arn": "arn:aws:s3:::ukmda-live" - }, - "object": { - "key": "KEYHERE" - } - } - } - ] -} diff --git a/archive/utils/updateDb.sh b/archive/utils/loadBrightnessCsvMDB.sh similarity index 61% rename from archive/utils/updateDb.sh rename to archive/utils/loadBrightnessCsvMDB.sh index 2b8df8c27..b38b95c39 100644 --- a/archive/utils/updateDb.sh +++ b/archive/utils/loadBrightnessCsvMDB.sh @@ -11,10 +11,13 @@ else rundt=$1 fi -mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD + +cd $DATADIR/brightness +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value| sed 's/"//g') +mysql -p$passwd -ubatch -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; -load data local infile '/home/ec2-user/prod/data/brightness/CaptureNight_${rundt}.csv' +load data local infile './CaptureNight_${rundt}.csv' into table brightness fields terminated by ','; select count(*) from ukmon.brightness; EOD diff --git a/archive/utils/loadMatchCsvMDB.sh b/archive/utils/loadMatchCsvMDB.sh index 94363039c..4dfaccb51 100644 --- a/archive/utils/loadMatchCsvMDB.sh +++ b/archive/utils/loadMatchCsvMDB.sh @@ -1,7 +1,6 @@ #!/bin/bash # Copyright (C) 2018-2023 Mark McIntyre # -# script to clear out old log files and other old data here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 @@ -18,8 +17,9 @@ if [ ! -f $csvname ] ; then echo "$csvname not found" ; exit ; fi echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch -passwd=$(cat ~/.ssh/db_batch.passwd ) -mysql -u${user} -p${passwd} << EOD +# leading space to prevent being inserted into environment +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') +mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.matches; LOAD DATA LOCAL INFILE '$csvname' INTO TABLE ukmon.matches diff --git a/archive/utils/loadSingleCsvMDB.sh b/archive/utils/loadSingleCsvMDB.sh index a3b9c8018..6d7b7b02b 100644 --- a/archive/utils/loadSingleCsvMDB.sh +++ b/archive/utils/loadSingleCsvMDB.sh @@ -1,7 +1,6 @@ #!/bin/bash # Copyright (C) 2018-2023 Mark McIntyre # -# script to clear out old log files and other old data here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 @@ -18,8 +17,9 @@ if [ ! -f $csvname ] ; then echo "$csvname not found" ; exit ; fi echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch -passwd=$(cat ~/.ssh/db_batch.passwd ) -mysql -u${user} -p${passwd} << EOD +# leading space to prevent being inserted into environment +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') +mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.singles; LOAD DATA LOCAL INFILE '$csvname' INTO TABLE ukmon.singles diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index a5687db67..d68f3f9cb 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -8,6 +8,8 @@ fi RUNTIME_ENV=$1 envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') +#export AWS_PROFILE=default + # read from AWS SSM Parameterset SRC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_srcdir --query Parameters[0].Value | tr -d '"') SITEURL=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_siteurl --query Parameters[0].Value | tr -d '"') @@ -17,23 +19,32 @@ UKMONLIVEBUCKET=s3://$(aws ssm get-parameters --region eu-west-2 --names ${envna RMS_LOC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_rmshome --query Parameters[0].Value | tr -d '"') WMPL_LOC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_wmplhome --query Parameters[0].Value | tr -d '"') SERVERINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcinstance --query Parameters[0].Value | tr -d '"') -BKPINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_backupinstance --query Parameters[0].Value | tr -d '"') SERVERSSHKEY=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_sshkey --query Parameters[0].Value | tr -d '"') NJLOGGRP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_batchloggroup --query Parameters[0].Value | tr -d '"') +SERVERUSERID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcuser --query Parameters[0].Value | tr -d '"') +CALCSERVERIP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcserverip --query Parameters[0].Value | tr -d '"') +unset AWS_PROFILE # hardcoded PYLIB=$SRC/ukmon_pylib TEMPLATES=$SRC/website/templates -RCODEDIR=$SRC/R DATADIR=$SRC/data AWS_DEFAULT_REGION=eu-west-2 MATCHSTART=2 MATCHEND=0 RMS_ENV=RMS WMPL_ENV=wmpl -APIKEY=$(cat ~/.ssh/gmapsapikey) KMLTEMPLATE=*70km.kml +# variables that can be used in scripts and python files to ensure test data doesnt overwrite live data +if [ "$RUNTIME_ENV" == "DEV" ] ; then + TESTMODE="true" + TESTSUFF="/test" +else + TESTMODE="false" + TESTSUFF="" +fi + # create the config file now=$(date +%Y-%m-%d-%H:%M:%S) CFGFILE=~/${envname}/config.ini @@ -47,10 +58,10 @@ echo "UKMONSHAREDBUCKET=${UKMONSHAREDBUCKET}" >> ${CFGFILE} echo "UKMONLIVEBUCKET=${UKMONLIVEBUCKET}" >> ${CFGFILE} echo "PYLIB=${PYLIB}" >> ${CFGFILE} echo "TEMPLATES=${TEMPLATES}" >> ${CFGFILE} -echo "RCODEDIR=${RCODEDIR}" >> ${CFGFILE} echo "DATADIR=${DATADIR}" >> ${CFGFILE} echo "SERVERINSTANCEID=${SERVERINSTANCEID}" >> ${CFGFILE} -echo "BKPINSTANCEID=${BKPINSTANCEID}" >> ${CFGFILE} +echo "SERVERUSERID=${SERVERUSERID}" >> ${CFGFILE} +echo "CALCSERVERIP=${CALCSERVERIP}" >> ${CFGFILE} echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> ${CFGFILE} echo "RMS_ENV=${RMS_ENV}" >> ${CFGFILE} echo "WMPL_ENV=${WMPL_ENV}" >> ${CFGFILE} @@ -59,24 +70,25 @@ echo "WMPL_LOC=${WMPL_LOC}" >> ${CFGFILE} echo "MATCHSTART=${MATCHSTART}" >> ${CFGFILE} echo "MATCHEND=${MATCHEND}" >> ${CFGFILE} echo "SERVERSSHKEY=${SERVERSSHKEY}" >> ${CFGFILE} -echo "APIKEY=${APIKEY}" >> ${CFGFILE} echo "KMLTEMPLATE=${KMLTEMPLATE}" >> ${CFGFILE} echo "NJLOGGRP=${NJLOGGRP}" >> ${CFGFILE} +echo "TESTMODE=${TESTMODE}" >> ${CFGFILE} +echo "TESTSUFF=${TESTSUFF}" >> ${CFGFILE} echo "" >> ${CFGFILE} echo "export RUNTIME_ENV SRC SITEURL" >> ${CFGFILE} echo "export WEBSITEBUCKET UKMONSHAREDBUCKET" >> ${CFGFILE} -echo "export PYLIB TEMPLATES RCODEDIR DATADIR BKPINSTANCEID AWS_DEFAULT_REGION" >> ${CFGFILE} +echo "export PYLIB TEMPLATES DATADIR AWS_DEFAULT_REGION" >> ${CFGFILE} echo "export RMS_ENV RMS_LOC WMPL_ENV WMPL_LOC" >> ${CFGFILE} echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFILE} echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} -echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID NJLOGGRP" >> ${CFGFILE} +echo "export KMLTEMPLATE SERVERINSTANCEID SERVERUSERID CALCSERVERIP NJLOGGRP" >> ${CFGFILE} echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib" >> ${CFGFILE} +echo "export TESTMODE TESTSUFF" >> ${CFGFILE} + # enable miniconda echo "source ~/.condaon" >> ${CFGFILE} # function to log to cloudwatch echo 'function log2cw() { ' >> ${CFGFILE} -echo 'aws logs put-log-events --log-group-name $1 --log-stream-name $2 --log-events "[{\"timestamp\": $(date +%s%3N), \"message\": \"$3\"}]" --profile ukmonshared > /dev/null ' >> ${CFGFILE} +echo 'aws logs put-log-events --log-group-name $1 --log-stream-name $2 --log-events "[{\"timestamp\": $(date +%s%3N), \"message\": \"$3\"}]" > /dev/null ' >> ${CFGFILE} echo 'logger -s -t $4 RUNTIME $SECONDS $3' >> ${CFGFILE} echo '}' >> ${CFGFILE} - - diff --git a/archive/utils/monthlyCleardown.sh b/archive/utils/monthlyCleardown.sh deleted file mode 100644 index 89ca8f7bf..000000000 --- a/archive/utils/monthlyCleardown.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# script to backup then clear down orbits from the local disk. - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -what=$1 -if [ "$what" == "" ] ; then - echo "usage: ./monthlyCleardown.sh single|raw|matched|reports|search|other" - exit -fi - -cyr=$(date +%Y) -cmt=$(date +%m) -lyr=$(date --date='last year' +%Y) -lym=$(date --date='last month' +%Y%m) - -mkdir -p $DATADIR/olddata - -# SINGLE STATION DATA -case "$what" in -single) - logger -s -t monthlyClearDown "single-station data" - cd $DATADIR/single/new/processed - # backup then remove last year's data if still present - if compgen -G "$DATADIR/single/new/processed/ukmon*_${lyr}????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-singlecsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-singlecsv.tar ukmon*_${lyr}????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-singlecsv.tar ukmon*_${lyr}????_*.csv - fi - rm -f $DATADIR/single/new/processed/ukmon*_${lyr}????_*.csv - else - echo "no ukmon-csv files from ${lyr} to archive" - fi - # backup last month's data - if compgen -G "$DATADIR/single/new/processed/ukmon*_${lym}??_*.csv" > /dev/null ; then - if [ $cmt -ne 1 ] ; then - if [ ! -f $DATADIR/olddata/${lym}-singlecsv.tar ] ; then - tar cvf $DATADIR/olddata/${lym}-singlecsv.tar ukmon*_${lym}??_*.csv - else - tar uvf $DATADIR/olddata/${lym}-singlecsv.tar ukmon*_${lym}??_*.csv - fi - fi - rm -f $DATADIR/single/new/processed/ukmon*_${lym}??_*.csv - else - echo "no ukmon-csv files from ${lym} to archive" - fi - ;; -raw) - logger -s -t monthlyClearDown "rawcsv data" - # raw CSV files from cameras - cd $DATADIR/single/rawcsvs - # backup then remove last year's data if still present - if compgen -G "$DATADIR/single/rawcsvs/*${lyr}????_??????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-rawcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lyr}????_??????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lyr}????_??????_*.csv - fi - find . -maxdepth 1 -name "*${lyr}????_??????_*.csv" -print0 | xargs -0 rm -f - else - echo "no rawcsv files from ${lyr} to archive" - fi - # backup last month's data - if compgen -G "$DATADIR/single/rawcsvs/*${lym}??_??????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-rawcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lym}??_??????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lym}??_??????_*.csv - fi - rm -f $DATADIR/single/rawcsvs/*${lym}??_??????_*.csv - else - echo "no rawcsv files from ${lym} to archive" - fi - ;; -matched) - logger -s -t monthlyClearDown "match data" - # MATCHES - # last year's fullcsv files - cd $DATADIR/orbits/$lyr/fullcsv/processed - if compgen -G "$DATADIR/orbits/${lyr}/fullcsv/processed/${lyr}*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-matchfullcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-matchfullcsv.tar ${lyr}*.csv - else - tar uvf $DATADIR/olddata/${lyr}-matchfullcsv.tar ${lyr}*.csv - fi - rm -f $DATADIR/orbits/${lyr}/fullcsv/processed/${lyr}*.csv - else - echo "no fullcsv files from ${lyr} to archive" - fi - - # and now last months fullcsv data - cd $DATADIR/orbits/$cyr/fullcsv/processed - if compgen -G "$DATADIR/orbits/${cyr}/fullcsv/processed/${lym}??-*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lym}-matchfullcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lym}-matchfullcsv.tar ${lym}??-*.csv - else - tar uvf $DATADIR/olddata/${lym}-matchfullcsv.tar ${lym}??-*.csv - fi - rm -f $DATADIR/orbits/${cyr}/fullcsv/processed/${lym}??-*.csv - else - echo "no fullcsv files from ${lym} to archive" - fi - ;; -reports) - logger -s -t monthlyClearDown "reports and so forth" - # SHOWER, MONTHLY AND ANNUAL REPORTS FROM PREVIOUS YEAR - cd $DATADIR/reports/${lyr} - nf=$(ls -1 | wc -l) - if [ $nf -ne 0 ] ; then - if [ ! -f $DATADIR/olddata/${lyr}reports.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-reports.tar * - else - tar uvf $DATADIR/olddata/${lyr}-reports.tar * - fi - rm -Rf * - else - echo "no report files from ${lyr} to archive" - fi - ;; -search) - logger -s -t monthlyClearDown "old search indexes" - # SEARCH INDEXES FOR PRIOR YEAR - cd $DATADIR/searchidx - if compgen -G "$DATADIR/searchidx/${lyr}-allevents.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-searchidx.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-searchidx.tar ${lyr}-allevents.csv - else - tar uvf $DATADIR/olddata/${lyr}-searchidx.tar ${lyr}-allevents.csv - fi - rm -Rf $DATADIR/searchidx/${lyr}-allevents.csv - else - echo "no search indexes from ${lyr} to archive" - fi - ;; -ftpdata) - logger -s -t monthlyClearDown "old ftpdetect and platepars_all files" - # cleardown the raw input data. This is already backed up elsewhere - inputym=$(date --date='3 months ago' +%Y%m) - #python $PYLIB/maintenance/dataMaintenance.py $inputym - ;; -*) - echo missing argument - ;; -esac -# now push it all to the offline backup -logger -s -t monthlyClearDown "syncing to backup bucket" -aws s3 sync $DATADIR/olddata/ s3://ukmda-admin/backups/ --exclude "*" --include "${lyr}*.tar" --quiet -rm -f $DATADIR/olddata/${lyr}*.tar - -aws s3 sync $DATADIR/olddata/ s3://ukmda-admin/backups/ --exclude "*" --include "${lym}*.tar" --quiet -# dont do this in case data arrives late -#rm -f $DATADIR/olddata/${lym}*.tar diff --git a/archive/utils/rerunFTPtoUKMONlambda.sh b/archive/utils/rerunFTPtoUKMONlambda.sh index 9c768d024..38d6b1f63 100644 --- a/archive/utils/rerunFTPtoUKMONlambda.sh +++ b/archive/utils/rerunFTPtoUKMONlambda.sh @@ -22,6 +22,6 @@ cat /tmp/ftp2ukmon.txt | while read i do fullname=${i} cat $SRC/utils/cftpd_templ.json | sed "s|KEYGOESHERE|${fullname}|g" > ./tmp.json - aws lambda invoke --profile ukmonshared --function-name ftpToUkmon --log-type Tail --payload file://./tmp.json --region eu-west-2 --cli-binary-format raw-in-base64-out res.log + aws lambda invoke --function-name ftpToUkmon --log-type Tail --payload file://./tmp.json --region eu-west-2 --cli-binary-format raw-in-base64-out res.log done rm ./tmp.json \ No newline at end of file diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh index 3235281a2..170ebb0bf 100644 --- a/archive/utils/statsToMqtt.sh +++ b/archive/utils/statsToMqtt.sh @@ -5,15 +5,15 @@ if [ "$(hostname)" == "wordpresssite" ] ; then source /home/bitnami/venvs/openhabstuff/bin/activate python $here/statsToMqtt.py diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') - if [ $diskpct -gt 9 ] ; then + if [ $diskpct -gt 90 ] ; then /home/bitnami/src/maintenance/mysql_checks.sh msg="$(hostname) diskspace was ${diskpct}%" subj="Diskspace alert for $(hostname)" hn="$(hostname)@aws" - python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" + python -c "from utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi elif [ "$(hostname)" == "ukmcalcserver" ] ; then - source /home/ec2-user/venvs/wmpl/bin/activate + source $HOME/venvs/wmpl/bin/activate python $here/statsToMqtt.py diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') if [ $diskpct -gt 80 ] ; then @@ -31,6 +31,6 @@ else msg="$(hostname) diskspace is ${diskpct}%" subj="Diskspace alert for $(hostname)" hn="$(hostname)@aws" - python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" + python -c "from utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi fi diff --git a/archive/utils/stopstart-calcengine.sh b/archive/utils/stopstartCalcengine.sh similarity index 90% rename from archive/utils/stopstart-calcengine.sh rename to archive/utils/stopstartCalcengine.sh index 85d31f29c..366ed507f 100644 --- a/archive/utils/stopstart-calcengine.sh +++ b/archive/utils/stopstartCalcengine.sh @@ -5,4 +5,4 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 -aws ec2 $1-instances --instance-ids $SERVERINSTANCEID --region eu-west-2 --profile ukmonshared +aws ec2 $1-instances --instance-ids $SERVERINSTANCEID --region eu-west-2 diff --git a/archive/utils/userAudit.sh b/archive/utils/userAudit.sh index 205f7e209..98e7123f0 100644 --- a/archive/utils/userAudit.sh +++ b/archive/utils/userAudit.sh @@ -8,22 +8,24 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -cd $DATADIR -echo "{\"Subject\": {\"Data\": \"User Audit\",\"Charset\": \"utf-8\"}," > auditmsg.json -echo -n "\"Body\": {\"Text\": {\"Data\": \"" >> auditmsg.json +echo "{\"Subject\": {\"Data\": \"User Audit\",\"Charset\": \"utf-8\"}," > $DATADIR/admin/auditmsg.json +echo -n "\"Body\": {\"Text\": {\"Data\": \"" >> $DATADIR/admin/auditmsg.json # general audit of all users export AWS_PROFILE=ukmonshared -python -m maintenance.getUserAndKeyInfo audit > $DATADIR/useraudit.txt +python -m maintenance.getUserAndKeyInfo audit > $DATADIR/admin/useraudit.txt # accounts not used for 30 days or more export MAXAGE=30 -python -m maintenance.getUserAndKeyInfo dormant >> $DATADIR/useraudit.txt +python -m maintenance.getUserAndKeyInfo dormant >> $DATADIR/admin/useraudit.txt -awk -v ORS='\\n' '1' useraudit.txt >> auditmsg.json -echo "\",\"Charset\": \"utf-8\"}}}" >> auditmsg.json +awk -v ORS='\\n' '1' $DATADIR/admin/useraudit.txt >> $DATADIR/admin/auditmsg.json +echo "\",\"Charset\": \"utf-8\"}}}" >> $DATADIR/admin/auditmsg.json + +pushd $DATADIR/admin aws ses send-email --destination ToAddresses="markmcintyre99@googlemail.com" --message file://auditmsg.json --from "noreply@ukmeteors.co.uk" --region eu-west-2 +popd +rm $DATADIR/admin/auditmsg.json unset MAXAGE unset AWS_PROFILE -rm auditmsg.json diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh index 537a7b2fe..385fb0353 100644 --- a/archive/website/cameraStatusReport.sh +++ b/archive/website/cameraStatusReport.sh @@ -27,9 +27,28 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} export PYTHONPATH=$PYLIB logger -s -t cameraStatusReport "starting" -sudo grep publickey /var/log/secure* | grep -v ec2-user | awk '{printf("%s-%s, %s, %s\n", $1, $2, $3,$9)}' | awk -F ":" '{print $2$3$4}' > $DATADIR/reports/lastlogins.txt +echo TEMPORARY HACK REMOVE CODE PERTAINING TO OLD SERVER +platform=$(grep "^NAME" /etc/os-release | awk -F'"' '{print $2}') +if [ "$platform" == "Ubuntu" ] ; then + authlog=/var/log/auth.log + batchuser=ubuntu +else + authlog=/var/log/secure + batchuser=ec2-user +fi +sudo grep publickey $authlog* | grep -v $batchuser > $DATADIR/reports/lastlogins.txt + +if [ "$platform" == "Ubuntu" ] +then + ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | grep -v ec2-user" > $DATADIR/reports/lastlogins_old.txt +fi +cat $DATADIR/reports/lastlogins_old.txt >> $DATADIR/reports/lastlogins.txt +echo TO HERE python -m metrics.camMetrics $rundate +rm -f $DATADIR/reports/lastlogins.txt +rm -f $DATADIR/reports/lastlogins_old.txt + aws s3 cp $DATADIR/reports/stationlogins.txt $WEBSITEBUCKET/reports/stationlogins.txt --region eu-west-2 --quiet diff --git a/archive/website/costReport.sh b/archive/website/costReport.sh index cd60ec06f..1ad8efa04 100644 --- a/archive/website/costReport.sh +++ b/archive/website/costReport.sh @@ -27,10 +27,8 @@ if [ $# -gt 1 ] ; then numdays = $1 ; fi mkdir -p $DATADIR/costs > /dev/null -export AWS_PROFILE=ukmonshared logger -s -t costReport "getting data for $numdays for $AWS_PROFILE" python -m metrics.costMetrics $DATADIR/costs eu-west-2 $numdays -unset AWS_PROFILE tod=$(date +%d) if [ "$tod" == "03" ] ; then @@ -74,8 +72,8 @@ done echo "var outer_div = document.getElementById(\"docindex\"); outer_div.appendChild(table); })" >> $frjs logger -s -t costReport "publishing data" -aws s3 sync $DATADIR/costs/ $WEBSITEBUCKET/docs/financial/raw/ --exclude "*" --include "costs-18*.csv" --quiet --profile ukmonshared -aws s3 cp $costfile $WEBSITEBUCKET/reports/ --quiet --profile ukmonshared -aws s3 cp $DATADIR/$imgfile3 $WEBSITEBUCKET/reports/ --quiet --profile ukmonshared -aws s3 cp $frjs $WEBSITEBUCKET/docs/ --quiet --profile ukmonshared +aws s3 sync $DATADIR/costs/ $WEBSITEBUCKET/docs/financial/raw/ --exclude "*" --include "costs-18*.csv" --quiet +aws s3 cp $costfile $WEBSITEBUCKET/reports/ --quiet +aws s3 cp $DATADIR/$imgfile3 $WEBSITEBUCKET/reports/ --quiet +aws s3 cp $frjs $WEBSITEBUCKET/docs/ --quiet logger -s -t costReport "done" diff --git a/archive/website/createFireballPage.sh b/archive/website/createFireballPage.sh index 7c63ba4bb..396aeb65d 100644 --- a/archive/website/createFireballPage.sh +++ b/archive/website/createFireballPage.sh @@ -88,7 +88,7 @@ echo "\$(document).ready(function() {" >> reportindex.js # need single quotes here to allow the hash to be printed echo '$("#fbtableid").DataTable({' >> reportindex.js echo "columnDefs : [" >> reportindex.js -echo "{ Type : \"numeric\", targets : [2]}" >> reportindex.js +echo "{ Type : \"numeric\", targets : [1]}" >> reportindex.js echo " ]," >> reportindex.js echo "order : [[ 1, \"asc\"],[2,\"desc\"]]," >> reportindex.js echo "paging: false" >> reportindex.js diff --git a/archive/website/createOrbitIndex.sh b/archive/website/createOrbitIndex.sh index 2d92cec21..3b05e1e8f 100644 --- a/archive/website/createOrbitIndex.sh +++ b/archive/website/createOrbitIndex.sh @@ -90,9 +90,9 @@ else j=0 echo "" >> $idxfile fi - if [ $domth -eq 1 ] ; then - echo "$i" >> $DATADIR/orbits/$yr/$ym.txt - fi + #if [ $domth -eq 1 ] ; then + # echo "$i" >> $DATADIR/orbits/$yr/$ym.txt + #fi done fi echo "
    DateTimeSourceShowerMagCameraLink
    " >> $idxfile diff --git a/archive/website/createReportIndex.sh b/archive/website/createReportIndex.sh index edf24c4f8..f758ae1d2 100644 --- a/archive/website/createReportIndex.sh +++ b/archive/website/createReportIndex.sh @@ -60,7 +60,7 @@ if [ -f $curryr/tmp.txt ] ; then rm -f $curryr/tmp.txt ; fi aws s3 ls $WEBSITEBUCKET/reports/$curryr/ | grep PRE | egrep -v "ALL|orbits|stations|fireballs|showers|.js|.html" | awk '{print $2 }' | while read j do - python -c "from meteortools.utils import getShowerDets ; x=getShowerDets('${j:0:3}', True); print(x)" >> $curryr/tmp.txt + python -c "from utils.getActiveShowers import getShowerDets ; x=getShowerDets('${j:0:3}', True); print(x)" >> $curryr/tmp.txt done sort -n $curryr/tmp.txt > $curryr/shwrs.txt rm -f $curryr/tmp.txt diff --git a/archive/website/createShwrExtracts.sh b/archive/website/createShwrExtracts.sh index beb015b48..d90f0463e 100644 --- a/archive/website/createShwrExtracts.sh +++ b/archive/website/createShwrExtracts.sh @@ -44,7 +44,7 @@ cd $DATADIR/browse/showers/ # get a list of files on the website aws s3 ls $WEBSITEBUCKET/browse/showers/ | awk '{ print $4 }' | grep csv > /tmp/browseshwr.txt -shwrs=$(python -c "from meteortools.utils.getActiveShowers import getActiveShowersStr ; getActiveShowersStr('${ymd}')") +shwrs=$(python -c "from utils.getActiveShowers import getActiveShowersStr ; getActiveShowersStr('${ymd}')") for shwr in $shwrs do now=$(date '+%Y-%m-%d %H:%M:%S') diff --git a/archive/website/createSummaryTable.sh b/archive/website/createSummaryTable.sh index fab46348f..caa0fd83b 100644 --- a/archive/website/createSummaryTable.sh +++ b/archive/website/createSummaryTable.sh @@ -9,6 +9,7 @@ # Consumes # the single station and matched station data files in $DATADIR/single and $DATADIR/matched # the most recent match run logfile +# information about the number of active cameras # # Produces # a webpage showing the current summary - the site homepage @@ -32,37 +33,37 @@ python -c "from reports.createSummaryTable import createSummaryTable; createSumm logger -s -t createSummaryTable "create a coverage map from the kmls" # make sure correct version of GEOS and PROJ4 available for mapping routines -LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib -export LD_LIBRARY_PATH +#LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib +#export LD_LIBRARY_PATH mkdir -p $DATADIR/kmls aws s3 sync $WEBSITEBUCKET/img/kmls/ $DATADIR/kmls --quiet export KMLTEMPLATE="*25km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} export KMLTEMPLATE="*70km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} export KMLTEMPLATE="*100km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} -python -c "from reports.makeCoverageMap import createCoveragePage as ccp ; ccp() ;" +python -c "from reports.makeCoverageMap import createCoveragePage as ccp ; ccp('reports/${yr}') ;" logger -s -t createSummaryTable "create year-to-date barchart" -pushd $DATADIR python -c "from reports.createAnnualBarChart import createBarChart; createBarChart('${DATADIR}','${yr}')" -popd # update index page -numcams=$(cat $DATADIR/activecamcount.txt) -cat $TEMPLATES/frontpage.html | sed "s/#NUMCAMS#/$numcams/g" > $DATADIR/newindex.html +numcams=$(cat $DATADIR/consolidated/activecamcount.txt) +cat $TEMPLATES/frontpage.html | sed "s/#NUMCAMS#/$numcams/g" > $DATADIR/reports/${yr}/newindex.html logger -s -t createSummaryTable "copying to website" -aws s3 cp $DATADIR/latest/coverage-maps.html $WEBSITEBUCKET/latest/ --quiet -aws s3 cp $DATADIR/summarytable.js $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-100km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/coverage.html --quiet -aws s3 cp $DATADIR/coverage-25km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/Annual-${yr}.jpg $WEBSITEBUCKET/YearToDate.jpg --quiet -aws s3 cp $DATADIR/newindex.html $WEBSITEBUCKET/index.html --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-maps.html $WEBSITEBUCKET/latest/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-100km.html $WEBSITEBUCKET/data/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-70km.html $WEBSITEBUCKET/data/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-70km.html $WEBSITEBUCKET/data/coverage.html --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-25km.html $WEBSITEBUCKET/data/ --quiet + +aws s3 cp $DATADIR/reports/${yr}/Annual-${yr}.jpg $WEBSITEBUCKET/YearToDate.jpg --quiet + +aws s3 cp $DATADIR/reports/${yr}/newindex.html $WEBSITEBUCKET/index.html --quiet +aws s3 cp $DATADIR/reports/${yr}/summarytable.js $WEBSITEBUCKET/data/ --quiet aws s3 cp $SRC/website/templates/header.html $WEBSITEBUCKET/templates/ --quiet aws s3 cp $SRC/website/templates/footer.html $WEBSITEBUCKET/templates/ --quiet diff --git a/archive/website/templates/searchdialog.js b/archive/website/templates/searchdialog.js deleted file mode 100644 index e85d171cf..000000000 --- a/archive/website/templates/searchdialog.js +++ /dev/null @@ -1,122 +0,0 @@ -// initialize input widgets first -// Copyright (C) 2018-2023 Mark McIntyre - -$('#selectInterval .time').timepicker({ - 'showDuration': true, - 'timeFormat': 'g:ia' -}); - -$('#selectInterval .date').datepicker({ - 'format': 'd/m/yyyy', - 'autoclose': true, - 'endDate': '0' - -}); - -// initialize datepair -var res = document.getElementById("selectInterval"); -var dateSelect = new Datepair(res, { - 'defaultDateDelta': 0, // days - 'defaultTimeDelta': 900000 // milliseconds = 15 minutes -}); - -$('#selectInterval').on('rangeSelected', function(){ - var startdate = $('#selectInterval .date:first').datepicker('getDate'); - var starttime = $('#selectInterval .time:first').timepicker('getTime', startdate); - if(starttime != null){ - var dateval = starttime.getTime(); - var timediff = dateSelect.getTimeDiff(); - var endval = dateval + timediff; - endtime = new Date(); - endtime.setTime(endval); - $('#datestart').text(starttime.toISOString()); - $('#dateend').text(endtime.toISOString()); - if(timediff > 0){ - $('#statusfield').text('Valid range selected'); - }else{ - $('#statusfield').text('Date range must be > 0'); - } - } -}); - -var apiurl = 'https://api.ukmeteors.co.uk/detections'; -var form = document.querySelector("form"); -form.addEventListener("submit", function (event) { - //console.log("Saving value", form.elements.value.value); - var d1 = document.getElementById("datestart").innerHTML; - var d2 = document.getElementById("dateend").innerHTML; - var matchOnly = document.getElementById("matchesOnly").checked; - console.log(matchOnly) - var op = ""; - if (matchOnly == true ) { - op = op + "t:_"; - } - var magSelect = document.getElementById("magselect").value; - if (magSelect != 1 ) { - if(magSelect == 2) {op = op + "m:0_"} - if(magSelect == 3) {op = op + "m:-4_"} - } - var shwrSelect = document.getElementById("shwrselect").value; - if (shwrSelect != 1 ) { - var e = document.getElementById("shwrselect"); - var strshwr = e.options[e.selectedIndex].text; - op = op + "s:" + strshwr + "_"; - } - var statSelect = document.getElementById("statselect").value; - if (statSelect != 1 ) { - var e = document.getElementById("statselect"); - var strstat = e.options[e.selectedIndex].text; - op = op + "l:" + strstat + "_"; - } - //jQuery.support.cors = true; - payload = {"d1": d1, "d2": d2, "opts": op }; - console.log(payload); - document.getElementById("searchresults").innerHTML = "Searching...."; - $.ajax({ - url: apiurl, - type: "GET", - data: payload, - dataType: 'jsonp', - error: function (xhr, status, ex ) { - if (status === 'error' ) { - alert("Too much data, try a narrower range"); - console.log(xhr.status); - } - }, - complete: function (xhr, status) { - } - }); - event.preventDefault();} - ); - -function myFunc(myObj) { - console.log(myObj); - var x, txt = ""; - txt = "" - txt += ""; - txt += ""; - for (x in myObj) { - console.log(myObj[x]); - var dtarr = myObj[x].split(','); - var cams = dtarr[4].replaceAll(";", "; "); - if (dtarr[0] > 0 ){ - txt +='"; - } - } - txt += "
    DateTimeSourceShowerMagCameraLink
    '; - dt = new Date(dtarr[0]*1000); - txt += dt.toISOString(); - txt += ""; - txt += dtarr[1]; - txt += ""; - txt += dtarr[2]; - txt += ""; - txt += dtarr[3]; - txt += ""; - txt += cams; - txt += ""; - errimg = "onerror=\"this.onerror=null;this.src='/img/missing.png';\""; - txt += "\"Image"; - txt += "
    "; - document.getElementById("searchresults").innerHTML = txt; -} diff --git a/archive/website/updateCameraDets.sh b/archive/website/updateCameraDets.sh new file mode 100644 index 000000000..c620bde50 --- /dev/null +++ b/archive/website/updateCameraDets.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre +# create various html and js files of camera info for the website +# +# Parameters +# none +# +# Consumes +# +# +# Produces +# camera locations file cameraLocs.json +# camera details file camera-details.csv +# three html files used by the search page + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# load the configuration +source $here/../config.ini >/dev/null 2>&1 +conda activate ${WMPL_ENV} + +logger -s -t updateCameraDets "starting" + +python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();" +aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --quiet +aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --quiet +aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --quiet + +# create the CSV file of camera info and the html versions for search functions on the website + +python -c "from reports.CameraDetails import createCDCsv; createCDCsv('consolidated','searchidx');" +aws s3 cp $DATADIR/consolidated/camera-details.csv $UKMONSHAREDBUCKET/consolidated/ --quiet +aws s3 cp $DATADIR/searchidx/statopts.html $WEBSITEBUCKET/search/ --quiet +aws s3 cp $DATADIR/searchidx/activestatopts.html $WEBSITEBUCKET/search/ --quiet +aws s3 cp $DATADIR/searchidx/activestatlocs.html $WEBSITEBUCKET/search/ --quiet + +logger -s -t updateCameraDets "finished" diff --git a/bumpver.toml b/bumpver.toml index 9cab7605b..17d501fc7 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2024.04.2" +current_version = "2026.05.0" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true @@ -13,6 +13,6 @@ push = true "bumpver.toml" = [ 'current_version = "{version}"', ] -".github/workflows/build_usermgmt.yml" = [ - 'tag_name: {version}' +"archive/README.md" = [ + "version: {version}" ] diff --git a/install_or_update.sh b/install_or_update.sh new file mode 100755 index 000000000..83713e346 --- /dev/null +++ b/install_or_update.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre + +if [[ "$1" != "PROD" && "$1" != "DEV" ]] ; then + echo "must provide runtime env of PROD or DEV" + exit -1 +fi +RUNTIME_ENV=$1 +envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +echo "Updating codebase..." +cd $here/archive +git pull + +echo "Updating code..." +mkdir -p ~/${envname} +[ -d ~/$envname/data ] && msg="upgrade" || msg="install" +for loc in analysis ukmon_pylib website cronjobs utils static_content +do + rsync -a --delete $loc/ ~/${envname}/$loc + chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 +done +rsync -a share/ ~/${envname}/share + +echo "Creating data folders..." +DATADIR=~/$envname/data +mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls,manualuploads} +mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} +mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} +mkdir -p ~/$envname/logs +mkdir -p ~/.logrotate +mkdir -p ~/.aws + +echo "Checking conda environment..." +if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] +then + pushd $here + source ~/.condaon + conda activate wmpl + pip install -r archive/ukmon_pylib/additional_requirements.txt | grep -v "already satisfied" + echo "" + popd +fi + +echo "Updating meteor shower tables..." +# update the IMO and GMN meteor shower tables if missing +if [ ! -f ~/${envname}/share/IMO_Working_Meteor_Shower_List.xml ] +then + ~/$envname/cronjobs/getImoWSfile.sh + echo "" +fi +echo "" +read -n 1 -p "Update $envname configuration files? (y/N) " yesno +if [[ "$yesno" == "y" || "$yesno" == "Y" ]] +then + echo "" + echo "Updating config for $envname" + ~/$envname/utils/makeConfig.sh $RUNTIME_ENV + for fil in .bashrc .bash_aliases .vimrc .condaon + do + rsync -a server_setup/$fil ~ + done + mkdir -p ~/.logrotate + rsync -a server_setup/logs.conf ~/.logrotate +else + echo "" + echo skipping config and bashrc +fi +echo "" +echo "$msg complete" diff --git a/tests/test_apis.py b/tests/test_apis.py index 4394b5d16..3d98c763d 100644 --- a/tests/test_apis.py +++ b/tests/test_apis.py @@ -56,7 +56,7 @@ def test_getMatchPickle(): for chunk in res.iter_content(chunk_size=4096): data = data + chunk jsd = json.loads(data) - assert len(jsd) == 62 + assert len(jsd) == 64 assert len(jsd['observations']) == 5 else: assert False diff --git a/using_vault.sh b/using_vault.sh deleted file mode 100644 index b87bd80d5..000000000 --- a/using_vault.sh +++ /dev/null @@ -1,49 +0,0 @@ -# testing vault - -# set the default to JSON exports -export VAULT_FORMAT="json" - -# create an approle thats bound to your cidr blocks - vault write auth/approle/role/usermaint \ - secret_id_ttl=10m \ - token_num_uses=10 \ - token_ttl=20m \ - token_max_ttl=30m \ - secret_id_num_uses=40 \ - secret_id_bound_cidrs="192.168.1.0/24" - -# a policy must exist to allow the role to access the secret -echo 'path "secret/test" { capabilities = ["list"]}' > usermaint.hcl -echo 'path "secret/test" { capabilities = ["read"] }' >> usermaint.hcl -vault policy write usermaint usermaint.hcl -vault write auth/approle/role/usermaint policies=usermaint - -# a token must exist for the server to initially authenticate. -# This needs a policy which grants access the approle -# in a corporate environment this step would be done with LDAP or AD authentication - -echo 'path "auth/approle/role" { capabilities = ["list"]}' > server1.hcl -echo 'path "auth/approle/role/usermaint/*" { capabilities = ["read", "update"] }' >> server1.hcl -vault policy write server1 server1.hcl -servertoken=$(vault token create -policy=server1 -format=json | jq -r .auth.client_token) - - -# now on the target server we can use this unprivileged token to retrieve a more privileged one -export VAULT_TOKEN=$servertoken -# get the role ID - this could be statically stored, its not considered Secret -vault_role_id=$(vault read -format=json auth/approle/role/usermaint/role-id | jq -r .data.role_id) - -# then either get a wrapped token and unwrap it to get a secret -# this is more secure because the wrapped token is single-use and expires after 120s -wraptoken=$(vault write -format=json -wrap-ttl=120s -f auth/approle/role/usermaint/secret-id | jq -r .wrap_info.token) -vault_secret_id=$(vault unwrap -format=json $wraptoken | jq -r .data.secret_id) - -# or just straight up get the secret. -vault_secret_id=$(vault write -format=json -force auth/approle/role/usermaint/secret-id | jq -r .data.secret_id) - -# then login to get a token with permission to read the actual secret -vault_token=$(vault write -format=json auth/approle/login role_id=$vault_role_id secret_id=$vault_secret_id | jq -r .auth.client_token) - -# now you can use vault_token in subsequent calls eg -VAULT_TOKEN=$vault_token vault kv get -format=json -mount=secret test - diff --git a/utils/reimageBatchServer.ps1 b/utils/reimageBatchServer.ps1 new file mode 100644 index 000000000..8f513e4df --- /dev/null +++ b/utils/reimageBatchServer.ps1 @@ -0,0 +1,9 @@ +# script to re-image the UKMON calc server +# copyright Mark McIntyre, 2026- + +$dtstr=(get-date -uformat "%Y%m%d") + +$instdetails=(aws ec2 describe-instances --filters Name="tag:Name",Values="batchserver" --profile ukmonshared --region eu-west-2) +$instanceid= (Write-Output $instdetails| convertfrom-json)[0].reservations.instances.instanceid + +aws ec2 create-image --instance-id $instanceid --name "batchserver_${dtstr}" --description "Latest Batchserver image" --profile ukmonshared --region eu-west-2 \ No newline at end of file diff --git a/utils/reimageCalcServer.ps1 b/utils/reimageCalcServer.ps1 new file mode 100644 index 000000000..9f776af31 --- /dev/null +++ b/utils/reimageCalcServer.ps1 @@ -0,0 +1,9 @@ +# script to re-image the UKMON calc server +# copyright Mark McIntyre, 2026- + +$dtstr=(get-date -uformat "%Y%m%d") + +$instdetails=(aws ec2 describe-instances --filters Name="tag:Name",Values="calcengine_ub" --profile ukmonshared --region eu-west-2) +$instanceid= (Write-Output $instdetails| convertfrom-json)[0].reservations.instances.instanceid + +aws ec2 create-image --instance-id $instanceid --name "calcengine_${dtstr}" --description "Latest Calc Engine image" --profile ukmonshared --region eu-west-2 \ No newline at end of file