diff --git a/README.md b/README.md index 595de7c54..9f059c11c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2026.04.1 +version: 2026.05.0 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/api_examples/README.md b/api_examples/README.md new file mode 100644 index 000000000..4de90ef88 --- /dev/null +++ b/api_examples/README.md @@ -0,0 +1,6 @@ +#Available APIs and Examples + +We have a number of APIs which can be used to retrieve our data programatically, either using commandline tools like curl and wget, or from programming languages such as Python. + +Data are returned in JSON format which can readily be converted into a python list, array or Pandas DataFrame. Simple examples are shown on our main website (here)[https://ukmeteornetwork.org/our-data-apis/], and a set of Python examples are available here. A Data Dictionary in Excel format can be downloaded from (here)[https://archive.ukmeteors.co.uk/browse/datadictionary.xlsx]. + diff --git a/archive/README.md b/archive/README.md index 108c7fbf6..c37b478af 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,7 +1,7 @@ Data Processing and Flows ========================== -version: 2026.04.1 +version: 2026.05.0 This diagram shows the overall flow of data from Cameras to websites and out to the public. diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 2fba436f8..59872560d 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -23,12 +23,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" findAllMatches +logger -s -t findAllMatches "starting" [ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d) @@ -52,40 +47,39 @@ mkdir -p $SRC/logs/distrib > /dev/null 2>&1 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000') -log2cw $NJLOGGRP $NJLOGSTREAM "solving for ${startdt} to ${enddt}" findAllMatches -log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches -$SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND +logger -s -t findAllMatches "solving for ${startdt} to ${enddt}" +logger -s -t findAllMatches "start runDistrib" -log2cw $NJLOGGRP $NJLOGSTREAM "clean duplicate/deleted trajs" findAllMatches +$SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND $SRC/utils/cleanupDeletedTrajs.sh -log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches +logger -s -t findAllMatches "Solving Run Done" + success=$(grep "Total run time:" $SRC/logs/matchJob.log) if [ "$success" == "" ] then - python -c "from meteortools.utils import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')" + python -c "from utils.sendAnEmail import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')" echo problems with solver fi -log2cw $NJLOGGRP $NJLOGSTREAM "Solving Run Done" findAllMatches -log2cw $NJLOGGRP $NJLOGSTREAM "start rerunFailedLambdas" findAllMatches python -m maintenance.rerunFailedLambdas cd $here -log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches + +logger -s -t findAllMatches "start reportOfLatestMatches" + matchlog=${SRC}/logs/matchJob.log -python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate -python -m metrics.getMatchStats $matchlog +python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $rundate +python -m metrics.getMatchStats $matchlog $rundate # copy stats to S3 so the daily report can run if [ "$RUNTIME_ENV" == "PROD" ] ; then aws s3 sync $DATADIR/dailyreports/ $UKMONSHAREDBUCKET/matches/RMSCorrelate/dailyreports/ --quiet fi -log2cw $NJLOGGRP $NJLOGSTREAM "start purgeLogs" findAllMatches find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \; find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \; -log2cw $NJLOGGRP $NJLOGSTREAM "finished findAllMatches" findAllMatches +logger -s -t findAllMatches "finished" diff --git a/archive/analysis/getLogData.sh b/archive/analysis/getLogData.sh index 1f46b6962..67342a511 100644 --- a/archive/analysis/getLogData.sh +++ b/archive/analysis/getLogData.sh @@ -10,7 +10,7 @@ if [ "$1" != "" ] ; then logfile=$DATADIR/lastlogs/lastlog-${rundate}.html else rundate=$(date +%Y%m%d) - logfile=$DATADIR/lastlog.html + logfile=$DATADIR/lastlogs/lastlog.html fi # create performance metrics @@ -78,7 +78,7 @@ echo "

> $logfile -if [ ! -d $DATADIR/lastlogs ] ; then mkdir $DATADIR/lastlogs ; fi +mkdir -p $DATADIR/lastlogs if [ "$1" == "" ] ; then aws s3 cp $logfile $WEBSITEBUCKET/reports/ --quiet cp $logfile $DATADIR/lastlogs/lastlog-${rundate}.html @@ -92,3 +92,5 @@ ls -1r $DATADIR/lastlogs/last*.html | head -90 | while read i ; do done cat $TEMPLATES/footer.html >> $DATADIR/lastlogs/index.html aws s3 cp $DATADIR/lastlogs/index.html $WEBSITEBUCKET/reports/lastlogs/ --quiet + +find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \; diff --git a/archive/analysis/reportActiveShowers.sh b/archive/analysis/reportActiveShowers.sh index a42c90d3e..e26e04638 100644 --- a/archive/analysis/reportActiveShowers.sh +++ b/archive/analysis/reportActiveShowers.sh @@ -30,7 +30,7 @@ fi logger -s -t reportActiveShowers "report on active showers" python -m reports.reportActiveShowers -m -python -c "from meteortools.utils import getActiveShowers; getActiveShowers('$rundt', inclMinor=True)" | while read shwr +python -c "from utils.getActiveShowers import getActiveShowers;getActiveShowers('$rundt', inclMinor=True)" | while read shwr do aws s3 sync $DATADIR/reports/${yr}/$shwr $WEBSITEBUCKET/reports/${yr}/${shwr} --quiet done diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 9350e5db3..1cf23ab35 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -20,129 +20,110 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "starting runDistrib" runDistrib - -# set the profile to the UKMDA account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared +logger -s -t runDistrib "starting runDistrib" if [ $# -gt 0 ] ; then if [ "$1" != "" ] ; then - log2cw $NJLOGGRP $NJLOGSTREAM "selecting range" runDistrib MATCHSTART=$1 fi if [ "$2" != "" ] ; then MATCHEND=$2 else - log2cw $NJLOGGRP $NJLOGSTREAM "matchend was not supplied, using 2 days" runDistrib MATCHEND=$(( $MATCHSTART - 2 )) fi fi begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" runDistrib +logger -s -t runDistrib "running phase 1 for dates ${begdate} to ${rundate}" +logger -s -t runDistrib "start correlation server" aws ec2 start-instances --instance-ids $SERVERINSTANCEID stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) while [ "$stat" -ne 16 ]; do sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" runDistrib stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate}" runDistrib - conda activate $HOME/miniconda3/envs/${WMPL_ENV} -log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" runDistrib +logger -s -t runDistrib "creating the run script" execdist=execdistrib.sh execMatchingsh=/tmp/$execdist python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE chmod +x $execMatchingsh -log2cw $NJLOGGRP $NJLOGSTREAM "get server details" runDistrib -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -while [ "$privip" == "" ] ; do - sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" runDistrib - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -done - -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" runDistrib +logger -s -t runDistrib "deploy the script to the server $CALCSERVERIP and run it" -scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist +scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist + scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist done # push the python code and ECS templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/ShowerAssociation.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/utils/convertSolLon.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/utils/ # now run the script -log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist" +logger -s -t runDistrib "start distributed processing" +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execdist" -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" -log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib +logger -s -t runDistrib "job run, stop the server again" aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -log2cw $NJLOGGRP $NJLOGSTREAM "monitoring and waiting for completion" runDistrib +logger -s -t runDistrib "monitoring and waiting for completion" python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}', '${TESTMODE}'); " mkdir -p $DATADIR/distrib cd $DATADIR/distrib -log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib +logger -s -t runDistrib "restarting server to consolidate results" + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi -log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib while [ "$stat" -ne 16 ]; do sleep 30 if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi - log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) execcons=execconsol.sh execConsolsh=/tmp/$execcons python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', '$TESTMODE')" chmod +x $execConsolsh -log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib -scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$privip:data/distrib/$execcons -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execcons" +logger -s -t runDistrib "running consolidation" + +scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execcons +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execcons" -log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation, copying databases" runDistrib -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib +logger -s -t runDistrib "finished consolidation, copying databases" -# remote temporary files -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" -log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib +logger -s -t runDistrib "stopping calcserver again" aws ec2 stop-instances --instance-ids $SERVERINSTANCEID +logger -s -t runDistrib "copying data to batch server and tidying up" + # grab a copy of the indvidual container dbs so we can get a list of new solutions rm -Rf $DATADIR/latest/contdbs/ mkdir -p $DATADIR/latest/contdbs/ @@ -154,8 +135,6 @@ aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATAD aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $SRC/logs/distrib/ --exclude "*" --include "correl*.log" --exclude "logs/" --quiet aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "correl*.log" --exclude "logs/" --recursive --quiet -log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib - mkdir -p $DATADIR/trajdb tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db mkdir -p $DATADIR/distrib/containers @@ -168,7 +147,4 @@ find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \; find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \; rm -f $DATADIR/distrib/${rundate}.pickle - -# and then clear the profile again -unset AWS_PROFILE -log2cw $NJLOGGRP $NJLOGSTREAM "finished runDistrib" runDistrib +logger -s -t "finished runDistrib" diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index beed33373..1ef92a489 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -17,86 +17,67 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "starting updatePlotsAndDetStatus" updatePlotsAndDetStatus - -# set the profile to the EE account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared +logger -s -t updatePlotsAndDetStatus "starting updatePlotsAndDetStatus" if [ $# -gt 0 ] ; then if [ "$1" != "" ] ; then - log2cw $NJLOGGRP $NJLOGSTREAM "selecting range" updatePlotsAndDetStatus MATCHSTART=$1 fi if [ "$2" != "" ] ; then MATCHEND=$2 else - log2cw $NJLOGGRP $NJLOGSTREAM "matchend was not supplied, using 2 days" updatePlotsAndDetStatus MATCHEND=$(( $MATCHSTART - 2 )) fi fi begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "updating plots etc for dates ${begdate} to ${rundate}" +logger -s -t updatePlotsAndDetStatus "start correlation server" + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" updatePlotsAndDetStatus while [ "$stat" -ne 16 ]; do sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" updatePlotsAndDetStatus stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -log2cw $NJLOGGRP $NJLOGSTREAM "updating plots etc for dates ${begdate} to ${rundate}" updatePlotsAndDetStatus + conda activate $HOME/miniconda3/envs/${WMPL_ENV} -log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "creating the run script" execrerun=execreplot.sh execrerunsh=/tmp/$execrerun python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh', '$TESTMODE')" chmod +x $execrerunsh -log2cw $NJLOGGRP $NJLOGSTREAM "get server details" updatePlotsAndDetStatus -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -while [ "$privip" == "" ] ; do - sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" updatePlotsAndDetStatus - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -done - -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "deploy the script to the server $CALCSERVERIP and run it" -scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun +scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" updatePlotsAndDetStatus - scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun + scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun done # push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj # now run the script -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execrerun" +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execrerun" + +logger -s -t updatePlotsAndDetStatus "job run, stop the server again" -log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" updatePlotsAndDetStatus aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -log2cw $NJLOGGRP $NJLOGSTREAM "get a list of uncalibrated data" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "get a list of uncalibrated data" + aws s3 sync $UKMONSHAREDBUCKET/matches/consumed/ $DATADIR/single/used/ --exclude "*" --include "*.txt" --quiet rundate=$(cat $DATADIR/rundate.txt) -python -c "from analysis.gatherDetectionData import getUncalibratedImageList;getUncalibratedImageList('$rundate');" +python -c "from utils.getUncalImages import getUncalibratedImageList;getUncalibratedImageList('$rundate');" -# and then clear the profile again -unset AWS_PROFILE # refresh the website index pages just in case any new data dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) $SRC/website/updateIndexPages.sh $dailyrep -log2cw $NJLOGGRP $NJLOGSTREAM "finished updatePlotsAndDetStatus" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "finished updatePlotsAndDetStatus" diff --git a/archive/cronjobs/captureBright.sh b/archive/cronjobs/captureBright.sh index 06ce1561f..b06ed51d2 100644 --- a/archive/cronjobs/captureBright.sh +++ b/archive/cronjobs/captureBright.sh @@ -6,15 +6,9 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} cd $DATADIR/brightness rundt=$(date -d "yesterday" +%Y%m%d) -python -m analysis.compareBrightnessData $rundt +python -m utils.compareBrightnessData $rundt -mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD -use ukmon; -select count(*) from ukmon.brightness; -load data local infile '${DATADIR}/brightness/CaptureNight_${rundt}.csv' -into table brightness fields terminated by ','; -select count(*) from ukmon.brightness; -EOD +$SRC/utils/loadBrightnessCsvMDB.sh find ${DATADIR}/brightness -name "CaptureNight*" -mtime +30 -exec rm -f {} \; - +find ${DATADIR}/brightness -name "matcheddata*" -mtime +30 -exec rm -f {} \; \ No newline at end of file diff --git a/archive/cronjobs/gatherMonthlyVideos.sh b/archive/cronjobs/gatherMonthlyVideos.sh index 52f670486..e8c9c1464 100644 --- a/archive/cronjobs/gatherMonthlyVideos.sh +++ b/archive/cronjobs/gatherMonthlyVideos.sh @@ -70,7 +70,7 @@ echo "mkdir \$1 ; cd \$1" >> $outdir/getVideos.ps1 echo "foreach(\$line in Get-Content ../best_\$1.txt) { wget https://archive.ukmeteors.co.uk/\$line }" >> $outdir/getVideos.ps1 echo "cd .." >> $outdir/getVideos.ps1 -aws s3 sync $outdir $UKMONSHAREDBUCKET/videos/ --profile ukmonshared --quiet --exclude "*" --include "getVideos*" --include "README*" +aws s3 sync $outdir $UKMONSHAREDBUCKET/videos/ --quiet --exclude "*" --include "getVideos*" --include "README*" -aws s3 sync $outdir $WEBSITEBUCKET/data/bestvideos/ --profile ukmonshared --quiet --include "*.txt" +aws s3 sync $outdir $WEBSITEBUCKET/data/bestvideos/ --quiet --include "*.txt" diff --git a/archive/cronjobs/getImoWSfile.sh b/archive/cronjobs/getImoWSfile.sh index 021b6bad6..d7cb03996 100644 --- a/archive/cronjobs/getImoWSfile.sh +++ b/archive/cronjobs/getImoWSfile.sh @@ -19,7 +19,7 @@ git checkout wmpl/share/streamfulldata.csv #git checkout wmpl/share/ShowerLookUpTable.txt git checkout wmpl/share/gmn_shower_table_20230518.txt -python -c "from meteortools.utils.getShowerDates import numpifyShowerData; numpifyShowerData()" +python -c "from utils.getActiveShowers import numpifyShowerData; numpifyShowerData()" logger -s -t getImoWSfile "finished" diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index 1c9eb1204..5d63a02aa 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -7,12 +7,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) -export NJLOGSTREAM -aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared - -# log2cw is a function defined in config.ini and logs to cloudwatch, syslog and console -log2cw $NJLOGGRP $NJLOGSTREAM "start nightlyJob" nightlyJob +logger -s -t nightlyJob "start nightlyJob" # dates to process for rundate=$(date +%Y%m%d) @@ -25,139 +20,118 @@ mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} -# update the JSON file containing camera location details. This is used by the website -log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera location/dir/fov database" nightlyJob -python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();" -aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --profile ukmonshared --quiet -aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --profile ukmonshared --quiet - -# create the CSV file of camera info and the html versions for search functions on the website -log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera details files for searching" nightlyJob -python -c "from reports.CameraDetails import createCDCsv; createCDCsv('consolidated');" -aws s3 cp $DATADIR/consolidated/camera-details.csv $UKMONSHAREDBUCKET/consolidated/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/statopts.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/activestatopts.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/activestatlocs.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet +#################################################################################### +# START OF DATA PROCESSING. FIRST WE UPDATE THE SEARCH PAGE WITH SINGLE-STATION +# AND CAMERA DETAILS SO IT CAN BE SEARCHED WITHOUT WAITING FOR THE MATCHING ENGINE +#################################################################################### -# set up logging for the match process -log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" nightlyJob -matchlog=matchJob.log -if [ -f $SRC/logs/$matchlog ] ; then - suff=$(stat matchJob.log -c %X) - mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff -fi +# update the JSON and html files containing camera and location details, used by the website +$SRC/website/updateCameraDets.sh -log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" nightlyJob -# this creates the parquet table for Athena +# consolidate the single-station data - we do this here so the search index can be updated earlier +# and we can search for single-station events without waiting for the main batch $SRC/analysis/getRMSSingleData.sh if [ "$(date +%m%d)" == "0101" ] ; then # catch any data uploaded on 01/01 that is for 31/12 the previous year $SRC/analysis/getRMSSingleData.sh $(date -d 'last year' +%Y) fi -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" nightlyJob - +# now update the search index with single-station data $SRC/analysis/createSearchable.sh $yr singles +#################################################################################### +# NOW THE RUN THE MATCHING ENGINE VIA findAllmatches.sh +# Take care rerunning it as it will re-create the daily report +#################################################################################### + +# set up logging for the match process +matchlog=matchJob.log + +# save the existing log, in case the process is being rerun on the same day +if [ "$(find $SRC/logs -name $matchlog -mmin +1380 -ls)" != "" ] ; then + dt=$(stat $SRC/logs/$matchlog -c %y) + suff=$(date --date ${dt:0:10} +%Y%m%d) + mv -f $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff +fi + +logger -s -t nightlyJob "start findAllMatches" # Run the match process - run this only once as it scoops up all unprocessed data ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 -log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" nightlyJob + +#################################################################################### +# FROM HERE DOWN WE'RE CONSOLIDATING DATA AND CREATING REPORTS +# and everything can be rerun safely provided the data are present +#################################################################################### + +# update the website daily, monthly and annual index pages where needed $SRC/website/updateIndexPages.sh -# FIRST CONSOLIDATE THE DATA AND CREATE SEARCH INDEXES -# consolidate the output of the match process for further analysos -log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob +# consolidate the output of the match process for further analysis $SRC/analysis/consolidateOutput.sh ${yr} if [ "$(date +%m%d)" == "0101" ] ; then # catch any data uploaded on 01/01 that is for 31/12 the previous year $SRC/analysis/consolidateOutput.sh $(date -d 'last year' +%Y) fi -# create the search indexes used on the website -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 2" nightlyJob +# update the search indexes used on the website $SRC/analysis/createSearchable.sh $yr matches -# FROM HERE DOWN WE'RE CREATING REPORTS - # add daily report to the website -log2cw $NJLOGGRP $NJLOGSTREAM "start publishDailyReport" nightlyJob $SRC/website/publishDailyReport.sh # create monthly and per-shower CSV extracts of the data -log2cw $NJLOGGRP $NJLOGSTREAM "start createMthlyExtracts" nightlyJob ${SRC}/website/createMthlyExtracts.sh ${mth} - -log2cw $NJLOGGRP $NJLOGSTREAM "start createShwrExtracts" nightlyJob ${SRC}/website/createShwrExtracts.sh ${rundate} # create the fireballs page -log2cw $NJLOGGRP $NJLOGSTREAM "start createFireballPage" nightlyJob #requires search index to have been updated first ${SRC}/website/createFireballPage.sh ${yr} -3.99 # create a report of activity for the current month and whole year -log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $mth" nightlyJob $SRC/analysis/showerReport.sh ALL ${mth} force - -log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $yr" nightlyJob $SRC/analysis/showerReport.sh ALL ${yr} force # if we ran on the 1st of the month we need to catch any late-arrivals for last month if [ $(date +%d) -eq 1 ] ; then lastmth=$(date -d '-1 month' +%Y%m) - log2cw $NJLOGGRP $NJLOGSTREAM "start showerMthlyExtracts ALL $lastmth" nightlyJob ${SRC}/website/createMthlyExtracts.sh ${lastmth} - log2cw $NJLOGGRP $NJLOGSTREAM "start createShwrExtracts ALL $lastmth" nightlyJob ${SRC}/website/createShwrExtracts.sh ${lastmth}28 - log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $lastmth" nightlyJob $SRC/analysis/showerReport.sh ALL ${lastmth} force fi # create a per-shower report for any currently active showers -log2cw $NJLOGGRP $NJLOGSTREAM "start reportActiveShowers" nightlyJob ${SRC}/analysis/reportActiveShowers.sh ${yr} # create the website front page -log2cw $NJLOGGRP $NJLOGSTREAM "start createSummaryTable" nightlyJob ${SRC}/website/createSummaryTable.sh # create the camera status reports -log2cw $NJLOGGRP $NJLOGSTREAM "start cameraStatusReport" nightlyJob ${SRC}/website/cameraStatusReport.sh $rundate -log2cw $NJLOGGRP $NJLOGSTREAM "start createExchangeFiles" nightlyJob +# create the files for exchange with other networks - i think this is unused python -c "from reports.createExchangeFiles import createAll;createAll();" -aws s3 sync $DATADIR/browse/daily/ $WEBSITEBUCKET/browse/daily/ --region eu-west-2 --quiet +aws s3 sync $DATADIR/browse/daily/ $WEBSITEBUCKET/browse/daily/ --quiet cd $DATADIR # do this manually when on PC required as it requires too much memory for the batch server; closes #61 # python $PYLIB/maintenance/plotStationsOnMap.py False -aws s3 cp $DATADIR/stations.png $WEBSITEBUCKET/ --region eu-west-2 --quiet +aws s3 cp $DATADIR/stations.png $WEBSITEBUCKET/ --quiet rm -f $SRC/data/.nightly_running # various reports for management - bad stations, costs, next batch start time. -log2cw $NJLOGGRP $NJLOGSTREAM "start getBadStations" nightlyJob $SRC/analysis/getBadStations.sh - -log2cw $NJLOGGRP $NJLOGSTREAM "start costReport" nightlyJob $SRC/website/costReport.sh - -# set time of next run python $PYLIB/maintenance/getNextBatchStart.py 150 # create station reports. This takes a while hence done after everything else -log2cw $NJLOGGRP $NJLOGSTREAM "start stationReports" nightlyJob $SRC/analysis/stationReports.sh # clear down space where possible -log2cw $NJLOGGRP $NJLOGSTREAM "start clearSpace" nightlyJob $SRC/utils/clearSpace.sh # load the MariaDB with the latest data. The mariadb database isn't used much -log2cw $NJLOGGRP $NJLOGSTREAM "update MariaDB tables" nightlyJob $SRC/utils/loadMatchCsvMDB.sh $SRC/utils/loadSingleCsvMDB.sh @@ -165,12 +139,10 @@ $SRC/utils/loadSingleCsvMDB.sh # has to be run quite late as not all trajectories have synced to the website earlier $SRC/analysis/updatePlotsAndDetStatus.sh +# push the API data dictionary to the website for end-user use aws s3 sync $SRC/share/ s3://ukmda-website/browse --exclude "*" --include "datadictionary.xlsx" --quiet -log2cw $NJLOGGRP $NJLOGSTREAM "finished nightlyJob" nightlyJob +logger -s -t nightlyJog "finished nightlyJob" # grab the logs for the website - run this last to capture the above Finished message $SRC/analysis/getLogData.sh - - - diff --git a/archive/database/Setup.md b/archive/database/Setup.md index 1aef81b10..328cab345 100644 --- a/archive/database/Setup.md +++ b/archive/database/Setup.md @@ -1,42 +1,16 @@ # Installing MariaDB -## Create /etc/yum.repos.d/MariaDB.repo - -[mariadb] -name = MariaDB -baseurl = http://yum.mariadb.org/10.3/centos7-amd64 -gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB -gpgcheck=1 - -## Install packages - -yum update -yum install mariadb-server mariadb-client mariadb-devel mariadb-shared mariadb-common - -## Create Databases and Tables -``` -mysql -u root -h localhost -set root password with -update mysql.user set password= PASSWORD('newpassword') where user = 'root'; -flush privileges; -exit; -``` -execute contents of create_dbs_and_users.sql -execute contents of create_tables.sql - -## migrating to new server -make sure you've installed mariadb on the new server and created databases and users -make sure you can ssh from old to new -then on the old server run -``` -mysqldump -u batch -p'passwd' ukmon | ssh ukmonhelper2 mysql -u batch -p'passwd' ukmon -``` +## Installing and Migrating +For installation and migration instructions see `migratingBatchServer.md` in the `server_setup` folder ## Connecting from Python +These packages are installed as part of the ukmda tooling. +``` bash pip install mysql-connector-python pip install pymysql - +``` +``` python import pymysql.cursors connection = pymysql.connect(host='localhost', @@ -46,12 +20,13 @@ pip install pymysql cursorclass=pymysql.cursors.DictCursor) try: with connection.cursor() as cursor: - sql = "INSERT INTO books VALUES ({},'{}', {}, {})".format(sys.argv[1],'foo',34,56) + #sql = "INSERT INTO books VALUES ({},'{}', {}, {})".format(sys.argv[1],'foo',34,56) cursor.execute(sql) connection.commit() - sql = "SELECT * from `books` " + sql = "SELECT * from matches limit 10 " cursor.execute(sql) result = cursor.fetchall() print(result) finally: connection.close() +``` \ No newline at end of file diff --git a/archive/database/ddl/create_tables.sql b/archive/database/ddl/create_tables.sql index c51027569..ba036622e 100644 --- a/archive/database/ddl/create_tables.sql +++ b/archive/database/ddl/create_tables.sql @@ -28,8 +28,9 @@ create table singles( AngVel float, Shwr varchar(6), filname varchar(64), - Dtstamp double(18,6)), - status varchar(3); + Dtstamp double(18,6), + status varchar(3) + ); alter ignore table ukmon.singles add unique uniq_Dtstamp (Dtstamp); diff --git a/archive/deployment/analysis.yml b/archive/deployment/analysis.yml deleted file mode 100644 index f65068693..000000000 --- a/archive/deployment/analysis.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/analysis exists - file: path={{destdir}}/analysis state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/analysis/updatePlotsAndDetStatus.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/consolidateOutput.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/createSearchable.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/findAllMatches.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getBadStations.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getLogData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getRMSSingleData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/reportActiveShowers.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/runDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/showerReport.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/stationReports.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/templates/extracsv.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/match_hdr_full.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/UA_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/UO_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/ukmon-single.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/README.md', dest: '{{destdir}}/analysis', mode: '644', backup: no } diff --git a/archive/deployment/config.yml b/archive/deployment/config.yml deleted file mode 100644 index 88ad1408a..000000000 --- a/archive/deployment/config.yml +++ /dev/null @@ -1,29 +0,0 @@ -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: dev-vars.yml - tags: dev - - name: import prod variables - include_vars: prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/utils exists - file: path={{destdir}}/utils state=directory - tags: [dev,prod] - - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/utils/makeConfig.sh', dest: '{{destdir}}/utils/', mode: '754', backup: yes } - - {src: '{{srcdir}}/utils/stopstart-calcengine.sh', dest: '{{destdir}}/utils/', mode: '754', backup: yes } - - {src: '{{srcdir}}/server_setup/.bashrc', dest: '/home/ec2-user', mode: '644', backup: yes } - - {src: '{{srcdir}}/server_setup/.bash_aliases', dest: '/home/ec2-user', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/.condaon', dest: '/home/ec2-user', mode: '644', backup: no } - - - name: create Config file - shell: '{{destdir}}/utils/makeConfig.sh {{env}}' - tags: [dev,prod] diff --git a/archive/deployment/cronjobs.yml b/archive/deployment/cronjobs.yml deleted file mode 100644 index 6a80c1b5e..000000000 --- a/archive/deployment/cronjobs.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/Dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/cronjobs exists - file: path={{destdir}}/cronjobs state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/cronjobs/nightlyJob.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/captureBright.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/getImoWSfile.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/mergeNewOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/deleteDuplicateOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/gatherMonthlyVideos.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/README.md', dest: '{{destdir}}/cronjobs', mode: '644', backup: no } diff --git a/archive/deployment/database.yml b/archive/deployment/database.yml deleted file mode 100644 index 3abf9d787..000000000 --- a/archive/deployment/database.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/database exists - file: path={{destdir}}/database state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - - {src: '{{srcdir}}/database/ddl/create_brightness_table.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/ddl/create_dbs_and_users.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/ddl/create_tables.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/dml/load_data.sql', dest: '{{destdir}}/database/dml', mode: '644', backup: no } - - {src: '{{srcdir}}/database/Setup.md', dest: '{{destdir}}/database', mode: '644', backup: no } - - {src: '{{srcdir}}/database/testDB.py', dest: '{{destdir}}/database', mode: '644', backup: no } - - {src: '{{srcdir}}/database/README.md', dest: '{{destdir}}/database', mode: '644', backup: no } diff --git a/archive/deployment/dev-vars.yml b/archive/deployment/dev-vars.yml deleted file mode 100644 index 86dfcf575..000000000 --- a/archive/deployment/dev-vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -destdir: /home/ec2-user/dev -env: DEV -s3profile: default diff --git a/archive/deployment/full_deployment.yml b/archive/deployment/full_deployment.yml deleted file mode 100644 index b682b18d1..000000000 --- a/archive/deployment/full_deployment.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- import_playbook: cronjobs.yml -- import_playbook: analysis.yml -- import_playbook: website.yml -- import_playbook: database.yml -- import_playbook: pylib.yml -- import_playbook: server_setup.yml -- import_playbook: shwrinfo.yml -- import_playbook: utils.yml diff --git a/archive/deployment/prod-vars.yml b/archive/deployment/prod-vars.yml deleted file mode 100644 index 20171b912..000000000 --- a/archive/deployment/prod-vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -destdir: /home/ec2-user/prod -env: PROD -s3profile: ukmda_admin \ No newline at end of file diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml deleted file mode 100644 index 313173390..000000000 --- a/archive/deployment/pylib.yml +++ /dev/null @@ -1,112 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/ukmon_pylib exists - file: path={{destdir}}/ukmon_pylib state=directory - tags: [dev,prod] - - name: clean local filesystem - ansible.builtin.command: 'pyclean {{srcdir}}/ukmon_pylib -v --debris all' - delegate_to: localhost - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - - {src: '{{srcdir}}/ukmon_pylib/analysis/compareBrightnessData.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/gatherDetectionData.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/meteorFlux.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/showerAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/stationAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/summaryAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/__init__.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/README.md', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/converters/gmnTxtToPandas.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/toParquet.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/__init__.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/README.md', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/maintenance/compressOldArchiveData.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/cleanDdbTables.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/dataMaintenance.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/deDuplicate.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/getNextBatchStart.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/getUserAndKeyInfo.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/hcVaultActions.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/manageTraj.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/plotStationsOnMap.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/purgeOldImages.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/purgeOldOrbits.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/recreateOrbitPages.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/rerunFailedLambdas.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/sortToDateDirs.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/README.md', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/metrics/camMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/costMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/getMatchStats.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/statsToMqtt.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/timingMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/volumeMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/README.md', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/reports/CameraDetails.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/cameraStatusReport.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createAnnualBarChart.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createExchangeFiles.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createSearchableFormat.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createSummaryTable.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/dailyReport.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/extractors.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/filterBylocDirBri.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findBestMp4s.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findFailedMatches.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findFireballs.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/getLivestreamData.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/getSolutionStati.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/makeCoverageMap.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportActiveShowers.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportBadCameras.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportOfLatestMatches.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/trackThumbnails.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/__init__.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/README.md', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/requirements.txt', dest: '{{destdir}}/ukmon_pylib', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/README.md', dest: '{{destdir}}/ukmon_pylib', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BM.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMNG_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMUK_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMWE_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BM_medres.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/images.json', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/population.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/README.md', dest: '{{destdir}}/ukmon_pylib/share', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mm.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetailstest-ukmda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/consolidateDistTraj.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/createDistribMatchingSh.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/distributeCandidates.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/gatherMissedPlatepars.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/manualBulkReruns.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/pickleAnalyser.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/plotOSMGroundTrack.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/ShowerAssociation.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner_test.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/README.md', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } diff --git a/archive/deployment/server_setup.yml b/archive/deployment/server_setup.yml deleted file mode 100644 index f53ea3a7b..000000000 --- a/archive/deployment/server_setup.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/server_setup exists - file: path={{destdir}}/server_setup state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/server_setup/copyTestData.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-cartopy.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-geos.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-proj4.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-sqlite3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install_opencv.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/migratingBatchServer.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/newserver.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/remount-s3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/ssh-setup.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/yum-installs.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/README.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } diff --git a/archive/deployment/shwrinfo.yml b/archive/deployment/shwrinfo.yml deleted file mode 100644 index c7bbfd11f..000000000 --- a/archive/deployment/shwrinfo.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/data/shwrinfo exists - file: path={{destdir}}/data/shwrinfo state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/shwrinfo/AUR.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/CAP.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/COM.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/costnotes.html', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/DAD.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/ETA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/GEM.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/HYD.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/KCG.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/LEO.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/LYR.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/NOO.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/NTA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/ORI.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/PER.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/QUA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/SDA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/STA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/template.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/URS.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/README.md', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } diff --git a/archive/deployment/utils.yml b/archive/deployment/utils.yml deleted file mode 100644 index 73dd5f02f..000000000 --- a/archive/deployment/utils.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/utils exists - file: path={{destdir}}/utils state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/utils/calcserver/clearSpace.sh', dest: '{{destdir}}/utils/calcserver', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/cftpd_templ.json', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/checkAndRollKeys.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/clearCaches.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/clearSpace.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/createTestDataSet.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/deleteOrbit.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/getCostMetrics.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/liveevent.template', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/loadMatchCsvMDB.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/loadSingleCsvMDB.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/makeConfig.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/monthlyCleardown.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/rerunFTPtoUKMONlambda.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/rerunGetExtraFiles.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/ses-msg-templ.json', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/statsToMqtt.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/stopstart-calcengine.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateDb.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateFireballFlag.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateFireballImage.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/userAudit.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/README.md', dest: '{{destdir}}/utils', mode: '644', backup: no } diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml deleted file mode 100644 index 78956ddaa..000000000 --- a/archive/deployment/website.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/website exists - file: path={{destdir}}/website state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/website/cameraStatusReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/costReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createFireballPage.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createMthlyExtracts.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createOrbitIndex.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createReportIndex.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createShwrExtracts.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createSummaryTable.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/publishDailyReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/pushStatic.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/README.md', dest: '{{destdir}}/website', mode: '644', backup: no } - - {src: '{{srcdir}}/website/updateIndexPages.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - - {src: '{{srcdir}}/website/templates/coverage-maps.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/fbreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/footer.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/frontpage.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/header.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/reportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/shwrcsvindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/statreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } diff --git a/archive/deployment/website_static.yml b/archive/deployment/website_static.yml deleted file mode 100644 index c15044f48..000000000 --- a/archive/deployment/website_static.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/static_content exists - file: path={{destdir}}/static_content state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/static_content/browse/', dest: '{{destdir}}/static_content/browse', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/css/', dest: '{{destdir}}/static_content/css', mode: '754', backup: no, directory_mode: yes } - #- {src: '{{srcdir}}/static_content/data/', dest: '{{destdir}}/static_content/data', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/docs/', dest: '{{destdir}}/static_content/docs', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/fonts/', dest: '{{destdir}}/static_content/fonts', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/img/', dest: '{{destdir}}/static_content/img', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/js/', dest: '{{destdir}}/static_content/js', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/latest/', dest: '{{destdir}}/static_content/latest', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/live/', dest: '{{destdir}}/static_content/live', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/search/', dest: '{{destdir}}/static_content/search', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/templates/', dest: '{{destdir}}/static_content/templates', mode: '754', backup: no, directory_mode: yes } - - - {src: '{{srcdir}}/static_content/favicon.ico', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/fundraising.html', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/privacy_data.html', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/README.md', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/robots.txt', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - - #- name: Update Static Content on the S3 website bucket - # shell: '{{destdir}}/website/pushStatic.sh {{env}}' - # tags: [dev,prod] diff --git a/archive/lambdas/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py index 3000292de..3f4d1f178 100644 --- a/archive/lambdas/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -67,7 +67,7 @@ def getStationData(statid, dtstr, period=None): statfrag = f"and s.stations like '%{statid}%' " if statid is not None else "" perfrag = periodToSqlFragment(period) if period is not None else "" with connection.cursor() as cursor: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}" + sql = f"SELECT s.orbname from matches s where s.orbname like '{dtstr}%' {statfrag} {perfrag}" cursor.execute(sql) result = cursor.fetchall() finally: @@ -81,11 +81,11 @@ def getStationData(statid, dtstr, period=None): def getSummaryData(dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() perfrag = periodToSqlFragment(period) if period is not None else "" - connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' - expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%' {perfrag}" + expr = f"SELECT {fieldlist} from matches s where s.orbname like '{dtstr}%' {perfrag}" result=[] try: with connection.cursor() as cursor: @@ -171,7 +171,7 @@ def lambda_handler(event, context): res = '{"points": "unavailable"}' else: - res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}' + res = '{"invalid request type - must be one of \'matches\', \'detail\', \'station\',\'summary\'"}' print(res) return { diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index 00d569938..fa6eb3529 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -4,6 +4,7 @@ alias h='history' alias df='df -h' alias du='du -h' +alias ls='ls --color=auto' alias data='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/data && pwd ; fi' alias logs='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/logs && pwd ; fi' @@ -16,28 +17,30 @@ alias stats='if [ "$DATADIR" == "" ] ; then echo select env first; else tail $DA alias matchstatus='if [ "$SRC" == "" ] ; then echo select env first; else grep "Running" $SRC/logs/matchJob.log && grep TRAJ $SRC/logs/matchJob.log | grep SOLVING && echo -n "Completed " && grep Observations: $SRC/logs/matchJob.log | wc -l && grep "nightlyJob" $SRC/logs/nightlyJob.log ; fi ' alias spacecalc='ls -1 | egrep -v "ukmon-shared" | while read i ; do \du -s $i ; done | sort -n' -alias startcalc='$SRC/utils/stopstart-calcengine.sh start' -alias stopcalc='$SRC/utils/stopstart-calcengine.sh stop' +alias startcalc='$SRC/utils/stopstartCalcengine.sh start' +alias stopcalc='$SRC/utils/stopstartCalcengine.sh stop' function dev { - source ~/dev/config.ini >/dev/null - conda activate $HOME/miniconda3/envs/${WMPL_ENV} - PS1="(wmpl) (dev) [\W]\$ " + source ~/dev/config.ini + #conda activate $HOME/miniconda3/envs/${WMPL_ENV} + conda activate ${WMPL_ENV} + PS1="(dev) [\W]\$ " cd ~/dev } function prd { - source ~/prod/config.ini >/dev/null - conda activate $HOME/miniconda3/envs/${WMPL_ENV} - PS1="(wmpl) (prd) [\W]\$ " + source ~/prod/config.ini #>/dev/null + #conda activate $HOME/miniconda3/envs/${WMPL_ENV} + conda activate ${WMPL_ENV} + PS1="(prd) [\W]\$ " cd ~/prod } function calcserver { - sts=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State --output text --profile ukmonshared) - ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) + sts=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State --output text) + ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) isrunning=$(echo $sts | cut -d " " -f 1) if [ $isrunning -ne 16 ] ; then - $HOME/prod/utils/stopstart-calcengine.sh start + $HOME/prod/utils/stopstartCalcengine.sh start echo "starting server on ${ipaddr}... waiting 10s..." sleep 10 fi diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index 79e18c870..a7f39fa35 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -42,15 +42,27 @@ export PATH=$PATH:$(dirname $(which node)) # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! -__conda_setup="$('/home/ec2-user/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +__conda_setup="$('/home/ubuntu/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" else - if [ -f "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" ]; then - . "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" + if [ -f "/home/ubuntu/miniconda3/etc/profile.d/conda.sh" ]; then + . "/home/ubuntu/miniconda3/etc/profile.d/conda.sh" else - export PATH="/home/ec2-user/miniconda3/bin:$PATH" + export PATH="/home/ubuntu/miniconda3/bin:$PATH" fi fi unset __conda_setup # <<< conda initialize <<< + +# If this is an xterm set the title to user@host:dir +case "$TERM" in + xterm*|rxvt*) + PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" + ;; + *) + ;; +esac +export rundate=$(date +%Y%m%d) +conda activate wmpl + diff --git a/archive/server_setup/.condaon b/archive/server_setup/.condaon index b69b186cd..a8766953f 100644 --- a/archive/server_setup/.condaon +++ b/archive/server_setup/.condaon @@ -1,13 +1,14 @@ # >>> conda initialize >>> -__conda_setup="$('/home/ec2-user/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +__conda_setup="$('~/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" else - if [ -f "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" ]; then - . "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" + if [ -f "~/miniconda3/etc/profile.d/conda.sh" ]; then + . "~/miniconda3/etc/profile.d/conda.sh" else - export PATH="/home/ec2-user/miniconda3/bin:$PATH" + export PATH="~/miniconda3/bin:$PATH" fi fi unset __conda_setup +. ~/miniconda3/etc/profile.d/conda.sh # <<< conda initialize <<< diff --git a/archive/server_setup/.vimrc b/archive/server_setup/.vimrc new file mode 100644 index 000000000..2fd7a2b6c --- /dev/null +++ b/archive/server_setup/.vimrc @@ -0,0 +1 @@ +colo slate diff --git a/archive/server_setup/logs.conf b/archive/server_setup/logs.conf new file mode 100644 index 000000000..910c3bf2b --- /dev/null +++ b/archive/server_setup/logs.conf @@ -0,0 +1,24 @@ +daily +rotate 7 +notifempty +~/prod/logs/*.log { + daily + rotate 45 + compress + delaycompress + dateext + notifempty + maxage 45 + # create +} + +~/dev/logs/*.log { + daily + rotate 45 + compress + delaycompress + dateext + notifempty + maxage 45 + # create +} diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh new file mode 100755 index 000000000..52d5add0f --- /dev/null +++ b/archive/server_setup/migrateSftpAccts.sh @@ -0,0 +1,44 @@ +#!/bin/bash + + +addOneUser() { + userid=$1 + srchost=$2 + grep -w $userid /etc/passwd + if [ $? -eq 1 ] ; then + dt=$(date +%Y-%m-%d) + logger -s -t addSftpUser "Creating unix user $userid" + sudo useradd --system --shell /usr/sbin/nologin --groups sftp --home /var/sftp/$userid --comment "${dt}" $userid + sudo mkdir /var/sftp/$userid + sudo chown root:sftp /var/sftp/$userid + sudo chmod 751 /var/sftp/$userid + # create the .ssh folder, platepar folder and empty client copy of the ini file + sudo mkdir /var/sftp/$userid/.ssh + sudo mkdir /var/sftp/$userid/platepar + sudo touch /var/sftp/$userid/ukmon.ini.client + # make these three writeable by the client + sudo chown $userid:$userid /var/sftp/$userid/platepar /var/sftp/$userid/.ssh /var/sftp/$userid/ukmon.ini.client + else + logger -s -t addSftpUser "Unix user $userid already exists" + fi + sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid + sudo cat /var/sftp/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini + sudo mv /tmp/$userid.ini /var/sftp/$userid/ukmon.ini + sudo scp /var/sftp/$userid/ukmon.ini $srchost:/var/sftp/$userid/ +} + +if [ $# -lt 2 ] ; then + echo "Usage: ./migrateSftpAccounts.sh oldservername userfile" + exit +fi + +echo "Warning: this must only be run on the new server" +read -p "press ctrl-c to quit or enter to continue" + +oldserver=$1 +srcfile=$2 + +cat $srcfile | while read stn +do + addOneUser $stn $oldserver +done \ No newline at end of file diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index 584a4d90b..1bf7db16e 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -1,68 +1,172 @@ # Replacing the UKMON helper server The ukmon helper server provides two services. -* camera authentication and key management * batch processing +* camera authentication and key management -# how to move batch processing -The best approach is to reinstall all the code using the ansible deployment scripts. -Both dev and prod envs should be created, though arguably the dev env should be on a separate server. -Additionally, WMPL will need to be reinstalled. -After doing this, the additional UKMON python requirements can be added. +## how to move batch processing +* Build a new Ubuntu server from the Terraform by cloning the existing batch server details in `terraform/ukmda/batchserver.tf` and making any necessary changes then deploying it with terraform in the normal way. +* I do not recommend using Amazon Linux as this doesn't contain many of the base requirements like C++, development libraries, GEOS and PROJ, so you would need to install and/or build these from source which is tedious. +* If you want to set the hostname to eg `batchserver1`, then do the following + * edit `/etc/cloud/cloud.cfg` and set `preserve_hostname: true` + * run `sudo hostnamctl hostname batchserver1` + * now the new hostname should be preserved through boot -The contents of ~/prod/data should be replicated to the new server. -The contents of ~/keymgmt should be replicated to the new server. +### Prerequisites +* install some prerequisites via apt +``` bash +sudo apt-get install unzip net-tools dos2unix mariadb-server +``` -Finally, double check that all required SSM variables exist in the account holding the server. These -are used to build the config file, but originally were only in Mark McIntyre's account. +### Miniconda +* Install miniconda with the default settings -## how to move accounts to a new server -The basic process is to extract the user accounts from the current server along with -group and password info, then import it back in on the new server. Its important to avoid -accidentally overwriting system or otherwise-existing accounts on the new server. +### WMPL +* Install WMPL + * create a 'wmpl' conda environment with at least python 3.13 + * install the python requirements + * See note below for how to get PyQt5 working. + * Alternatively you can delete subfolders of wmpl that rely on QT (`CAM0` `MetSIM` and `Utils/DynamicMassFit.py`). +``` bash +conda create -n wmpl python=3.13 +mkdir -p ~/src +cd ~/src +git clone --recursive git@github.com:markmac99/WesternMeteorPyLib.git +cd WesternMeteorPyLib +conda activate wmpl +pip install -r requirements.txt +python setup.py +``` -On most systems, accoutns below 500 are system accounts. Check though, as some AWS servers create -new accounts starting at 1000 and working both upwards and downwards! +### UKMDA Dataprocessing +* Clone the ukmda git repo and then install all the code using the deployment script. Both dev and prod envs should be created, though arguably the dev env should be on a separate server. +``` bash +cd ~/src +git clone git@github.com:ukmda/ukmda-dataprocessing.git +cd ukmda-dataprocessing +./install_or_upgrade.sh PROD +``` +* double check that all required SSM variables exist in the account holding the server. These +are used to build the config file and are deployed via terraform so should be present! For example: +``` bash +aws ssm get-parameters --region eu-west-2 --names prod_siteurl --query Parameters[0].Value +"https://archive.ukmeteors.co.uk" +``` -Steps in brief. NB must all be done as root, of course. +### SSH and other API keys +copy the below files from ~/.ssh on the old server, and make sure permissions are correct (should all be 0600). At a minimum you need the following files +``` bash +gmailcreds.json +gmailtoken.json +``` +These are used to authenticate against gmail. The keys are currently specific to Mark McIntyre's account and will need to be replaced with keys for `ukmeteors@gmail.com` + +You may also need the `github` keys though this depends on how you authenticate with GitHub. -### On the old server +## data +Replicate `~/prod/data`, `~/prod/logs` and `~/keymgmt` to the new server. (once for prod and once for dev). + + This will have to be repeated every day till golive (strictly, the key data only needs to be replicated if a new camera is added). Also keep the MariaDB SQL database up to date. ``` bash -mkdir /root/move/ -export UGIDLIMIT=500 -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/passwd > /root/move/passwd.mig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/group > /root/move/group.mig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534) {print $1}' /etc/passwd | tee - |egrep -f - /etc/shadow > /root/move/shadow.mig -cp /etc/gshadow /root/move/gshadow.mig +cd $DATADIR +rsync -avz ukmonhelper2:prod/data/ . +cd $SRC/logs +rsync -avz ukmonhelper2:prod/logs/ . +cd ~ +rsync -avz ukmonhelper2:keymgmt/ ./keymgmt +$SRC/utils/loadSingleCsvMDB.sh +$SRC/utils/loadMatchCsvMDB.sh + +``` +## Mariadb database +Find the mariadb config file (normally in /etc, might ber called `my.cnf`) and ensure that `bind-address` is set to `0.0.0.0`. The default on many installations is to bind to 127.0.0.1 which allows traffic only from localhost. The UKMON APIs access the database externally. After changing the config file, restart the database. -# Also backup the user homedirs. In our case, they're all in /var/sftp -tar cvfz /root/move/varsftp.tar.gz /var/sftp -# now copy the files to target server, -scp /root/move/* newserver:/tmp +Now sudo to root and run the following to set a root password +``` bash +mysql -u root -p +ALTER USER 'root'@'localhost' IDENTIFIED VIA mysql_native_password USING PASSWORD('ROOTPASSWORD'); +FLUSH PRIVILEGES; +quit ``` +Pbviously, replace `ROOTPASSWORD` with the password in the SSM variable `prod_dbpw` -### On the new server -In summary: backup the existing files, remove any accounts from the .mig files -that are already present in the target, then append the filtered data. +Exit the root shell, and as a 'normal' user execute the following: +``` bash +cd $SRC/database/ +mysql -u root -pROOTPASSWORD < ddl/create_dbs_and_users.sql +mysql -u root -pROOTPASSWORD < ddl/create_tables.sql +mysql -u root -pROOTPASSWORD < ddl/create_brightness_table.sq +``` +Now dump and reload the databases from the old server. +Make sure the old server can connect to the new one then on the old server, run the following: +``` bash +mysqldump -u batch -p'passwd' ukmon | ssh newserver mysql -u batch -p'passwd' ukmon +``` -When comparing groups, remember new ids will get added to the sftp group. +### To install PyQt5 +* needs at least 3GB memory so add 2GB swap if the server has less. +``` bash +sudo fallocate -l 2G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab + +sudo apt install qtbase5-dev qt5-qmake +pip install pyqt5 +``` + +### To get conda working in batch scripts +This is done by sourcing a file `.condaon` which is automatically installed by the ukmda deployment script and sourced from the environments' config.ini files. Make sure any scripts activate conda like this: + +``` bash +source $HOME/prod/config.ini >/dev/null 2>&1 +conda activate ${WMPL_ENV} +``` + +## how to move accounts to a new server + +### Install and configure sftp +First set up the SFTP server on the new host. +Run the following as root: + +``` bash +groupadd sftp +mkdir -p /var/sftp +chown root:root /var/sftp +chmod 751 /var/sftp +``` + +Add this to /etc/ssh/sshd_config +``` bash +Match group sftp +ChrootDirectory /var/sftp/%u +AllowTCPForwarding no +X11Forwarding no +ForceCommand internal-sftp +``` +and then reload sshd ``` bash -mkdir -p /root/move/bkp -mv /tmp/*.mig /tmp/varsftp.tar.gz /root/move -cp /etc/passwd /etc/group /etc/shadow /etc/gshadow /root/move/bkp +service sshd reload +``` -cd / -tar -xvf /root/move/varsftp.tar.gz . +### Now move the user accounts and update camera config -export UGIDLIMIT=500 -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/passwd > /root/move/passwd.orig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/group > /root/move/group.orig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534) {print $1}' /etc/passwd | tee - |egrep -f - /etc/shadow > /root/move/shadow.orig +First, ensure that root on the new server can connect to the old server as the batch user by creating a default SSH keypair for root, and adding its public half to root's authorized_keys file on the old server. -cd /root/move -diff passwd.orig passwd.mig -diff shadow.orig shadow.mig -diff group.orig group.mig +* on the new server as the standard batch user: + * create a folder `move` + * Create a list of the desired SFTP user accounts using +``` bash +ssh oldserver "sudo ls -1 /var/sftp" > ./move/sftp_accts.txt ``` + * Edit the list to exclude defunct accounts and other entries not related to a camera account. + * use `$SRC/$utils/migrateSftpAccts.sh` to create accounts on the new server and copy over the user data +``` bash +$SRC/utils/migrateSftpAccts.sh oldserverFQDN ./move/sftp_accts.txt +``` +This will also update the `ukmon.ini` file for each station on both server. Upon next connection, the station should update its local ini file and thereafter connect to the new server. + +This means that the old server must be kept running for a few days till all cameras have switched over. diff --git a/archive/server_setup/ssh-setup.sh b/archive/server_setup/ssh-setup.sh deleted file mode 100644 index f2c0cbf77..000000000 --- a/archive/server_setup/ssh-setup.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -echo $1 -scp -i ~/.ssh/markskey.pem ~/.ssh/ukmon-shared-keys $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/markskey.pem $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/config $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/gm* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/ukmon* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/marksk* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/s3* $1:.ssh - -ssh -i ~/.ssh/markskey.pem $1 mkdir setup -ssh -i ~/.ssh/markskey.pem $1 mkdir data -ssh -i ~/.ssh/markskey.pem $1 mkdir src - diff --git a/archive/server_setup/install-cartopy.sh b/archive/server_setup/unused/install-cartopy.sh similarity index 100% rename from archive/server_setup/install-cartopy.sh rename to archive/server_setup/unused/install-cartopy.sh diff --git a/archive/server_setup/install-geos.sh b/archive/server_setup/unused/install-geos.sh similarity index 100% rename from archive/server_setup/install-geos.sh rename to archive/server_setup/unused/install-geos.sh diff --git a/archive/server_setup/install-proj4.sh b/archive/server_setup/unused/install-proj4.sh similarity index 100% rename from archive/server_setup/install-proj4.sh rename to archive/server_setup/unused/install-proj4.sh diff --git a/archive/server_setup/install-sqlite3.sh b/archive/server_setup/unused/install-sqlite3.sh similarity index 100% rename from archive/server_setup/install-sqlite3.sh rename to archive/server_setup/unused/install-sqlite3.sh diff --git a/archive/server_setup/install_opencv.sh b/archive/server_setup/unused/install_opencv.sh similarity index 100% rename from archive/server_setup/install_opencv.sh rename to archive/server_setup/unused/install_opencv.sh diff --git a/archive/server_setup/newserver.sh b/archive/server_setup/unused/newserver.sh similarity index 100% rename from archive/server_setup/newserver.sh rename to archive/server_setup/unused/newserver.sh diff --git a/archive/server_setup/remount-s3.sh b/archive/server_setup/unused/remount-s3.sh similarity index 100% rename from archive/server_setup/remount-s3.sh rename to archive/server_setup/unused/remount-s3.sh diff --git a/archive/server_setup/yum-installs.sh b/archive/server_setup/unused/yum-installs.sh similarity index 100% rename from archive/server_setup/yum-installs.sh rename to archive/server_setup/unused/yum-installs.sh diff --git a/archive/shwrinfo/costnotes.html b/archive/shwrinfo/costnotes.html index 09a5714c8..08b211c37 100644 --- a/archive/shwrinfo/costnotes.html +++ b/archive/shwrinfo/costnotes.html @@ -4,19 +4,4 @@

  • Amounts of less than $0.10 per day are consolidated under Other.
  • AWS cost data may lag by up to a day, so yesterday's costs data are incomplete.
  • AWS charge VAT at the start of the month for the previous month's total bill.
    • -
  • Until September 2023, costs were split across two accounts for historical reasons. We've now consolidated into one account.
    • -
    -2023 Costs prior to 13th September were as follows: -
  • S3: $1135
    • -
  • ECS: $212
    • -
  • EC2-other: $159
    • -
  • EC2-instance: $131
    • -
  • Cloudwatch: $44
    • -
  • Registrar: $29
    • -
  • KMS: $23
    • -
  • DynamoDB: $15
    • -
  • Route53: $6
    • -
  • ECR: $5
    • -
  • Other: £8
    • - -Total: $1768 + VAT +
    \ No newline at end of file diff --git a/archive/terraform/mjmm/aws.tf b/archive/terraform/mjmm/aws.tf index e18c7d50e..a9871b9e3 100644 --- a/archive/terraform/mjmm/aws.tf +++ b/archive/terraform/mjmm/aws.tf @@ -6,8 +6,8 @@ provider "aws" { } provider "aws" { - alias = "eeacct" - region = var.remote_region - profile = "ukmonshared" + profile = var.profile + region = "eu-west-1" + alias = "eu-west-1-prov" } diff --git a/archive/terraform/mjmm/crossacct-role.tf b/archive/terraform/mjmm/crossacct-role.tf index fabcd4ecd..858365ade 100644 --- a/archive/terraform/mjmm/crossacct-role.tf +++ b/archive/terraform/mjmm/crossacct-role.tf @@ -1,7 +1,5 @@ # Copyright (C) 2018-2023 Mark McIntyre -data "aws_caller_identity" "eeacct" {provider = aws.eeacct } - # RESTRICT THESE PERMISSIONS!! data "aws_iam_policy" "terraformpol" { name = "AdministratorAccess" diff --git a/archive/terraform/mjmm/dev_ssm_parameters.tf b/archive/terraform/mjmm/dev_ssm_parameters.tf index a9216604f..819844f67 100644 --- a/archive/terraform/mjmm/dev_ssm_parameters.tf +++ b/archive/terraform/mjmm/dev_ssm_parameters.tf @@ -65,6 +65,15 @@ resource "aws_ssm_parameter" "dev_calcuser" { } } +resource "aws_ssm_parameter" "dev_calcip" { + name = "dev_calcserverip" + type = "String" + value = "172.32.16.137" + tags = { + "billingtag" = "ukmon" + } +} + resource "aws_ssm_parameter" "dev_backupinstance" { name = "dev_backupinstance" type = "String" @@ -127,3 +136,4 @@ resource "aws_ssm_parameter" "dev_batchloggroup" { "billingtag" = "ukmon" } } + diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 73dff36bc..8889f3a64 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -1,6 +1,45 @@ # Copyright (C) 2018-2023 Mark McIntyre # SSM parameters for use in the environment config +resource "aws_ssm_parameter" "prod_dbhost" { + provider = aws.eu-west-1-prov + name = "prod_dbhost" + type = "String" + value = "3.11.55.160" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbname" { + provider = aws.eu-west-1-prov + name = "prod_dbname" + type = "String" + value = "ukmon" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbpw" { + provider = aws.eu-west-1-prov + name = "prod_dbpw" + type = "SecureString" + value = "Batch33mdl" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbuser" { + provider = aws.eu-west-1-prov + name = "prod_dbuser" + type = "String" + value = "batch" + tags = { + "billingtag" = "ukmda" + } +} resource "aws_ssm_parameter" "prod_websitebucket" { name = "prod_websitebucket" @@ -67,6 +106,14 @@ resource "aws_ssm_parameter" "prod_calcuser" { } } +resource "aws_ssm_parameter" "prod_calcip" { + name = "prod_calcserverip" + type = "String" + value = "172.32.16.137" + tags = { + "billingtag" = "ukmon" + } +} resource "aws_ssm_parameter" "prod_backupinstance" { name = "prod_backupinstance" type = "String" @@ -130,3 +177,11 @@ resource "aws_ssm_parameter" "prod_batchloggroup" { } } +resource "aws_ssm_parameter" "prod_gmapsapikey" { + name = "prod_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmon" + } +} diff --git a/archive/terraform/ukmda/batchserver.tf b/archive/terraform/ukmda/batchserver.tf new file mode 100644 index 000000000..7ff5993f8 --- /dev/null +++ b/archive/terraform/ukmda/batchserver.tf @@ -0,0 +1,47 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +resource "aws_instance" "batchserver" { + ami = "ami-06f0cf249257c89b3" + instance_type = "t4g.micro" + iam_instance_profile = aws_iam_instance_profile.calcserverrole.name + key_name = aws_key_pair.marks_key.key_name + force_destroy = false + + root_block_device { + tags = { + "Name" = "batchservervol" + "billingtag" = "ukmda" + } + volume_size = 50 + volume_type = "gp3" + throughput = 125 + iops = 3000 + encrypted = true + kms_key_id = aws_kms_key.container_key.arn + } + + tags = { + "Name" = "batchserver" + "billingtag" = "ukmda" + "Route53FQDN" = "batchserver.ukmeteors.co.uk" + "DNSRecordType" = "A" + } + primary_network_interface { + network_interface_id = aws_network_interface.batchserver_if.id + } + metadata_options { + http_tokens = "required" + } +} + +resource "aws_network_interface" "batchserver_if" { + subnet_id = aws_subnet.ec2_subnet.id + description = "Primary network interface" + private_ips = [var.batchserverip] + security_groups = [aws_security_group.ec2_secgrp.id] + ipv6_address_list_enabled = false + tags = { + "Name" = "batchserver" + "billingtag" = "ukmda" + } +} diff --git a/archive/terraform/ukmda/calcserver.tf b/archive/terraform/ukmda/calcserver.tf index fe1248458..a1aa2a468 100644 --- a/archive/terraform/ukmda/calcserver.tf +++ b/archive/terraform/ukmda/calcserver.tf @@ -1,46 +1,5 @@ # Copyright (C) 2018-2023 Mark McIntyre -resource "aws_instance" "calc_server" { - ami = "ami-0df2d8f6def0bd716" - instance_type = "c8g.2xlarge" - iam_instance_profile = aws_iam_instance_profile.calcserverrole.name - key_name = aws_key_pair.marks_key.key_name - force_destroy = false - tags = { - "Name" = "Calcengine" - "billingtag" = "ukmda" - "Route53FQDN" = "calcengine.ukmeteors.co.uk" - "DNSRecordType" = "A" - } - root_block_device { - tags = { - "Name" = "calcengine" - "billingtag" = "ukmda" - } - volume_size = 120 - } - primary_network_interface { - network_interface_id = aws_network_interface.calcserver_if.id - } - - metadata_options { - http_tokens = "required" - } -} - -# elastic network interface attached to the calc server - -resource "aws_network_interface" "calcserver_if" { - subnet_id = aws_subnet.ec2_subnet.id - description = "Primary network interface" - private_ips = [var.calcserverip] - security_groups = [aws_security_group.ec2_secgrp.id] - ipv6_address_list_enabled = false - tags = { - "Name" = "calcengine" - "billingtag" = "ukmda" - } -} ################################################ # Ubuntu calc server ################################################ diff --git a/archive/terraform/ukmda/cloudwatch.tf b/archive/terraform/ukmda/cloudwatch.tf index 61ab690a0..ee336b1dd 100644 --- a/archive/terraform/ukmda/cloudwatch.tf +++ b/archive/terraform/ukmda/cloudwatch.tf @@ -21,7 +21,7 @@ resource "aws_cloudwatch_metric_alarm" "calcServerIdle" { "arn:aws:automate:${var.region}:ec2:stop", ] dimensions = { - "InstanceId" = aws_instance.calc_server.id + "InstanceId" = aws_instance.ubuntu_calc_server.id } tags = { "billingtag" = "ukmda" diff --git a/archive/terraform/ukmda/dev_ssm_variables.tf b/archive/terraform/ukmda/dev_ssm_variables.tf new file mode 100644 index 000000000..bca543597 --- /dev/null +++ b/archive/terraform/ukmda/dev_ssm_variables.tf @@ -0,0 +1,191 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +# SSM parameters for use in Lambdas and the batch + +resource "aws_ssm_parameter" "dev_dbhost" { + provider = aws.eu-west-1-prov + name = "dev_dbhost" + type = "String" + value = "3.11.55.160" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_dbname" { + provider = aws.eu-west-1-prov + name = "dev_dbname" + type = "String" + value = "ukmon" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_dbpw" { + provider = aws.eu-west-1-prov + name = "dev_dbpw" + type = "SecureString" + value = "Batch33mdl" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_rootdbpw" { + provider = aws.eu-west-1-prov + name = "dev_rootdbpw" + type = "SecureString" + value = "Wombat33mdb" + tags = { + "billingtag" = "ukmda" + } +} + + +resource "aws_ssm_parameter" "dev_dbuser" { + provider = aws.eu-west-1-prov + name = "dev_dbuser" + type = "String" + value = "batch" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_websitebucket" { + name = "dev_websitebucket" + type = "String" + value = var.dev_websitebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_sharedbucket" { + name = "dev_sharedbucket" + type = "String" + value = var.dev_sharedbucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_livebucket" { + name = "dev_livebucket" + type = "String" + value = var.dev_livebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_siteurl" { + name = "dev_siteurl" + type = "String" + value = "https://www.ukmeteors.co.uk/dummy/" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_envname" { + name = "dev_envname" + type = "String" + value = "DEV" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_calcinstance" { + name = "dev_calcinstance" + type = "String" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_calcuser" { + name = "dev_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmda" + } +} + + +resource "aws_ssm_parameter" "dev_calcserverip" { + name = "dev_calcserverip" + type = "String" + value = var.ubuntu_calcserverip + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_wmplhome" { + name = "dev_wmplhome" + type = "String" + value = "$HOME/src/wmpldev" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_rmshome" { + name = "dev_rmshome" + type = "String" + value = "$HOME/src/RMS" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_srcdir" { + name = "dev_srcdir" + type = "String" + value = "$HOME/dev" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_caminfo" { + name = "dev_caminfo" + type = "String" + value = "$HOME/dev/data/consolidated/camera-details.csv" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_sshkey" { + name = "dev_sshkey" + type = "String" + value = "$HOME/.ssh/markskey.pem" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_batchloggroup" { + name = "dev_batchloggroup" + type = "String" + value = "/ukmondev/nightlyjob" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_gmapsapikey" { + name = "dev_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmda" + } +} + diff --git a/archive/terraform/ukmda/secgrp.tf b/archive/terraform/ukmda/secgrp.tf index 1ae17dd00..a6aa5e1d4 100644 --- a/archive/terraform/ukmda/secgrp.tf +++ b/archive/terraform/ukmda/secgrp.tf @@ -46,8 +46,8 @@ resource "aws_security_group" "ec2_secgrp" { description = "Security Group for EC2 instances" vpc_id = aws_vpc.ec2_vpc.id ingress = [ -/* { - cidr_blocks = [data.aws_vpc.mjmm_ec2_vpc.cidr_block] + { + cidr_blocks = [aws_vpc.ec2_vpc.cidr_block] description = "SSH for Admin" from_port = 22 protocol = "tcp" @@ -56,9 +56,9 @@ resource "aws_security_group" "ec2_secgrp" { prefix_list_ids = [] security_groups = [] self = false - },*/ + }, { - cidr_blocks = [aws_vpc.ec2_vpc.cidr_block] + cidr_blocks = ["0.0.0.0/0"] description = "SSH for Admin" from_port = 22 protocol = "tcp" @@ -69,16 +69,38 @@ resource "aws_security_group" "ec2_secgrp" { self = false }, { - cidr_blocks = ["0.0.0.0/0"] - description = "SSH for Admin" - from_port = 22 + cidr_blocks = ["194.0.0.0/8"] + description = "MariaDB" + from_port = 3306 protocol = "tcp" - to_port = 22 + to_port = 3306 ipv6_cidr_blocks = [] prefix_list_ids = [] security_groups = [] self = false - } + }, + { + cidr_blocks = ["0.0.0.0/0"] + description = "icmp" + from_port = -1 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "icmp" + security_groups = [] + self = false + to_port = -1 + }, + { + cidr_blocks = ["0.0.0.0/0"] + description = "mariadb" + from_port = 3306 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "tcp" + security_groups = [] + self = false + to_port = 3306 + }, ] egress = [ { diff --git a/archive/terraform/ukmda/ssm_variables.tf b/archive/terraform/ukmda/ssm_variables.tf index 29fba5ac6..5661b5078 100644 --- a/archive/terraform/ukmda/ssm_variables.tf +++ b/archive/terraform/ukmda/ssm_variables.tf @@ -1,14 +1,14 @@ # Copyright (C) 2018-2023 Mark McIntyre -# SSM parameters for use in Lambdas +# SSM parameters for use in Lambdas and the batch resource "aws_ssm_parameter" "prod_dbhost" { provider = aws.eu-west-1-prov name = "prod_dbhost" type = "String" - value = "3.11.55.160" + value = "batchserver.ukmeteors.co.uk" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } @@ -18,7 +18,7 @@ resource "aws_ssm_parameter" "prod_dbname" { type = "String" value = "ukmon" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } @@ -28,7 +28,17 @@ resource "aws_ssm_parameter" "prod_dbpw" { type = "SecureString" value = "Batch33mdl" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_rootdbpw" { + provider = aws.eu-west-1-prov + name = "prod_rootdbpw" + type = "SecureString" + value = "Wombat33mdb" + tags = { + "billingtag" = "ukmda" } } @@ -38,6 +48,143 @@ resource "aws_ssm_parameter" "prod_dbuser" { type = "String" value = "batch" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_websitebucket" { + name = "prod_websitebucket" + type = "String" + value = var.websitebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_sharedbucket" { + name = "prod_sharedbucket" + type = "String" + value = var.sharedbucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_livebucket" { + name = "prod_livebucket" + type = "String" + value = var.livebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_siteurl" { + name = "prod_siteurl" + type = "String" + value = "https://archive.ukmeteors.co.uk" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_envname" { + name = "prod_envname" + type = "String" + value = "PROD" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_calcinstance" { + name = "prod_calcinstance" + type = "String" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_calcuser" { + name = "prod_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmda" } } + + +resource "aws_ssm_parameter" "prod_calcserverip" { + name = "prod_calcserverip" + type = "String" + value = var.ubuntu_calcserverip + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_wmplhome" { + name = "prod_wmplhome" + type = "String" + value = "$HOME/src/WesternMeteorPyLib" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_rmshome" { + name = "prod_rmshome" + type = "String" + value = "$HOME/src/RMS" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_srcdir" { + name = "prod_srcdir" + type = "String" + value = "$HOME/prod" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_caminfo" { + name = "prod_caminfo" + type = "String" + value = "$HOME/prod/data/consolidated/camera-details.csv" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_sshkey" { + name = "prod_sshkey" + type = "String" + value = "$HOME/.ssh/markskey.pem" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_batchloggroup" { + name = "prod_batchloggroup" + type = "String" + value = "/ukmonbatch/nightlyjob" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_gmapsapikey" { + name = "prod_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmda" + } +} + diff --git a/archive/terraform/ukmda/variables.tf b/archive/terraform/ukmda/variables.tf index e73774644..01d6aef6b 100644 --- a/archive/terraform/ukmda/variables.tf +++ b/archive/terraform/ukmda/variables.tf @@ -8,6 +8,10 @@ variable "sharedbucket" { default = "ukmda-shared" } variable "livebucket" { default = "ukmda-live" } variable "adminbucket" { default = "ukmda-admin" } +variable "dev_websitebucket" { default = "ukmda-website" } +variable "dev_sharedbucket" { default = "ukmda-shared" } +variable "dev_livebucket" { default = "ukmda-live" } + variable "region" { default = "eu-west-2"} variable "liveregion" { default = "eu-west-1"} @@ -18,14 +22,13 @@ data "aws_canonical_user_id" "current" {} variable "remote_profile" { default = "default"} variable "remote_account_id" { default = "317976261112" } variable "remote_region" {default = "eu-west-2"} -#variable "eeaccountid" {default = "822069317839" } variable "main_cidr" {default = "172.32.0.0/16" } variable mgmt_cidr { default = "172.32.36.0/22" } variable lambda_cidr { default = "172.32.32.0/22" } variable ec2_cidr { default = "172.32.16.0/20" } -variable calcserverip { default = "172.32.16.136" } variable ubuntu_calcserverip { default = "172.32.16.137" } +variable batchserverip { default = "172.32.16.138" } variable archalias { default = "archive.ukmeteors.co.uk"} diff --git a/archive/ukmon_pylib/requirements.txt b/archive/ukmon_pylib/additional_requirements.txt similarity index 67% rename from archive/ukmon_pylib/requirements.txt rename to archive/ukmon_pylib/additional_requirements.txt index 8b9f56c62..c5942af22 100644 --- a/archive/ukmon_pylib/requirements.txt +++ b/archive/ukmon_pylib/additional_requirements.txt @@ -1,52 +1,37 @@ # Copyright (C) 2018-2023 Mark McIntyre #cartopy INSTALL THIS WITH CONDA -# skip this -#https://github.com/matplotlib/basemap/archive/master.zip -basemap basemap-data-hires boto3 -Cython gmplot google-auth google-auth-oauthlib google-api-python-client googleapis-common-protos -jplephem -Keras -matplotlib==3.3.2 -numpy oauthlib -pandas Pillow pycparser pyephem -pyproj==2.6.1.post1 -PyQt5 +pyproj pyswarms -pytz -PyYAML -scipy Shapely xmltodict -tweepy fastparquet cryptography python-crontab simplekml astropy -gitpython pymysql mysql-connector-python pre-commit s3fs -# pyarrow +dynamodb_json +requests +paho-mqtt +scp +# testing requirements pytest behave pdoc -dynamodb_json pytest pytest-cov -requests -paho-mqtt -scp \ No newline at end of file diff --git a/archive/ukmon_pylib/analysis/README.md b/archive/ukmon_pylib/analysis/README.md index 732ae6a45..a4b09f7d4 100644 --- a/archive/ukmon_pylib/analysis/README.md +++ b/archive/ukmon_pylib/analysis/README.md @@ -1,9 +1,8 @@ # Analysis -This folder contains functions to analyse the single station and matched data. +This folder contains functions to analyse the single station and matched data for website reporting. * ShowerAnalysis.py - analysis for a named shower for a specific year eg GEM 2022 * stationAnalysis.py - analysis for a named year or year+month, for one or all stations. -* meteorFlux - plot a graph of meteor flux -* gatherDetectionData.py -collects all available data on a detection or detections matching a pattern YYYYMMDD_HHMMSS. If the seconds are omitted, then all detections within the specified minute will be returned. - +* summaryAnalysis.py - summary analysis for a given period + diff --git a/archive/ukmon_pylib/analysis/gatherDetectionData.py b/archive/ukmon_pylib/analysis/gatherDetectionData.py deleted file mode 100644 index d7e45b9b8..000000000 --- a/archive/ukmon_pylib/analysis/gatherDetectionData.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Collect data for a potential match at a specific time -# -# Copyright (C) 2018-2023 Mark McIntyre -# -import os -import sys -import datetime -import pandas as pd - -from meteortools.ukmondb import getECSVs, getLiveJpgs, getFBfiles, getDetections - - -def getUncalibratedImageList(dtstr=None): - datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') - flines = open(logfile, 'r').readlines() - uncal = [f for f in flines if 'not recalibrated' in f] - imglist = [x.strip(' ').split(' ')[1][:-1] for x in uncal] - with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: - for li in imglist: - outf.write(f'{li}\n') - return imglist - - -def getRawData(idlist, outpth): - for _,row in idlist.iterrows(): - stat =row.ID - dts = datetime.datetime.strptime(row.Dtstamp, '%Y%m%d_%H%M%S') - dt = dts.strftime('%Y-%m-%dT%H:%M:%S') - fboutdir = os.path.join(outpth, stat) - dtstr = row.Filename[10:25] - getLiveJpgs(dtstr, outpth, False) - getFBfiles(f'{stat}_{dtstr}', fboutdir) - getECSVs(stat, dt, True, fboutdir) - return - - -def updateSingleDB(yr, consdt): - datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - sngls = pd.read_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv')) - consdf = pd.read_csv(os.path.join(datadir, 'single', 'used', f'consumed_{consdt}.txt'), names=['Filename']) - conslist = list(consdf.Filename) - for ff in conslist: - mtch = sngls.Filename == ff - if mtch.any(): - print(f'updating {ff}') - sngls.loc[mtch, 'status'] = 'Y' - else: - print(f'skipping {ff}') - sngls.to_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv'), index=False) - - -if __name__ == '__main__': - outpth = f'./{sys.argv[1]}' - os.makedirs(outpth, exist_ok=True) - idlist = getDetections(sys.argv[1]) - if not isinstance(idlist, bool): - getRawData(idlist, outpth) - ids = list(idlist.ID) - with open(os.path.join(outpth, 'ids.txt'), 'w') as outf: - outf.writelines(line + '\n' for line in ids) - print(ids) - else: - print('no matches') diff --git a/archive/ukmon_pylib/analysis/getUsedUnused.py b/archive/ukmon_pylib/analysis/getUsedUnused.py new file mode 100644 index 000000000..0541ecad6 --- /dev/null +++ b/archive/ukmon_pylib/analysis/getUsedUnused.py @@ -0,0 +1,65 @@ +# +# Python code to analyse meteor shower data +# +# Copyright (C) 2018-2023 Mark McIntyre + +import os +import boto3 +import json +import pymysql.cursors + + +def getSqlLoginDetails(): + # retrieve password and host from SSM. This allows me to manage them from Terraform + ssm = boto3.client('ssm', region_name='eu-west-1') + res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) + password = res['Parameter']['Value'] + res = ssm.get_parameter(Name='prod_dbhost') + host = res['Parameter']['Value'] + # should really do these too but they won't change often if at all + user = 'batch' + db = 'ukmon' + return host, user, password, db + + +def getAllImages(dtstr): + """ + Retrieve a list of all images for a given date from the SQL database + This is a list of all images containing detections, whether or not used + in a solution. + """ + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + sqlstr = f'select filname from singles where Y={dtstr[:4]} and M={int(dtstr[4:6])} and D={int(dtstr[6:8])}' + with connection.cursor() as cursor: + cursor.execute(sqlstr) + result = cursor.fetchall() + connection.close() + res=[] + for event in result: + res.append(event['filname']) + return res + + +def getKnownImages(dtstr, datatype='consumed'): + """ + Get a list of all images that were involved in a match or were uncalibrated, + depending on the value of datatype (consumed or uncal) + The raw data are generated by a process on the matching engine server + """ + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + srcfile = os.path.join(datadir,'single','used',f'{datatype}_{dtstr}.txt') + imglist = [] + if os.path.isfile(srcfile): + used = open(srcfile,'r').readlines() + for img in used: + imglist.append(img.strip()) + return imglist + + +def getUnusedImages(dtstr): + + allimages = getAllImages(dtstr) + uncal = getKnownImages(dtstr, 'uncal') + used = getKnownImages(dtstr, 'consumed') + unused = list(set(allimages)-set(used + uncal)) \ No newline at end of file diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index 02551d78a..7defd5930 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -10,8 +10,8 @@ from matplotlib import pyplot as plt import datetime -from meteortools.utils import getShowerDates as sd -from meteortools.fileformats import imoWorkingShowerList as imo +from utils.getActiveShowers import getShowerDets +from utils.imoWorkingShowerList import IMOshowerList SMALL_SIZE = 8 MEDIUM_SIZE = 10 @@ -97,8 +97,8 @@ def timeGraph(dta, shwrname, outdir, binmins=10): countcol = dta['Shwr'] # resample it - binned = countcol.resample('{}min'.format(binmins)).count() - binned.plot(kind='bar') + binned = countcol.resample(f'{binmins}min').count() + binned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals nbins = max(len(binned)/144, 2) @@ -138,8 +138,8 @@ def matchesGraphs(dta, shwrname, outdir, binmins=60, startdt=None, enddt=None, t #select just the shower ID col mcountcol = mdta['_ID1'] # resample it - mbinned = mcountcol.resample('{}min'.format(binmins)).count() - mbinned.plot(kind='bar') + mbinned = mcountcol.resample(f'{binmins}min').count() + mbinned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals ax = plt.gca() @@ -216,7 +216,7 @@ def velDistribution(dta, shwrname, outdir, vg_or_vs, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -254,7 +254,7 @@ def durationDistribution(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.1f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -292,7 +292,7 @@ def distanceDistribution(dta, shwrname, outdir, binwidth=1.0): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -356,13 +356,6 @@ def radiantDistribution(dta, shwrname, outdir): magdf.plot.scatter(x=idx, y=idx2) ax = plt.gca() ax.set(xlabel='RA (deg)', ylabel="Dec (deg)") - #plt.locator_params(axis='x', nbins=12) - #for lab in ax.get_xticklabels(): - # lab.set_fontsize(SMALL_SIZE) - - # format x-axes - #x_labels=["%.0f" % number for number in bins[:-1]] - #ax.set_xticklabels(x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -394,7 +387,7 @@ def semimajorDistribution(dta, shwrname, outdir, binwidth=0.5): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -424,7 +417,7 @@ def magDistributionAbs(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -454,7 +447,7 @@ def magDistributionVis(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -486,7 +479,7 @@ def showerAnalysis(shwr, dtstr): cols = ['Shwr','Dtstamp','Y','M','ID','Mag'] filt = None if shwr != 'ALL': - sl = imo.IMOshowerList() + sl = IMOshowerList() maxdt = sl.getEnd(shwr) + datetime.timedelta(days=10) mindt = sl.getStart(shwr) + datetime.timedelta(days=-10) @@ -498,7 +491,7 @@ def showerAnalysis(shwr, dtstr): # select the required data if shwr != 'ALL': - id, shwrname, sl, dt = sd.getShowerDets(shwr) + id, shwrname, sl, dt = getShowerDets(shwr) sngl = sngl[sngl['Shwr']==shwr] sngl = sngl[sngl.Dtstamp > mindt.timestamp()] sngl = sngl[sngl.Dtstamp < maxdt.timestamp()] @@ -531,7 +524,7 @@ def showerAnalysis(shwr, dtstr): # select the required data if shwr != 'ALL': - id, shwrname, sl, dt = sd.getShowerDets(shwr) + id, shwrname, sl, dt = getShowerDets(shwr) mtch = mtch[mtch['_stream']==shwr] mtch = mtch[mtch.dtstamp > mindt.timestamp()] mtch = mtch[mtch.dtstamp < maxdt.timestamp()] diff --git a/archive/ukmon_pylib/analysis/stationAnalysis.py b/archive/ukmon_pylib/analysis/stationAnalysis.py index fb7a2e9d5..7e888ae47 100644 --- a/archive/ukmon_pylib/analysis/stationAnalysis.py +++ b/archive/ukmon_pylib/analysis/stationAnalysis.py @@ -77,7 +77,7 @@ def timeGraph(dta, loc, outdir, when, sampleinterval): #select just the shower ID col countcol = dta['ID'] # resample it - binned = countcol.resample('{}'.format(sampleinterval)).count() + binned = countcol.resample(f'{sampleinterval}').count() binned.plot(kind='bar') # set ticks and labels every 144 intervals @@ -100,7 +100,7 @@ def timeGraph(dta, loc, outdir, when, sampleinterval): pass fname = os.path.join(outdir, 'station_plot_timeline_single.jpg') - plt.title('Observed stream activity {} intervals for {} in {}'.format(sampleinterval, loc, when)) + plt.title(f'Observed stream activity {sampleinterval} intervals for {loc} in {when}') try: plt.tight_layout() plt.savefig(fname) @@ -311,7 +311,7 @@ def pushToWebsite(fuloutdir, outdir, websitebucket): idlist = list(camlistfltr.stationid) if mth is None: - sampleinterval="1M" + sampleinterval="1ME" shortoutdir = os.path.join('reports', str(yr), 'stations', loc) outdir = os.path.join(datadir, shortoutdir) else: diff --git a/archive/ukmon_pylib/converters/toParquet.py b/archive/ukmon_pylib/converters/toParquet.py index c9454b7b8..a147501ce 100644 --- a/archive/ukmon_pylib/converters/toParquet.py +++ b/archive/ukmon_pylib/converters/toParquet.py @@ -15,7 +15,7 @@ if 'match' in fn: # fill in any #NAs in the mjd column - df.mjd.fillna(df._mjd, inplace=True) + df.fillna({'mjd': df._mjd}, inplace=True) if 'single' in fn: df = df.drop_duplicates() diff --git a/archive/ukmon_pylib/maintenance/cleanDdbTables.py b/archive/ukmon_pylib/maintenance/cleanDdbTables.py index 464d2be7f..26e17f80d 100644 --- a/archive/ukmon_pylib/maintenance/cleanDdbTables.py +++ b/archive/ukmon_pylib/maintenance/cleanDdbTables.py @@ -9,9 +9,7 @@ def deleteRows(): - archprof = os.getenv('ADM_PROFILE', default='ukmda_admin') - conn = boto3.Session(profile_name=archprof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') tbl = 'uploadtimes' idx = 'uploaddate-stationid-index' table = ddb.Table(tbl) @@ -28,9 +26,7 @@ def deleteRows(): def updateMissingExpiryDate(): - archprof = os.getenv('ADM_PROFILE', default='ukmda_admin') - conn = boto3.Session(profile_name=archprof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') tbl = 'uploadtimes' idx = 'uploaddate-stationid-index' table = ddb.Table(tbl) diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index 911abcb3e..32418b256 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -11,8 +11,8 @@ from time import sleep import pandas as pd import datetime - -import pymysql.cursors +import json +import operator from wmpl.Utils.TrajConversions import datetime2JD from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase @@ -55,18 +55,11 @@ def deleteS3FilesByMonth(flist, archbucket): def deleteFromCalcServerByMonth(outfname): - hname = os.getenv('HOSTNAME', default='none') env = os.getenv('RUNTIME_ENV', default='DEV').lower() - if hname != 'ukmonhelper2': - sess = boto3.Session(profile_name='default') - ssmc = sess.client('ssm', region_name='eu-west-2') - ec2 = boto3.client('ec2', region_name='eu-west-2') - else: - ssmc = boto3.client('ssm', region_name='eu-west-2') - prof = os.getenv('UKMPROFILE',default='ukmonshared') - sess = boto3.Session(profile_name=prof) - ec2 = sess.client('ec2', region_name='eu-west-2') + ssmc = boto3.client('ssm', region_name='eu-west-2') + ec2 = boto3.client('ec2', region_name='eu-west-2') + resp = ssmc.get_parameter(Name=f'{env}_calcinstance') instId = resp['Parameter']['Value'] print('clearing down calcserver') @@ -106,8 +99,7 @@ def deleteFromCalcServerByMonth(outfname): def removeDeletedTraj(csvfile): """ - Remove deleted trajectories from the consolidated match CSV and Parquet files - and from the search index. + Remove deleted trajectories from the consolidated match CSV, Parquet and search files """ csvdata = open(csvfile, 'r').readlines() @@ -117,28 +109,44 @@ def removeDeletedTraj(csvfile): jdt_end = datetime2JD(dt_end) else: jdt_end = float(csvdata[-1].split(',')[3]) + 2400000.5 - - jdt_beg =jdt_end - 21 + jdt_beg = jdt_end - 21 datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) masterdb_path = os.path.join(datadir, 'distrib') masterdb = TrajectoryDatabase(db_path=masterdb_path) cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') deltrajs = cur.fetchall() - masterdb.closeTrajDatabase() i=0 + # loop over the deleted trajectories, removing the corresponding one from the CSV file. + # Note that there's no need to update the MariaDB database, as it is populated + # from the CSV file *after* it has been purged + offs = 4 if 'search' in csvfile else 131 for traj in deltrajs: - fldr = os.path.basename(os.path.dirname(traj[0])) - match = [tr for tr in csvdata if fldr in tr] - if len(match) > 0: + cur = masterdb.dbhandle.execute(f'select jdt_ref, participating_stations from trajectories where traj_file_path="{traj[0]}" and status=0') + thistraj = cur.fetchall() + if len(thistraj) == 1: + fldr = os.path.basename(os.path.dirname(traj[0])) + + # find the rows in the CSV file that correspond to the deleted trajectory + # then go through them and compare obs_ids. The one with the same obs_ids + # is the one we want to remove + match = [tr for tr in csvdata if fldr in tr] + obs_ids = thistraj[0][1] + obs_ids_str = ';'.join(json.loads(obs_ids)) + ';' for thismtch in match: - print(f'removing {fldr}') - idx = csvdata.index(thismtch) - _ = csvdata.pop(idx) - i += 1 - print(f'removed {i} trajectories') + if thismtch.split(',')[offs] == obs_ids_str: + print(f'removing {fldr}') + idx = csvdata.index(thismtch) + _ = csvdata.pop(idx) + i += 1 + break + else: + print(f'skipping {traj[0]} as there are {len(thistraj)} active trajs with the same path') + masterdb.closeTrajDatabase() + print(f'removed {i} trajectories from the text file') + # save the CSV file again open(csvfile, 'w').writelines(csvdata) if 'search' not in csvfile: @@ -149,48 +157,64 @@ def removeDeletedTraj(csvfile): return -def getSqlLoginDetails(): - # retrieve password and host from SSM. This allows me to manage them from Terraform - ssm = boto3.client('ssm', region_name='eu-west-1') - res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) - password = res['Parameter']['Value'] - res = ssm.get_parameter(Name='prod_dbhost') - host = res['Parameter']['Value'] - # should really do these too but they won't change often if at all - user = 'batch' - db = 'ukmon' - return host, user, password, db - - -def removeDelTrajFromDb(): - dt_end = datetime.datetime.now(tz=datetime.timezone.utc) - jdt_end = datetime2JD(dt_end) - jdt_beg =jdt_end - 7 - - # get list of deleted trajectories - datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - masterdb_path = os.path.join(datadir, 'distrib') - masterdb = TrajectoryDatabase(db_path=masterdb_path) - cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') - deltrajs = cur.fetchall() - masterdb.closeTrajDatabase() +def removeDeletedTrajCsv(csvloc, targloc): + """ + Remove csv files corresponding to deleted trajectories before they are + consolidated into the annual CSV and Parquet files + """ + srcbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - # get connection to the SQL database - host, user, passwd, db = getSqlLoginDetails() - connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + csvfiles = [] + s3 = boto3.client('s3') + print('checking in ', csvloc, 'sending to ', targloc) + res = s3.list_objects(Bucket=srcbucket, Prefix=csvloc) + if 'Contents' in res: + for k in res['Contents']: + csvname = k["Key"] + csv_dt = datetime.datetime.strptime(os.path.basename(k["Key"])[:22], "%Y%m%d-%H%M%S.%f") + csvfiles.append({"name":csvname, "dt":csv_dt}) + csvfiles.sort(key=operator.itemgetter('name')) + jdt_beg = datetime2JD(csvfiles[0]['dt']) + jdt_end = datetime2JD(csvfiles[-1]['dt']) + + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + + i=0 + # loop over the deleted trajectories, removing the corresponding CSV file. + for del_traj in deltrajs: + cur = masterdb.dbhandle.execute(f'select jdt_ref, participating_stations from trajectories where traj_file_path="{del_traj[0]}" and status=0') + thistraj = cur.fetchall() + if len(thistraj) == 1: + obs_ids = thistraj[0][1] + obs_ids_str = ';'.join(json.loads(obs_ids)) + ';' + fldr = os.path.basename(os.path.dirname(del_traj[0]))[:19].replace('_', '-') + + matched_csvs = [x for x in csvfiles if fldr in x['name']] + for csv in matched_csvs: + csvdata = s3.get_object(Bucket=srcbucket, Key=csv['name'])['Body'].read() + csv_obs_ids = csvdata.decode('utf-8').split(',')[131].strip() + if csv_obs_ids == obs_ids_str: + # we have a match to within 0.001s and with the same observations + print(f'removing {fldr}') + bare_csv_name = os.path.basename(matched_csvs[0]['name']) + yr = bare_csv_name[:4] + destkey = f"{targloc}/{yr}/{bare_csv_name}" + s3.copy({"Bucket":srcbucket,"Key":csv['name']}, srcbucket, destkey) + s3.delete_object(Bucket=srcbucket, Key=matched_csvs[0]['name']) + i += 1 + break + else: + print(f'skipping {del_traj[0]} as there are {len(thistraj)} inactive trajs with the same path') + masterdb.closeTrajDatabase() + print(f'removed {i} trajectories from the text file') + else: + print('no fullcsv files to process') - count = 0 - for traj in deltrajs: - fldr = os.path.basename(os.path.dirname(traj[0])) - sqlstr = f"delete from matches where orbname like '{fldr}%'" - with connection.cursor() as cursor: - cursor.execute(sqlstr) - result = cursor.fetchall() - count += len(result) - connection.commit() - connection.close() - print(f'cleaned up {count} trajectories') - return + return if __name__ == '__main__': diff --git a/archive/ukmon_pylib/maintenance/getNextBatchStart.py b/archive/ukmon_pylib/maintenance/getNextBatchStart.py index 2dc8f39a6..d89b40304 100644 --- a/archive/ukmon_pylib/maintenance/getNextBatchStart.py +++ b/archive/ukmon_pylib/maintenance/getNextBatchStart.py @@ -6,7 +6,7 @@ import datetime import sys from crontab import CronTab -from meteortools.utils import getNextRiseSet +from utils.getRiseSet import getNextRiseSet offset = 90 diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py index 2dadf8cb6..85b3f04bc 100644 --- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py +++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py @@ -26,7 +26,7 @@ lnxdir='/home/ec2-user/keymgmt/csvkeys' else: csvdir=os.path.expanduser('~/keymgmt/csvkeys') - csvdir=os.path.expanduser('~/keymgmt/csvkeys') + lnxdir=os.path.expanduser('~/keymgmt/csvkeys') def checkIfOnGMN(camid): diff --git a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py index 59b37d3b8..e90cca86c 100644 --- a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py +++ b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py @@ -6,14 +6,11 @@ import numpy as np import os import sys -import fnmatch import matplotlib.pyplot as plt import json import cartopy.crs as ccrs -from meteortools.fileformats import UFOAnalyzerXML as ua - from PIL import Image @@ -21,15 +18,7 @@ def getBearingsForEvent(stns, fldr): brngs = np.zeros(len(stns)) listOfFiles = os.listdir(fldr) for entry in listOfFiles: - if fnmatch.fnmatch(entry, '*A.XML'): - fullname = os.path.join(fldr, entry) - xmlf = ua.UAXml(fullname) - sta, _, _, _, _, _ = xmlf.getStationDetails() - _, _, _, _, _, _, az1, _ = xmlf.getObjectStart(0) - i = stns.index(sta) - brngs[i] = az1 - - elif entry == 'FTPdetectinfo_UFO.txt': + if entry == 'FTPdetectinfo_UFO.txt': with open(os.path.join(fldr, 'FTPdetectinfo_UFO.txt'), 'r') as f: lis = f.readlines() for s in stns: diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py index 4f5f050cf..c64c746c5 100644 --- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py +++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py @@ -71,8 +71,7 @@ def createExtraJpgtxt(outdir, traj, availableimages): def createExtraJpgHtml(outdir, parentfldr, yr, ym): - mdasess = boto3.Session(profile_name='ukmda_admin') - s3mda = mdasess.client('s3') + s3mda = boto3.client('s3') webfldr = f'img/single/{yr}/{ym}' with open(os.path.join(outdir, 'extrajpgs.html'), 'w') as outf: jpgfldr = os.path.join(parentfldr, 'jpgs') @@ -194,8 +193,7 @@ def recreateOrbitFiles(outdir, pickname, doupload=False): if doupload: files = os.listdir(outdir) - mdasess = boto3.Session(profile_name='ukmda_admin') - s3mda = mdasess.client('s3') + s3mda = boto3.client('s3') if int(yr) > 2020: webfldr = f'reports/{yr}/orbits/{ym}/{ymd}' else: diff --git a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py index 24107d46b..3ef031d57 100644 --- a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py +++ b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py @@ -30,8 +30,7 @@ def findOtherBadEvents(): return badevents -def findFailedEvents(prof='ukmonshared'): - #session = boto3.Session(profile_name=prof) +def findFailedEvents(): logcli = boto3.client('logs', region_name='eu-west-2') datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') @@ -98,9 +97,8 @@ def checkFails(fails): return realfails -def rerunFails(fails, prof='ukmonshared'): - session=boto3.Session(profile_name=prof) - lambd = session.client('lambda', region_name='eu-west-2') +def rerunFails(fails): + lambd = boto3.client('lambda', region_name='eu-west-2') datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') @@ -120,10 +118,9 @@ def rerunFails(fails, prof='ukmonshared'): return -def resubmitPicklesForDay(dtstr, prof='ukmonshared'): +def resubmitPicklesForDay(dtstr): s3 = boto3.client('s3') - session = boto3.Session(profile_name=prof) - lambd = session.client('lambda', region_name='eu-west-2') + lambd = boto3.client('lambda', region_name='eu-west-2') pref = f'matches/RMSCorrelate/trajectories/{dtstr[:4]}/{dtstr[:6]}/{dtstr}/' buck = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] res = s3.list_objects_v2(Bucket=buck, Prefix=pref) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index e85ff3d66..36aeaf985 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -137,9 +137,9 @@ def backPopulate(stationid): s3bucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] basepath = os.path.expanduser('~/prod/ukmon-shared/matches/RMSCorrelate') - fldrs = glob.glob1(os.path.join(basepath, stationid), '*') + fldrs = glob.glob('*', root_dir=os.path.join(basepath, stationid)) for fldr in fldrs: - s3objects = glob.glob1(os.path.join(basepath, stationid. fldr), 'FTPd*') + s3objects = glob.glob('FTPd*', root_dir=os.path.join(basepath, stationid. fldr)) if len(s3objects) > 0: s3obj = s3objects[0] fullobj = f'matches/RMSCorrelate/{stationid}/{fldr}/{s3obj}' @@ -151,20 +151,6 @@ def backPopulate(stationid): datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) ddb = boto3.resource('dynamodb', region_name='eu-west-2') - if os.path.isfile('/sys/devices/virtual/dmi/id/board_asset_tag'): # crude check for EC2 - #print('asset tag file exists') - lis = open('/sys/devices/virtual/dmi/id/board_asset_tag').readlines() - if 'i-' in lis[0]: - sts_client = boto3.client('sts') - assumed_role_object=sts_client.assume_role( - RoleArn="arn:aws:iam::183798037734:role/service-role/S3FullAccess", - RoleSessionName="AssumeRoleSession1") - credentials=assumed_role_object['Credentials'] - - ddb = boto3.resource('dynamodb', region_name='eu-west-2', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) s,d,t,m,r = getDayCamTimings(sys.argv[1], ddb=ddb) newdata=pd.DataFrame(zip(s,d,t,m,r), columns=['stationid','upddate','uploadtime','manual','rundate']) @@ -187,49 +173,48 @@ def backPopulate(stationid): pd.options.mode.chained_assignment = 'warn' caminfo = camlist.drop(columns=['site','direction','oldcode','active','camtype','eMail', 'humanName']) - logindf = pd.read_csv(os.path.join(datadir, 'reports', 'lastlogins.txt'), names=['dateval','timeval','siteid'], skipinitialspace=True) - logindf['timeval'] = logindf.timeval.astype('str').str.pad(6,fillchar='0') - logindf.dateval.fillna('Jan-01',inplace=True) - logindf.timeval.fillna('00:00:00',inplace=True) - # handle case round yearend where the log may have prev year's details as well as current year - nowdt = datetime.datetime.now() - yrval = str(nowdt.year) + '-' - yrvalback = str(nowdt.year-1) + '-' - logindf['lastseen'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrval + logindf.dateval+'_'+logindf.timeval] - try: # will fail on 29th Feb in a leapyear, as previous year is not leap - logindf['lastseen2'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrvalback + logindf.dateval+'_'+logindf.timeval] - except Exception: - logindf['lastseen2'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrval + logindf.dateval+'_'+logindf.timeval] - logindf.loc[logindf.lastseen > nowdt, 'lastseen'] = logindf.lastseen2 - logindf = logindf.sort_values(by=['lastseen']) - logindf.drop_duplicates(subset=['siteid'], inplace=True, keep='last') - logindf.rename(columns={'siteid':'location'}, inplace=True) - logindf.drop(columns = ['lastseen2'], inplace=True) - - # create a merged dataframe with siteid and stationid - intdf = pd.merge(logindf,caminfo, on=['location'], how='outer') - - df = pd.merge(intdf, fulldf, on=['stationid']) - df.dateval.fillna('Jan-01',inplace=True) - df.timeval.fillna('00:00:00',inplace=True) - df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") - df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime - df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] - df = df.drop(columns=['timeval','stationid','manual','rundate', 'upddate','uploadtime', 'dateval']) - df['dateval']=[x.strftime('%b-%d') for x in df.lastupload] - df = df.sort_values(by=['lastupload']) - - outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') - zerodate = datetime.datetime(1970,1,1,0,0,0) - with open(outfile,'w') as outf: - outf.write('Last Upload, StationID, Last Login\n') - for _,rw in df.iterrows(): - dtval = rw.dateval - lastup = rw.lastupload.strftime('%H:%M:%S') - if pd.isnull(rw.lastseen): - lastseen = '> 1 month' - else: - lastseen = rw.lastseen.strftime('%b-%d %H:%M:%S') - if lastseen == 'Jan-01 00:00:00': - lastseen = '> 1 month' - outf.write(f'{dtval}, {lastup}, {rw.location:20s}, {lastseen}\n') + # process the last-login data from the SSHD log + lastlogs = open(os.path.join(datadir, 'reports', 'lastlogins.txt'),'r').readlines() + lodata = [] + now = datetime.datetime.now(tz=datetime.timezone.utc) + for li in lastlogs: + spls = li.split('ssh') + # could be ubuntu (auth.log) or amazon linux style log + dtstr = spls[0][spls[0].find(':')+1:][:19] + if ' ' in dtstr: + dtval = datetime.datetime.strptime(f'{now.year} {dtstr[:15]}', '%Y %b %d %H:%M:%S').replace(tzinfo=datetime.timezone.utc) + if dtval > now: + dtval = dtval.replace(year=now.year-1, tzinfo=datetime.timezone.utc) + targhost = 'ukmonhelper2' + else: + dtval = datetime.datetime.strptime(dtstr, '%Y-%m-%dT%H:%M:%S') + dtval = dtval.replace(tzinfo=datetime.timezone.utc) + targhost = 'batchserver' + location = spls[1].split(' for ')[1].split()[0] + lodata.append({'location':location, 'lastseen':dtval,'host': targhost}) + + + if len(lodata) > 0: + logindf = pd.DataFrame(lodata) + logindf = logindf.sort_values(by=['lastseen']) + logindf.drop_duplicates(subset=['location'], inplace=True, keep='last') + + # create a merged dataframe with siteid and stationid + intdf = pd.merge(logindf,caminfo, on=['location'], how='outer') + + df = pd.merge(intdf, fulldf, on=['stationid']) + df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") + df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime + df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] + df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) + df = df.sort_values(by=['lastupload']) + + outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') + zerodate = datetime.datetime(1970,1,1,0,0,0) + with open(outfile,'w') as outf: + outf.write('Last Upload, StationID, Last Login, Via\n') + for _,rw in df.iterrows(): + lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') + lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') + via = '' if pd.isnull(rw.host) else rw.host + outf.write(f'{lastup} , {rw.location:20s}, {lastlo}, {via}\n') diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index e1dcd4f96..05d098edf 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -36,7 +36,7 @@ def getDailyObsCounts(logf): return -def getMatchStats(logf): +def getMatchStats(logf, rundate): loglines = open(logf).readlines() events = [line.strip() for line in loglines if 'Analysing' in line and 'observations' in line] @@ -77,7 +77,7 @@ def getMatchStats(logf): nonphys = beglowr + badalti + badvelo + badangl tot = added + uncal + missdf - rtims = [line.strip() for line in loglines if 'runDistrib' in line] + rtims = [line.strip() for line in loglines if 'runDistrib' in line and '/prod/' not in line] stim = rtims[0][11:19] etim = rtims[-1][11:19] d1=datetime.strptime(stim,'%H:%M:%S') @@ -92,9 +92,6 @@ def getMatchStats(logf): cstime = str(d2 - d1) datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - rundate=os.getenv('rundate') - if not rundate: - rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() dailydbdir = os.path.join(datadir, 'latest','dailydbs') trajdb = TrajectoryDatabase(dailydbdir, f'{rundate}_trajectories.db') trajs = len(trajdb.getTrajBasics('.',[0,9999999])) @@ -102,11 +99,8 @@ def getMatchStats(logf): return tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime -def updateStats(obscount, candcount, trajcount, runtime): +def updateStats(obscount, candcount, trajcount, runtime, rundate): datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - rundate=os.getenv('rundate') - if not rundate: - rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() statsdir = os.path.join(datadir, 'dailyreports') reports = glob.glob(os.path.join(statsdir, f'{rundate}*.txt')) @@ -126,7 +120,10 @@ def updateStats(obscount, candcount, trajcount, runtime): if __name__ == '__main__': - tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(sys.argv[1]) - if len(sys.argv) < 3: - updateStats(added, cands, trajs, runtime) + matchfile = sys.argv[1] + rundate = sys.argv[2] + + tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(matchfile, rundate) + + updateStats(added, cands, trajs, runtime, rundate) print(tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime) diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index ab3fd50d9..3557c103d 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -41,9 +41,7 @@ def updateCamLocDirFovDB(datadir=None): def loadLocationDetails(table='camdetails', ddb=None, loadall=False): if not ddb: - prof = os.getenv('UKMPROFILE',default='ukmonshared') - conn = boto3.Session(profile_name=prof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table(table) res = table.scan() # strictly, should check for LastEvaluatedKey here, in case there was more than 1MB of data, @@ -71,7 +69,7 @@ def findLocationInfo(srchstring, ddb=None, statdets=None): return srchres -def createCDCsv(targetloc): +def createCDCsv(targetloc, srchidxdir): camdets = loadLocationDetails() cd4csv = camdets[['site','stationid','oldcode','direction','camtype','active']] pd.options.mode.chained_assignment = None @@ -83,28 +81,32 @@ def createCDCsv(targetloc): datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) outfname = os.path.join(datadir, targetloc, 'camera-details.csv') cd4csv.to_csv(outfname, index=False) - with open(os.path.join(datadir, 'activecamcount.txt'), 'w') as outf: + with open(os.path.join(datadir, targetloc, 'activecamcount.txt'), 'w') as outf: outf.write(str(len(cd4csv[cd4csv.active==1]))) - createStatOptsHtml(camdets, datadir) - createStatOptsHtml(camdets, datadir, True) - createStatOptsHtml(camdets, datadir, True, True) + + outdir = os.path.join(datadir, srchidxdir) + os.makedirs(outdir, exist_ok=True) + + createStatOptsHtml(camdets, outdir) + createStatOptsHtml(camdets, outdir, True) + createStatOptsHtml(camdets, outdir, True, True) return -def createStatOptsHtml(sitedets, datadir, active=False, locs=False): +def createStatOptsHtml(sitedets, outdir, active=False, locs=False): """ Create an HTML file containing a list of stations or locations These files are used by the website search functions """ if active: if locs: - siteidx = os.path.join(datadir, 'activestatlocs.html') + siteidx = os.path.join(outdir, 'activestatlocs.html') sitedets = sitedets.sort_values(by=['site', 'stationid']) else: - siteidx = os.path.join(datadir, 'activestatopts.html') + siteidx = os.path.join(outdir, 'activestatopts.html') sitedets = sitedets[sitedets.active==1] else: - siteidx = os.path.join(datadir, 'statopts.html') + siteidx = os.path.join(outdir, 'statopts.html') with open(siteidx, 'w') as outf: if not active: outf.write('\n') diff --git a/archive/ukmon_pylib/reports/cameraStatusReport.py b/archive/ukmon_pylib/reports/cameraStatusReport.py index 256ad0765..83e4850e0 100644 --- a/archive/ukmon_pylib/reports/cameraStatusReport.py +++ b/archive/ukmon_pylib/reports/cameraStatusReport.py @@ -30,7 +30,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): fldrlist = pd.merge(tmplist, caminfo, on=['stationid'], how='inner') nowdt = datetime.datetime.now() - fldrlist.rundate.fillna(fldrlist.upddate.astype(str) + '_' + fldrlist.uploadtime.astype(str).str.pad(width=6,fillchar='0'), inplace=True) + fldrlist.fillna({'rundate': fldrlist.upddate.astype(str) + '_' + fldrlist.uploadtime.astype(str).str.pad(width=6,fillchar='0')}, inplace=True) fldrlist['dtstamp'] = [datetime.datetime.strptime(f,'%Y%m%d_%H%M%S') for f in fldrlist.rundate] fldrlist = fldrlist.sort_values(by=['stationid','dtstamp']) fldrlist.drop_duplicates(keep='last', subset=['stationid'], inplace=True) diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 9426f5d6d..944c2ba03 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt import datetime -from meteortools.utils import jd2Date +from wmpl.Utils.TrajConversions import jd2Date def createBarChart(datadir=None, yr=None): @@ -16,7 +16,8 @@ def createBarChart(datadir=None, yr=None): yr=datetime.datetime.now().year if datadir is None: datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - fname = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) + yr = str(yr) + fname = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') if not os.path.isfile(fname): print('{} missing', fname) return None @@ -36,19 +37,20 @@ def createBarChart(datadir=None, yr=None): dts.append(dt) fig, ax = plt.subplots() - width = 0.35 + width = 0.35 nowdt=datetime.datetime.now() ax.set_ylabel('# matches') ax.set_xlabel('Date') ax.set_title('Number of matched events per day. Last updated {}'.format(nowdt.strftime('%Y-%m-%d %H:%M:%S'))) - li.append(0) # comes up one short + while len(li) < len(dts): + li.append(0) ax.bar(dts, li, width, label='Events') fig = plt.gcf() fig.set_size_inches(12, 5) fig.tight_layout() #plt.gca().invert_yaxis() - plt.savefig(os.path.join(datadir,f'Annual-{yr}.jpg'), dpi=100) + plt.savefig(os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg'), dpi=100) plt.close() return matches diff --git a/archive/ukmon_pylib/reports/createSearchableFormat.py b/archive/ukmon_pylib/reports/createSearchableFormat.py index 4424cc4bf..c567fa572 100644 --- a/archive/ukmon_pylib/reports/createSearchableFormat.py +++ b/archive/ukmon_pylib/reports/createSearchableFormat.py @@ -23,7 +23,7 @@ def checkUrl(s3, url): except Exception: #print(e) retval = '/img/missing-white.png' - print(url, retval) + #print(url, retval) return retval diff --git a/archive/ukmon_pylib/reports/createSummaryTable.py b/archive/ukmon_pylib/reports/createSummaryTable.py index a0d861bb1..b31500027 100644 --- a/archive/ukmon_pylib/reports/createSummaryTable.py +++ b/archive/ukmon_pylib/reports/createSummaryTable.py @@ -16,10 +16,13 @@ def createSummaryTable(curryr=None, datadir=None): """ if curryr is None: - curryr = str(datetime.datetime.now().year) + curryr = datetime.datetime.now().year if datadir is None: datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - fname = os.path.join(datadir, 'summarytable.js') + curryr = str(curryr) + outdir = os.path.join(datadir, 'reports', curryr) + os.makedirs(outdir, exist_ok=True) + fname = os.path.join(outdir, 'summarytable.js') with open(fname, 'w') as f: f.write('$(function() {\n') f.write('var table = document.createElement("table");\n') diff --git a/archive/ukmon_pylib/reports/dailyReport.py b/archive/ukmon_pylib/reports/dailyReport.py index a511c4055..3c5bc8979 100644 --- a/archive/ukmon_pylib/reports/dailyReport.py +++ b/archive/ukmon_pylib/reports/dailyReport.py @@ -6,7 +6,7 @@ import sys import datetime -from meteortools.utils import sendAnEmail +from utils.sendAnEmail import sendAnEmail def AddHeader(body, bodytext, stats): @@ -55,11 +55,7 @@ def AddRowRMS(body, bodytext, ele): return body, bodytext -def LookForMatchesRMS(doff, dayfile, statsfile): - # get stats - with open(statsfile, 'r') as inf: - lis = inf.readlines() - stats = lis[-1].strip().split(' ') +def LookForMatchesRMS(doff, dayfile, stats): bodytext = 'Daily notification of matches\n\n' body = '' @@ -100,7 +96,14 @@ def LookForMatchesRMS(doff, dayfile, statsfile): dailyreps.sort() dailyrep = dailyreps[-1] statfile = os.path.join(reppth, 'stats.txt') - _, _, body, bodytext = LookForMatchesRMS(doff, dailyrep, statfile) + lis = open(statfile, 'r').readlines() + stats = [li for li in lis if repdtstr in li] + if len(stats) > 0: + stats = stats[0].strip().split(' ') + else: + stats = lis[-1].strip().split(' ') + + _, _, body, bodytext = LookForMatchesRMS(doff, dailyrep, stats) #body = body.replace('assets/img/logo.svg', 'latest/dailyreports/dailyreportsidx.html') if doff == 1: @@ -123,5 +126,7 @@ def LookForMatchesRMS(doff, dayfile, statsfile): mailFrom = recs[-1].strip() #mailRecip = 'markmcintyre99@googlemail.com' # TODO for testing only yest = (datetime.date.today()-datetime.timedelta(days=doff)).strftime('%Y-%m-%d') - mailSubj = f'Latest Match Report for {yest}' - sendAnEmail(mailRecip, bodytext, mailSubj, mailFrom, msg_html=body.replace('href="/reports', f'href="{targeturl}/reports')) + # only send the email for the most recent report. + if doff == 1: + mailSubj = f'Latest Match Report for {yest}' + sendAnEmail(mailRecip, bodytext, mailSubj, mailFrom, msg_html=body.replace('href="/reports', f'href="{targeturl}/reports')) diff --git a/archive/ukmon_pylib/reports/extractors.py b/archive/ukmon_pylib/reports/extractors.py index 65670b38d..8541e7598 100644 --- a/archive/ukmon_pylib/reports/extractors.py +++ b/archive/ukmon_pylib/reports/extractors.py @@ -6,8 +6,8 @@ import os import sys import pandas as pd -from meteortools.fileformats import imoWorkingShowerList as imo -from meteortools.utils import getActiveShowers +from utils.imoWorkingShowerList import IMOshowerList +from utils.getActiveShowers import getActiveShowers def createSplitMatchFile(yr, mth=None, shwr=None, matches=None): @@ -143,7 +143,7 @@ def extractAllShowersData(ymd): showerlist = getActiveShowers(ymd, retlist=True) showerlist.append('spo') else: - sl = imo.IMOshowerList() + sl = IMOshowerList() showerlist = sl.getMajorShowers(True, True).strip().split(' ') print(f'processing data for {ymd}') @@ -159,8 +159,9 @@ def extractAllShowersData(ymd): fname = os.path.join(datadir, 'consolidated','M_{}-unified.csv'.format(yr)) if not os.path.isfile(fname): print(f'unable to open {fname}') - return - ufosingles = pd.read_csv(fname, skipinitialspace=True) + ufosingles = None + else: + ufosingles = pd.read_csv(fname, skipinitialspace=True) fname = os.path.join(datadir, 'single','singles-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): print(f'unable to open {fname}') @@ -174,8 +175,9 @@ def extractAllShowersData(ymd): print(f'processing RMS singles for {shwr}') createRMSSingleMonthlyExtract(yr, shwr=shwr, dta=rmssingles) createRMSSingleMonthlyExtract(yr, shwr=shwr, dta=rmssingles, withshower=True) - print(f'processing UFO singles for {shwr}') - createUFOSingleMonthlyExtract(yr, shwr=shwr, dta=ufosingles) + if ufosingles: + print(f'processing UFO singles for {shwr}') + createUFOSingleMonthlyExtract(yr, shwr=shwr, dta=ufosingles) return diff --git a/archive/ukmon_pylib/reports/findFailedMatches.py b/archive/ukmon_pylib/reports/findFailedMatches.py index c6005fe7e..1b110a1c6 100644 --- a/archive/ukmon_pylib/reports/findFailedMatches.py +++ b/archive/ukmon_pylib/reports/findFailedMatches.py @@ -46,7 +46,7 @@ def processOneLog(logfile, repfile): srcdir = os.getenv('SRC', default='.') logdir = os.path.join(srcdir, 'logs','distrib') - logs = glob.glob1(logdir, f'{repdt}*.log') + logs = glob.glob(f'{repdt}*.log', root_dir=logdir) for logf in logs: processOneLog(os.path.join(logdir, logf), reportfile) reportfile.close() diff --git a/archive/ukmon_pylib/reports/findFireballs.py b/archive/ukmon_pylib/reports/findFireballs.py index d30949fa6..178bd1247 100644 --- a/archive/ukmon_pylib/reports/findFireballs.py +++ b/archive/ukmon_pylib/reports/findFireballs.py @@ -12,7 +12,7 @@ from shutil import rmtree from traj.pickleAnalyser import getBestView -from meteortools.utils import jd2Date +from wmpl.Utils.TrajConversions import jd2Date # diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index bbb7f0fba..b25f8beca 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -6,8 +6,9 @@ import os import gmplot import glob +import boto3 from cryptography.fernet import Fernet -from meteortools.fileformats import readCameraKML +from utils.kmlHandlers import readCameraKML def decodeApiKey(enckey): @@ -24,16 +25,27 @@ def encodeApiKey(plainkey): return apikey.decode('utf-8') +def getApiKey(): + ssm = boto3.client('ssm') + res = ssm.get_parameter(Name='prod_gmapsapikey', WithDecryption=True) + if 'Parameter' in res: + return res['Parameter']['Value'] + else: + return False + + def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): - apikey = os.getenv('APIKEY') - apikey = decodeApiKey(apikey) + + apikey = getApiKey() + if not apikey: + return kmltempl = os.getenv('KMLTEMPLATE', default='*70km.kml') heightval = kmltempl[1:-4] gmap = gmplot.GoogleMapPlotter(52, -1.0, 5, apikey=apikey, title=f'Camera Coverage at {heightval}', map_type='satellite') - flist = glob.glob1(kmlsource, kmltempl) + flist = glob.glob(kmltempl, root_dir=kmlsource) cols = list(gmplot.color._HTML_COLOR_CODES.keys()) for col, fn in zip(cols,flist): @@ -55,14 +67,15 @@ def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): return -def createCoveragePage(): - apikey = os.getenv('APIKEY') - apikey = decodeApiKey(apikey) +def createCoveragePage(targdir): + apikey = getApiKey() + if not apikey: + return templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) with open(os.path.join(templdir, 'coverage-maps.html'), 'r') as inf: lis = inf.readlines() - with open(os.path.join(datadir, 'latest','coverage-maps.html'), 'w') as outf: + with open(os.path.join(datadir, targdir,'coverage-maps.html'), 'w') as outf: for li in lis: outf.write(li.replace('{{MAPSAPIKEY}}', apikey)) return diff --git a/archive/ukmon_pylib/reports/meteoriteTools.py b/archive/ukmon_pylib/reports/meteoriteTools.py index 4f7c72e7b..e699bb072 100644 --- a/archive/ukmon_pylib/reports/meteoriteTools.py +++ b/archive/ukmon_pylib/reports/meteoriteTools.py @@ -8,7 +8,7 @@ import numpy as np import requests -from meteortools.utils.Math import greatCircleDistance +from wmpl.Utils.Earth import greatCircleDistance def stationsNearPoint(lat, lon, dist=75, email_only=True): diff --git a/archive/ukmon_pylib/reports/reportActiveShowers.py b/archive/ukmon_pylib/reports/reportActiveShowers.py index 7a58d91fc..bf2a171a0 100644 --- a/archive/ukmon_pylib/reports/reportActiveShowers.py +++ b/archive/ukmon_pylib/reports/reportActiveShowers.py @@ -9,7 +9,7 @@ import shutil import argparse -from meteortools.utils import getActiveShowers +from utils.getActiveShowers import getActiveShowers from analysis.showerAnalysis import showerAnalysis from reports.findFireballs import findFireballs @@ -109,8 +109,8 @@ def createShowerIndexPage(dtstr, shwr, shwrname, outdir, datadir): outf.write('The graphs and histograms below show more information about the velocity, magnitude \n') outf.write('start and end altitude and other parameters. Click for larger view. \n') # add the charts and stuff - jpglist = glob.glob1(outdir, '*.jpg') - pnglist = glob.glob1(outdir, '*.png') + jpglist = glob.glob('*.jpg', root_dir=outdir) + pnglist = glob.glob('*.png', root_dir=outdir) outf.write('
    \n') for j in jpglist: outf.write(f'\n') diff --git a/archive/ukmon_pylib/reports/reportBadCameras.py b/archive/ukmon_pylib/reports/reportBadCameras.py index cc0938ee8..adf4d64ca 100644 --- a/archive/ukmon_pylib/reports/reportBadCameras.py +++ b/archive/ukmon_pylib/reports/reportBadCameras.py @@ -10,7 +10,7 @@ import textwrap from reports.CameraDetails import loadLocationDetails -from meteortools.utils import sendAnEmail +from utils.sendAnEmail import sendAnEmail mailfrom = 'ukmonhelper2@ukmeteors.co.uk' diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index 2d8738584..d79ddefff 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -12,6 +12,7 @@ import tempfile import boto3 import glob +import tarfile from traj.pickleAnalyser import getVMagCodeAndStations from reports.CameraDetails import findSite, loadLocationDetails @@ -41,11 +42,11 @@ def processLocalFolder(trajdir, basedir): return outstr -def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): +def getListOfNewMatches(dir_path, db_path, rundate): os.makedirs(db_path, exist_ok=True) - db_name = f'{rundate}_trajectories.db' if rundate else 'trajectories.db' + db_name = f'{rundate}_trajectories.db' dailydb = TrajectoryDatabase(db_path=db_path, db_name=db_name, purge_records=True) - flist = glob.glob(os.path.join(dir_path, 'trajectories_*.db')) + flist = glob.glob(os.path.join(dir_path, f'trajectories_{rundate}_*.db')) flist.sort() for fl in flist: tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') @@ -101,11 +102,11 @@ def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): return newdirs -def updatePairedDB(dir_path, db_path='/tmp', rundate=None): +def updatePairedDB(dir_path, db_path, rundate): os.makedirs(db_path, exist_ok=True) - db_name = f'{rundate}_observations.db' if rundate else 'observations.db' + db_name = f'{rundate}_observations.db' obsdb = ObservationsDatabase(db_path=db_path, db_name=db_name, purge_records=True) - flist = glob.glob(os.path.join(dir_path, 'observations_*.db')) + flist = glob.glob(os.path.join(dir_path, f'observations_{rundate}_*.db')) flist.sort() for fl in flist: tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') @@ -121,17 +122,14 @@ def updatePairedDB(dir_path, db_path='/tmp', rundate=None): return obscount -def findNewMatches(dir_path, out_path, offset, repdtstr): +def findNewMatches(dir_path, out_path, repdtstr): daily_path = os.path.join(os.path.split(dir_path)[0], 'dailydbs') newdirs = getListOfNewMatches(dir_path, daily_path, rundate=repdtstr) # load camera details caminfo = loadLocationDetails() caminfo = caminfo[caminfo.active==1] - if repdtstr is not None: - repdt = datetime.datetime.strptime(repdtstr, '%Y%m%d') - else: - repdt = datetime.datetime.now() - datetime.timedelta(int(offset)) + repdt = datetime.datetime.strptime(repdtstr, '%Y%m%d') os.makedirs(out_path, exist_ok=True) # create filename. Allow for three reruns in a day @@ -195,20 +193,28 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): if __name__ == '__main__': - repdtstr = None - if len(sys.argv) > 4: - repdtstr = sys.argv[4] cand_db_dir = sys.argv[1] daily_db_dir = sys.argv[2] - offset = sys.argv[3] + repdtstr = sys.argv[3] + + if repdtstr != datetime.datetime.now().strftime('%Y%m%d'): + print(f'running for a past date, extracting the container dbs to {cand_db_dir}') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + tarf = tarfile.open(os.path.join(datadir, 'distrib','containers',f'contdbs_{repdtstr}.tgz'), 'r') + members = [m for m in tarf.getmembers() if '.db' in m.name] + for m in members: + m.name = os.path.basename(m.name) + tarf.extractall(cand_db_dir, filter='fully_trusted') + tarf.close() flist = glob.glob(os.path.join(cand_db_dir, 'trajectories_*.db')) if len(flist) == 0: print('no container databases to process, aborting') else: # arguments dblocation, datadir, days ago, rundate eg 20220524 - findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) + findNewMatches(cand_db_dir, daily_db_dir, repdtstr) # update the daily database of paired observations daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) + print('reportOfLatestMatches finished') diff --git a/archive/ukmon_pylib/reports/trackThumbnails.py b/archive/ukmon_pylib/reports/trackThumbnails.py index 0e012e1db..65837803b 100644 --- a/archive/ukmon_pylib/reports/trackThumbnails.py +++ b/archive/ukmon_pylib/reports/trackThumbnails.py @@ -7,7 +7,7 @@ import argparse import requests from tempfile import mkdtemp -from meteortools.utils import greatCircleDistance +from wmpl.Utils.Earth import greatCircleDistance RAD2DEG=57.2958 diff --git a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py index 793c3ee11..7dac55f6c 100644 --- a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py +++ b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py @@ -17,14 +17,14 @@ def test_createBarChart(): # pass year and datadir res = createBarChart(datadir=datadir, yr=yr) - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) os.remove(outf) # pass datadir only - will fail if we're not in 2023 res = createBarChart(datadir=datadir, yr=None) - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: @@ -35,7 +35,7 @@ def test_createBarChart(): # no params passed res = createBarChart() - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: diff --git a/archive/ukmon_pylib/tests/test_reports_various.py b/archive/ukmon_pylib/tests/test_reports_various.py index d9aeac7b4..9ddc3ba64 100644 --- a/archive/ukmon_pylib/tests/test_reports_various.py +++ b/archive/ukmon_pylib/tests/test_reports_various.py @@ -10,8 +10,9 @@ def test_createSummaryTable(): - createSummaryTable(curryr=None, datadir=datadir) - fname = os.path.join(datadir, 'summarytable.js') + yr=2023 + createSummaryTable(curryr=yr, datadir=datadir) + fname = os.path.join(datadir,'reports',str(yr), 'summarytable.js') assert os.path.isfile(fname) lis = open(fname, 'r').readlines() assert 'cell.innerHTML = ' in lis[9] diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index c1fee16df..0202a06fe 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -4,11 +4,14 @@ import sys import glob import datetime +import logging from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase from wmpl.Trajectory.CorrelateRMS import RMSDataHandle from wmpl.Utils.TrajConversions import jd2Date +log = logging.getLogger("consol_disttraj") + def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=False, matchstart=3): """ @@ -63,6 +66,11 @@ def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=F dt_beg = jd2Date(int(jdt_end) + 1 - matchstart, dt_obj=True, tzinfo=datetime.timezone.utc) event_time_range = [dt_beg, dt_end] + # capture RMSDataHandle log messages to console + log.setLevel(logging.DEBUG) + console_handler = logging.StreamHandler() + log.addHandler(console_handler) + dh = RMSDataHandle(basedir, dt_range=event_time_range, db_dir=dbdir, output_dir=basedir,mcmode=1, archivemonths=0) dh.updateTrajectoryDatabase(dt_range=event_time_range) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index d3b6c99c7..dfe4624aa 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -116,7 +116,7 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): trajloc = f'trajectories/{yr}/{yr}{mth:02d}/{yr}{mth:02d}{dy:02d}' out_dir = '~/data/distrib' outf.write(f'python -c "from traj.pickleAnalyser import getAllImages;getAllImages(\'{trajloc}\', \'{out_dir}\');"\n') - outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt --quiet"\n') + outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt" --quiet\n') outf.write(f'rm {out_dir}/consumed_*.txt\n') return @@ -147,12 +147,15 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): outf.write('logger -s -t execConsol start\n') outf.write(f'aws s3 sync {srcpath}/ ~/data/distrib/canddbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --quiet\n') + # must run this before consolidation as the process updates the database to remove any missing + # trajectories ie not on disk. + outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') + refreshTrajectories(outf, matchstart, matchend, shbucket) + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/canddbs/ {calcdir}/dbs/\n') outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') - outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') - refreshTrajectories(outf, matchstart, matchend, shbucket) outf.write('logger -s -t execConsol creating density plots\n') createDensityPlots(outf, calcdir, enddt, includeyear) outf.write('logger -s -t execConsol pushing data back to S3\n') diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index e81d2f1e2..9255534e9 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -110,7 +110,7 @@ def distributeCandidates(rundate, srcdir, maxcount=20, istest=False): print(f'Reading from {srcdir}') # obtain a list of picklefiles and sort by name - flist = glob.glob1(srcdir, '*.pickle') + flist = glob.glob('*.pickle', root_dir=srcdir) if len(flist) == 0: print('no candidates to process') return False diff --git a/archive/ukmon_pylib/traj/manualBulkReruns.py b/archive/ukmon_pylib/traj/manualBulkReruns.py index 497310da0..98dd19b98 100644 --- a/archive/ukmon_pylib/traj/manualBulkReruns.py +++ b/archive/ukmon_pylib/traj/manualBulkReruns.py @@ -24,9 +24,10 @@ from wmpl.Utils.TrajConversions import geo2Cartesian from wmpl.Utils.TrajConversions import raDec2ECI from wmpl.Utils.Math import vectNorm, angleBetweenVectors, vectorFromPointDirectionAndAngle +from wmpl.Utils.TrajConversions import raDec2AltAz, altAz2RADec, datetime2JD +from wmpl.Utils.Earth import greatCircleDistance -from meteortools.utils import raDec2AltAz, altAz2RADec, datetime2JD, greatCircleDistance -from meteortools.fileformats import loadFTPDetectInfo, writeNewFTPFile +from utils.ftpDetectInfo import loadFTPDetectInfo, writeNewFTPFile # Dummy platepar structure needed by loadPlatePar @@ -199,7 +200,7 @@ def getOverlappingStations(datadir, stationid): checker = EventChecker(traj_constraints=trajcons) ppdir = os.path.join(datadir, 'consolidated','platepars') rp = loadPlatepar(ppdir, stationid) - ppfs = glob.glob1(ppdir, '*.json') + ppfs = glob.glob('*.json', root_dir=ppdir) overlaps=[] for ppf in ppfs: ppfname, _ = os.path.splitext(ppf) diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index 5afb69442..16f1fd4b4 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -14,12 +14,10 @@ import datetime import json +from utils.convertSolLon import sollon2jd +from traj.ShowerAssociation import associateShower + from wmpl.Utils.TrajConversions import jd2Date -try: - from meteortools.utils import sollon2jd - from traj.ShowerAssociation import associateShower -except: - pass from wmpl.Utils.Math import mergeClosePoints, angleBetweenSphericalCoords from wmpl.Utils.Physics import calcMass from wmpl.Utils.Pickling import loadPickle @@ -81,7 +79,7 @@ def getBestView(picklename): except Exception: pass beststatid = statids[vmags.index(bestvmag)] - imgfn = glob.glob1(outdir, '*{}*.jpg'.format(beststatid)) + imgfn = glob.glob(f'*{beststatid}*.jpg', root_dir=outdir) if len(imgfn) > 0: bestimg = imgfn[0] elif os.path.isfile(os.path.join(outdir, 'jpgs.lst')): @@ -587,19 +585,20 @@ def getListOfImages(picklename): class TrajQualityParams(object): def __init__(self): - self.min_traj_points = 6 - self.min_qc = 5.0 - self.max_e = 1.5 - self.max_radiant_err = 2.0 - self.max_vg_err = 10.0 + self.min_traj_points = 4 + self.min_qc = 3.0 + self.max_e = 2.5 + self.max_radiant_err = 5.0 + self.max_vg_err = 20.0 self.max_vg = 120.0 self.max_begin_ht = 160 - self.min_end_ht = 20 + self.min_end_ht = 0 def getAllImages(dir_path, outdir): outdir = os.path.expanduser(outdir) traj_quality_params = TrajQualityParams() + trajs = loadTrajectoryPickles(dir_path, traj_quality_params, verbose=True) imglist = [] for traj in trajs: diff --git a/archive/ukmon_pylib/utils/__init__.py b/archive/ukmon_pylib/utils/__init__.py new file mode 100644 index 000000000..1d590cf3f --- /dev/null +++ b/archive/ukmon_pylib/utils/__init__.py @@ -0,0 +1 @@ +# Copyright (C) 2018- Mark McIntyre diff --git a/archive/ukmon_pylib/analysis/analyseGmnData.py b/archive/ukmon_pylib/utils/analyseGmnData.py similarity index 100% rename from archive/ukmon_pylib/analysis/analyseGmnData.py rename to archive/ukmon_pylib/utils/analyseGmnData.py diff --git a/archive/ukmon_pylib/analysis/compareBrightnessData.py b/archive/ukmon_pylib/utils/compareBrightnessData.py similarity index 91% rename from archive/ukmon_pylib/analysis/compareBrightnessData.py rename to archive/ukmon_pylib/utils/compareBrightnessData.py index 1ace683cb..663fb9af3 100644 --- a/archive/ukmon_pylib/analysis/compareBrightnessData.py +++ b/archive/ukmon_pylib/utils/compareBrightnessData.py @@ -10,9 +10,7 @@ def getBrightnessData(yyyymmdd, outdir=None): - prof = os.getenv('UKMPROFILE',default='ukmonshared') - sess = boto3.Session(profile_name=prof) - ddb = sess.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table('LiveBrightness') resp = table.query(KeyConditionExpression=Key('CaptureNight').eq(yyyymmdd)) df = pd.DataFrame(ddbjson.loads(resp['Items'])) diff --git a/archive/ukmon_pylib/utils/convertSolLon.py b/archive/ukmon_pylib/utils/convertSolLon.py new file mode 100644 index 000000000..31e4f762c --- /dev/null +++ b/archive/ukmon_pylib/utils/convertSolLon.py @@ -0,0 +1,41 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +import numpy as np +from wmpl.Utils.TrajConversions import date2JD + + +def sollon2jd(Year, Month, Long): + """ + Calculate the julian date corresponding to a solar longitude. + Because Solar Longitude is relative to the Spring equinox, the exact date + of a given LS varies from year to year. + + Parameters: + Year: [int] year you wish to calculate in. + Month: [int] month you wish to calculate in. + Long: [float] The solar longitude to convert. + + Returns: + [float] julian date + + Notes: + The function is only stable for date ranges from 1900-2100. + """ + + Long = np.radians(Long) + N = Year - 2000 + if abs(N) > 100: + print("Algorithm is not stable for years below 1900 or above 2100") + + JDM0 = 2451182.24736 + 365.25963575 * N + ApproxJD = date2JD(Year, Month, 15, 12, 0, 0) + DiffJD = ApproxJD-2451545 + + Dt = 1.94330 * np.sin(Long - 1.798135) + 0.01305272 * np.sin(2*Long + 2.634232) + 78.195268 + 58.13165 * Long - 0.0000089408 * DiffJD + + if abs(ApproxJD - (JDM0 + Dt))>50: + Dt = Dt + 365.2596 + + JD1 = JDM0 + Dt + + return JD1 diff --git a/archive/ukmon_pylib/utils/ftpDetectInfo.py b/archive/ukmon_pylib/utils/ftpDetectInfo.py new file mode 100644 index 000000000..0e78d1300 --- /dev/null +++ b/archive/ukmon_pylib/utils/ftpDetectInfo.py @@ -0,0 +1,506 @@ +# +# Load an FTPdetectInfo file - copied from RMS +# +# Copyright (C) 2018-2023 Mark McIntyre + +import os +import numpy as np +import configparser as crp +import json +import datetime + +from wmpl.Utils.TrajConversions import date2JD +from wmpl.Utils.Math import angleBetweenSphericalCoords + + +def filterFTPforSpecificTime(ftpfile, dtstr): + """ filter FTPdetect file for a specific event, by time, and copy it into a new file + + Arguments: + ftpfile - [string] full path to the ftpdetect file to filter + dtstr - [string] date/time of required event in yyyymmdd_hhmmss format + + Returns: + tuple containing + full name of the new file containing just matching events + the number of matching events + """ + meteor_list = loadFTPDetectInfo(ftpfile, time_offsets=None, join_broken_meteors=True, locdata=None) + refdt = datetime.datetime.strptime(dtstr, '%Y%m%d_%H%M%S') + #print(refdt) + new_met_list = [] + for met in meteor_list: + #print(met.ff_name) + dtpart = datetime.datetime.strptime(met.ff_name[10:25], '%Y%m%d_%H%M%S') + tdiff = (refdt - dtpart).seconds + #print(tdiff) + if abs(tdiff) < 21: + print('adding one entry') + new_met_list.append(met) + newname = writeNewFTPFile(ftpfile, new_met_list) + return newname, len(new_met_list) + + +def writeNewFTPFile(srcname, metlist): + """ creates a FTPDetect file from a list of MeteorObservation objects + + Arguments: + srcname - [string] full path to the original FTP file + metlist - list of MeteorObservation objects + + Returns: + the full path to the created file + + """ + outdir, fname = os.path.split(srcname) + newname = os.path.join(outdir, f'{fname}.old') + try: + os.rename(srcname, newname) + except: + pass + if os.path.isfile(srcname): + srcname = srcname[:-4] + '_new.txt' + with open(srcname, 'w') as ftpf: + _writeFTPHeader(ftpf, len(metlist), outdir, False) + metno = 1 + ffname = '' + for met in metlist: + if ffname == met.ff_name: + metno = metno + 1 + else: + metno = 1 + ffname = met.ff_name + _writeOneMeteor(ftpf, metno, met.station_id, met.time_data, len(met.frames), met.fps, met.frames, + np.degrees(met.ra_data), np.degrees(met.dec_data), + np.degrees(met.azim_data), np.degrees(met.elev_data), + None, met.mag_data, False, met.x_data, met.y_data, met.ff_name) + return srcname + + +def loadFTPDetectInfo(ftpdetectinfo_file_name, time_offsets=None, + join_broken_meteors=True, locdata=None): + """ Loads an FTPDEtect file into a list of MeteorObservation objects + + Arguments: + ftpdetectinfo_file_name: [str] Path to the FTPdetectinfo file. + stations: [dict] A dictionary where the keys are stations IDs, and values are lists of: + - latitude +N in radians + - longitude +E in radians + - height in meters + + + Keyword arguments: + time_offsets: [dict] (key, value) pairs of (stations_id, time_offset) for every station. None by + default. + join_broken_meteors: [bool] Join meteors broken across 2 FF files. + + + Return: + meteor_list: [list] A list of MeteorObservation objects filled with data from the FTPdetectinfo file. + + """ + stations={} + if locdata is None: + dirname, fname = os.path.split(ftpdetectinfo_file_name) + cfgfile = os.path.join(dirname, '.config') + cfg = crp.ConfigParser() + cfg.read(cfgfile) + try: + lat = float(cfg['System']['latitude'].split()[0]) + lon = float(cfg['System']['longitude'].split()[0]) + height = float(cfg['System']['elevation'].split()[0]) + except: + # try reading from platepars file + ppf = os.path.join(dirname, 'platepars_all_recalibrated.json') + if not os.path.isfile(ppf): + return [] + js = json.load(open(ppf, 'r')) + if len(js) < 10: + return [] + lat = js[list(js.keys())[0]]['lat'] + lon = js[list(js.keys())[0]]['lon'] + height = js[list(js.keys())[0]]['elev'] + statid= fname.split('_')[1] + else: + statid = locdata['station_code'] + lat = float(locdata['lat']) + lon = float(locdata['lon']) + height = float(locdata['elev']) + + stations[statid] = [np.radians(lat), np.radians(lon), height*1000] + meteor_list = [] + + with open(ftpdetectinfo_file_name) as f: + # Skip the header + for i in range(11): + next(f) + + current_meteor = None + + bin_name = False + cal_name = False + meteor_header = False + + for line in f: + # Skip comments + if line.startswith("#"): + continue + + line = line.replace('\n', '').replace('\r', '') + + # Skip the line if it is empty + if not line: + continue + + if '-----' in line: + # Mark that the next line is the bin name + bin_name = True + + # If the separator is read in, save the current meteor + if current_meteor is not None: + current_meteor._finish() + meteor_list.append(current_meteor) + continue + + if bin_name: + bin_name = False + + # Mark that the next line is the calibration file name + cal_name = True + + # Save the name of the FF file + ff_name = line + + # Extract the reference time from the FF bin file name + line = line.split('_') + + # Count the number of string segments, and determine if it the old or new CAMS format + if len(line) == 6: + sc = 1 + else: + sc = 0 + + ff_date = line[1 + sc] + ff_time = line[2 + sc] + milliseconds = line[3 + sc] + + year = ff_date[:4] + month = ff_date[4:6] + day = ff_date[6:8] + + hour = ff_time[:2] + minute = ff_time[2:4] + seconds = ff_time[4:6] + + year, month, day, hour, minute, seconds, milliseconds = map(int, [year, month, day, hour, + minute, seconds, milliseconds]) + + # Calculate the reference JD time + jdt_ref = date2JD(year, month, day, hour, minute, seconds, milliseconds) + continue + + if cal_name: + cal_name = False + # Mark that the next line is the meteor header + meteor_header = True + continue + + if meteor_header: + meteor_header = False + line = line.split() + + # Get the station ID and the FPS from the meteor header + station_id = line[0].strip() + fps = float(line[3]) + + # Try converting station ID to integer + try: + station_id = int(station_id) + except: + pass + + # If the time offsets were given, apply the correction to the JD + if time_offsets is not None: + if station_id in time_offsets: + print('Applying time offset for station {:s} of {:.2f} s'.format(str(station_id), + time_offsets[station_id])) + + jdt_ref += time_offsets[station_id]/86400.0 + else: + print('Time offset for given station not found!') + + # Get the station data + if station_id in stations: + lat, lon, height = stations[station_id] + else: + print('ERROR! No info for station ', station_id, ' found in CameraSites.txt file!') + print('Exiting...') + break + # Init a new meteor observation + current_meteor = MeteorObservation(jdt_ref, station_id, lat, lon, height, fps, + ff_name=ff_name) + continue + + # Read in the meteor observation point + if (current_meteor is not None) and (not bin_name) and (not cal_name) and (not meteor_header): + + line = line.replace('\n', '').split() + + # Read in the meteor frame, RA and Dec + frame_n = float(line[0]) + x = float(line[1]) + y = float(line[2]) + ra = float(line[3]) + dec = float(line[4]) + azim = float(line[5]) + elev = float(line[6]) + + # Read the visual magnitude, if present + if len(line) > 8: + mag = line[8] + if mag == 'inf': + mag = None + else: + mag = float(mag) + else: + mag = None + + # Add the measurement point to the current meteor + current_meteor._addPoint(frame_n, x, y, azim, elev, ra, dec, mag) + + # Add the last meteor the the meteor list + if current_meteor is not None: + current_meteor._finish() + meteor_list.append(current_meteor) + + # Concatenate observations across different FF files ### + if join_broken_meteors: + + # Go through all meteors and compare the next observation + merged_indices = [] + for i in range(len(meteor_list)): + + # If the next observation was merged, skip it + if (i + 1) in merged_indices: + continue + + # Get the current meteor observation + met1 = meteor_list[i] + + if i >= (len(meteor_list) - 1): + break + + # Get the next meteor observation + met2 = meteor_list[i + 1] + + # Compare only same station observations + if met1.station_id != met2.station_id: + continue + + # Extract frame number + met1_frame_no = int(met1.ff_name.split("_")[-1].split('.')[0]) + met2_frame_no = int(met2.ff_name.split("_")[-1].split('.')[0]) + + # Skip if the next FF is not exactly 256 frames later + if met2_frame_no != (met1_frame_no + 256): + continue + + + # Check for frame continouty + if (met1.frames[-1] < 254) or (met2.frames[0] > 2): + continue + + # Check if the next frame is close to the predicted position + + # Compute angular distance between the last 2 points on the first FF + ang_dist = angleBetweenSphericalCoords(met1.dec_data[-2], met1.ra_data[-2], met1.dec_data[-1], + met1.ra_data[-1]) + + # Compute frame difference between the last frame on the 1st FF and the first frame on the 2nd FF + df = met2.frames[0] + (256 - met1.frames[-1]) + + # Skip the pair if the angular distance between the last and first frames is 2x larger than the + # frame difference times the expected separation + ang_dist_between = angleBetweenSphericalCoords(met1.dec_data[-1], met1.ra_data[-1], + met2.dec_data[0], met2.ra_data[0]) + + if ang_dist_between > 2*df*ang_dist: + continue + + # If all checks have passed, merge observations ### + # Recompute the frames + frames = 256.0 + met2.frames + + # Recompute the time data + time_data = frames/met1.fps + + # Add the observations to first meteor object + met1.frames = np.append(met1.frames, frames) + met1.time_data = np.append(met1.time_data, time_data) + met1.x_data = np.append(met1.x_data, met2.x_data) + met1.y_data = np.append(met1.y_data, met2.y_data) + met1.azim_data = np.append(met1.azim_data, met2.azim_data) + met1.elev_data = np.append(met1.elev_data, met2.elev_data) + met1.ra_data = np.append(met1.ra_data, met2.ra_data) + met1.dec_data = np.append(met1.dec_data, met2.dec_data) + met1.mag_data = np.append(met1.mag_data, met2.mag_data) + + # Merge the FF file name and create a list + if (met1.ff_name is not None) and (met2.ff_name is not None): + met1.ff_name = met1.ff_name + ',' + met2.ff_name + + # Sort all observations by time + met1._finish() + + # Indicate that the next observation is to be skipped + merged_indices.append(i + 1) + + # Removed merged meteors from the list + meteor_list = [element for i, element in enumerate(meteor_list) if i not in merged_indices] + + return meteor_list + + +class MeteorObservation(object): + """ Container for meteor observations. + """ + def __init__(self, jdt_ref, station_id, latitude, longitude, height, fps, ff_name=None): + """ Construct the MeteorObservation object. + Arguments: + jdt_ref: [float] Reference Julian date when the relative time is t = 0s. + station_id: [str] Station ID. + latitude: [float] Latitude +N in radians. + longitude: [float] Longitude +E in radians. + height: [float] Elevation above sea level (MSL) in meters. + fps: [float] Frames per second. + + Keyword arguments: + ff_name: [str] Name of the originating FF file. + """ + self.jdt_ref = jdt_ref + self.station_id = station_id + self.latitude = latitude + self.longitude = longitude + self.height = height + self.fps = fps + self.ff_name = ff_name + self.frames = [] + self.time_data = [] + self.x_data = [] + self.y_data = [] + self.azim_data = [] + self.elev_data = [] + self.ra_data = [] + self.dec_data = [] + self.mag_data = [] + self.abs_mag_data = [] + + def _addPoint(self, frame_n, x, y, azim, elev, ra, dec, mag): + """ Adds the measurement point to the meteor. + + Arguments: + frame_n: [flaot] Frame number from the reference time. + x: [float] X image coordinate. + y: [float] X image coordinate. + azim: [float] Azimuth, J2000 in degrees. + elev: [float] Elevation angle, J2000 in degrees. + ra: [float] Right ascension, J2000 in degrees. + dec: [float] Declination, J2000 in degrees. + mag: [float] Visual magnitude. + """ + self.frames.append(frame_n) + + # Calculate the time in seconds w.r.t. to the reference JD + point_time = float(frame_n)/self.fps + self.time_data.append(point_time) + self.x_data.append(x) + self.y_data.append(y) + + # Angular coordinates converted to radians + self.azim_data.append(np.radians(azim)) + self.elev_data.append(np.radians(elev)) + self.ra_data.append(np.radians(ra)) + self.dec_data.append(np.radians(dec)) + self.mag_data.append(mag) + + def _finish(self): + """ When the initialization is done, convert data lists to numpy arrays. """ + self.frames = np.array(self.frames) + self.time_data = np.array(self.time_data) + self.x_data = np.array(self.x_data) + self.y_data = np.array(self.y_data) + self.azim_data = np.array(self.azim_data) + self.elev_data = np.array(self.elev_data) + self.ra_data = np.array(self.ra_data) + self.dec_data = np.array(self.dec_data) + self.mag_data = np.array(self.mag_data) + # Sort by frame + temp_arr = np.c_[self.frames, self.time_data, self.x_data, self.y_data, self.azim_data, + self.elev_data, self.ra_data, self.dec_data, self.mag_data] + temp_arr = temp_arr[np.argsort(temp_arr[:, 0])] + self.frames, self.time_data, self.x_data, self.y_data, self.azim_data, self.elev_data, self.ra_data, \ + self.dec_data, self.mag_data = temp_arr.T + + +def _writeFTPHeader(ftpf, metcount, fldr, ufo=True): + """ + Internal function to create the header of the FTPDetect file + """ + l1 = 'Meteor Count = {:06d}\n'.format(metcount) + ftpf.write(l1) + ftpf.write('-----------------------------------------------------\n') + if ufo is True: + ftpf.write('Processed with UFOAnalyser\n') + else: + ftpf.write('Processed with RMS 1.0\n') + ftpf.write('-----------------------------------------------------\n') + l1 = 'FF folder = {:s}\n'.format(fldr) + ftpf.write(l1) + l1 = 'CAL folder = {:s}\n'.format(fldr) + ftpf.write(l1) + ftpf.write('-----------------------------------------------------\n') + ftpf.write('FF file processed\n') + ftpf.write('CAL file processed\n') + ftpf.write('Cam# Meteor# #Segments fps hnr mle bin Pix/fm Rho Phi\n') + ftpf.write('Per segment: Frame# Col Row RA Dec Azim Elev Inten Mag\n') + + +def _writeOneMeteor(ftpf, metno, sta, evttime, fcount, fps, fno, ra, dec, az, alt, b, mag, + ufo=True, x=None, y=None, ffname = None): + """ Internal function to write one meteor event into the file in FTPDetectInfo style + """ + ftpf.write('-------------------------------------------------------\n') + if ffname is None: + ms = '{:03d}'.format(int(evttime.microsecond / 1000)) + + fname = 'FF_' + sta + '_' + evttime.strftime('%Y%m%d_%H%M%S_') + ms + '_0000000.fits\n' + else: + fname = ffname + '\n' + ftpf.write(fname) + + if ufo is True: + ftpf.write('UFO UKMON DATA Recalibrated on: ') + else: + ftpf.write('RMS data reprocessed on: ') + ftpf.write(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f UTC\n')) + li = f'{sta} {metno:04d} {fcount:04d} {fps:04.2f} 000.0 000.0 00.0 000.0 0000.0 0000.0\n' + ftpf.write(li) + + for i in range(len(fno)): + # 204.4909 0422.57 0353.46 262.3574 +16.6355 267.7148 +23.0996 000120 3.41 + bri = 0 + if b is not None: + bri = int(b[i]) + if ufo is True: + # UFO is timestamped as at the first detection + thisfn = fno[i] - fno[0] + thisx = 0 + thisy = 0 + else: + thisfn = fno[i] + thisx = x[i] + thisy = y[i] + li = f'{thisfn:.4f} {thisx:07.2f} {thisy:07.2f} ' + li = li + f'{ra[i]:8.4f} {dec[i]:+7.4f} {az[i]:8.4f} ' + li = li + f'{alt[i]:+7.4f} {bri:06d} {mag[i]:.02f}\n' + ftpf.write(li) diff --git a/archive/ukmon_pylib/utils/getActiveShowers.py b/archive/ukmon_pylib/utils/getActiveShowers.py new file mode 100644 index 000000000..74d03b4cc --- /dev/null +++ b/archive/ukmon_pylib/utils/getActiveShowers.py @@ -0,0 +1,139 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +# simple script to get the active shower list from the IMO working list + +from utils.imoWorkingShowerList import IMOshowerList as iwsl +import datetime +import numpy as np +import os + + +def getActiveShowers(targdate, retlist=False, inclMinor=False): + """ + Return a list of showers active at the specified date + + Arguments: + targdate: [str] Date in YYYYMMDD format + + Keyword Arguments: + retlist: [bool] return a list, or print to console. Default False=print + inclMinor: [bool] include minor showers or only return major showers + + Returns: + If retlist is true, returns a python list of shower short-codes eg ['PER','LYR'] + + """ + sl = iwsl() + testdate = datetime.datetime.strptime(targdate, '%Y%m%d') + listofshowers=sl.getActiveShowers(testdate, True, inclMinor=inclMinor) + if retlist is False: + for shwr in listofshowers: + print(shwr) + else: + return listofshowers + + +def getActiveShowersStr(targdatestr): + """ + Prints a comma-separated list of showers active at the specified date + + Arguments: + targdate: [str] Date in YYYYMMDD format + + Returns: + nothing + + """ + shwrs = getActiveShowers(targdatestr, retlist=True) + shwrs.append('spo') + for s in shwrs: + print(s) + + +def getShowerDets(shwr, stringFmt=False, dataPth=None): + """ Get details of a shower + + Arguments: + shwr: [string] three-letter shower code eg PER + Keyword Arguments: + stringFmt [bool] default False, return a string rather than a list + dataPth [string] path to the datafiles. Default None means data read from internal files. + + Returns: + (id, full name, peak solar longitude, peak date mm-dd) + """ + sl = iwsl() + mtch = sl.getShowerByCode(shwr, useFull=True) + if len(mtch) > 0 and mtch['@id'] is not None: + id = int(mtch['@id']) + nam = mtch['name'] + pkdtstr = mtch['peak'] + dt = datetime.datetime.now() + yr = dt.year + pkdt = datetime.datetime.strptime(f'{yr} {pkdtstr}','%Y %b %d') + dtstr = pkdt.strftime('%m-%d') + pksollong = mtch['pksollon'] + else: + id, nam, pksollong, dtstr = 0, 'Unknown', 0, 'Unknown' + if stringFmt: + return f"{pksollong},{dtstr},{nam},{shwr}" + else: + return id, nam, pksollong, dtstr + + +def getShowerPeak(shwr): + """ Get date of a shower peak in MM-DD format + + Arguments: + shwr: [string] three-letter shower code eg PER + + Returns: + peak date mm-dd + """ + _, _, _, pk = getShowerDets(shwr) + return pk + + +def numpifyShowerData(): + """Refresh the numpy versions of the shower data files """ + srcdir = os.getenv('SRC', default=os.path.expanduser('~/prod')) + abs_path = os.getenv('WMPL_LOC', default=os.path.expanduser('~/src/WesternMeteorPyLib')) + + iau_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'streamfulldata.csv') + iau_shower_table_npy = os.path.join(abs_path, 'wmpl', 'share', 'streamfulldata.npy') + iau_shower_list = np.loadtxt(iau_shower_table_file, delimiter="|", usecols=range(20), dtype=str) + np.save(iau_shower_table_npy, iau_shower_list) + + iau_shower_table_npy = os.path.join(srcdir, 'share', 'streamfulldata.npy') + np.save(iau_shower_table_npy, iau_shower_list) + + gmn_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'gmn_shower_table_20230518.txt') + gmn_shower_table_npy = os.path.join(abs_path, 'wmpl', 'share', 'gmn_shower_table_20230518.npy') + gmn_shower_list = _loadGMNShowerTable(*os.path.split(gmn_shower_table_file)) + np.save(gmn_shower_table_npy, gmn_shower_list) + + gmn_shower_table_npy = os.path.join(srcdir, 'share', 'gmn_shower_table_20230518.npy') + np.save(gmn_shower_table_npy, gmn_shower_list) + + +def _loadGMNShowerTable(dir_path, file_name): + gmn_shower_list = [] + with open(os.path.join(dir_path, file_name), encoding='cp1252') as f: + for line in f: + if line.startswith('#'): + continue + line = line.strip() + line = line.replace('\n', '').replace('\r', '') + if not line: + continue + la_sun, L_g, B_g, v_g, dispersion, IAU_no, IAU_code = line.split() + gmn_shower_list.append([ + np.radians(float(la_sun)), + np.radians(float(L_g)), + np.radians(float(B_g)), + 1000*float(v_g), + np.radians(float(dispersion)), + int(IAU_no)] + + ) + return np.array(gmn_shower_list) diff --git a/archive/ukmon_pylib/utils/getRiseSet.py b/archive/ukmon_pylib/utils/getRiseSet.py new file mode 100644 index 000000000..081eeed98 --- /dev/null +++ b/archive/ukmon_pylib/utils/getRiseSet.py @@ -0,0 +1,32 @@ +# Copyright (C) 2018- Mark McIntyre +# +import ephem + + +def getNextRiseSet(lati, longi, elev, fordate=None): + """ Calculate the next rise and set times for a given lat, long, elev + + Paramters: + lati: [float] latitude in degrees + longi: [float] longitude in degrees (+E) + elev: [float] latitude in metres + fordate:[datetime] date to calculate for, today if none + + Returns: + rise, set: [date tuple] next rise and set as datetimes + + Note that set may be earlier than rise, if you're invoking the function during daytime. + + """ + obs = ephem.Observer() + obs.lat = float(lati) / 57.3 # convert to radians, close enough for this + obs.lon = float(longi) / 57.3 + obs.elev = float(elev) + obs.horizon = -6.0 / 57.3 # degrees below horizon for darkness + if fordate is not None: + obs.date = fordate + + sun = ephem.Sun() + rise = obs.next_rising(sun).datetime() + set = obs.next_setting(sun).datetime() + return rise, set diff --git a/archive/ukmon_pylib/utils/getUncalImages.py b/archive/ukmon_pylib/utils/getUncalImages.py new file mode 100644 index 000000000..381c96a7c --- /dev/null +++ b/archive/ukmon_pylib/utils/getUncalImages.py @@ -0,0 +1,25 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# + +import os +import datetime + + +def getUncalibratedImageList(dtstr=None): + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + now = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y%m%d') + if dtstr is not None and dtstr != now: + logfile = os.path.join(datadir, '..', 'logs', f'matchJob.log-{dtstr}') + else: + logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') + if os.path.isfile(logfile): + flines = open(logfile, 'r').readlines() + uncal = [f for f in flines if 'not recalibrated' in f] + imglist = [x.split('Skipping ')[1].split(',')[0] for x in uncal] + with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: + for li in imglist: + outf.write(f'{li}\n') + else: + print('file not found', logfile) + imglist=[] + return imglist diff --git a/archive/ukmon_pylib/utils/imoWorkingShowerList.py b/archive/ukmon_pylib/utils/imoWorkingShowerList.py new file mode 100644 index 000000000..abc76760a --- /dev/null +++ b/archive/ukmon_pylib/utils/imoWorkingShowerList.py @@ -0,0 +1,223 @@ +# +# python module to read the IMO Working Shower short List +# +# Copyright (C) 2018-2023 Mark McIntyre + +import xmltodict +import datetime +import os +import numpy as np +import copy + +from wmpl.Utils.TrajConversions import jd2Date +from utils.convertSolLon import sollon2jd + +try: + # imported from $SRC/share + from majorminor import majorlist, minorlist +except Exception: + c = ['QUA', 'LYR', 'ETA', 'CAP', 'SDA', 'PER', 'AUR', 'ORI', 'NTA', 'STA', 'LEO', 'GEM', 'URS'] + minorlist = ['SPE','OCT','DRA','EGE','MON','HYD','COM','NOO'] + + +class IMOshowerList: + """ + Class that loads and parses the IMO Working Shower list, or if needed, the unconfirmed list. + Not all known showers are in the IMO working list. If a shower is not in the Working List then + this library will reference the full shower list curated by Peter Jenniskens which contains + debated and unconfirmed showers. + + """ + def __init__(self, fname=None, fullstreamname=None): + if fname is None: + srcdir = os.getenv('SRC', default=os.path.expanduser('~/prod')) + fname = os.path.join(srcdir, 'share', 'IMO_Working_Meteor_Shower_List.xml') + if not os.path.isfile(fname): + print('unable to open data file') + return + + tmplist = xmltodict.parse(open(fname, 'rb').read()) + self.showerlist = tmplist['meteor_shower_list']['shower'] + if fullstreamname is None: + fullstreamname = os.path.join(srcdir, 'share', 'streamfulldata.npy') + self.fullstreamdata = np.load(fullstreamname) + #print('initialised') + + + def getShowerByCode(self, iaucode, useFull=False): + ds = {'@id':None, 'IAU_code':None,'start':None, 'end':None, + 'peak':None, 'r':None, 'name':None, 'V':None, 'ZHR':None, 'RA':None, 'DE':None, 'pksollon': None} + ds2 = {'@id':None, 'IAU_code':None,'start':None, 'end':None, + 'peak':None, 'r':None, 'name':None, 'V':None, 'ZHR':None, 'RA':None, 'DE':None, 'pksollon': None} + for shower in self.showerlist: + if shower['IAU_code'] == iaucode: + ds = shower + if ds['@id'] is None: + ds['@id'] = -1 + pksollong = -1 + #print(ds) + subset = self.fullstreamdata[np.where(self.fullstreamdata[:,3]==iaucode)] + if subset is not None and len(subset) > 0: + mtch = [sh for sh in subset if int(sh[6]) > -1] + if len(mtch) > 0: + ds2 = copy.deepcopy(ds) + ds2['IAU_code'] = mtch[-1][3].strip() + ds2['name'] = mtch[-1][4].strip() + ds2['V'] = mtch[-1][12] + ds2['@id'] = mtch[-1][1] + ds2['RA'] = mtch[-1][8] + ds2['DE'] = mtch[-1][9] + + pksollong = float(mtch[-1][7]) + dt = datetime.datetime.now() + yr = dt.year + mth = dt.month + jd = sollon2jd(yr, mth, pksollong) + pkdt = jd2Date(jd, dt_obj=True) + ds2['peak'] = pkdt.strftime('%h %d') + # start/end pop idx, ZHR not available in the IAU data + ds2['start'] = (pkdt + datetime.timedelta(days=-2)).strftime('%h %d') + ds2['end'] = (pkdt + datetime.timedelta(days=2)).strftime('%h %d') + ds2['pksollon'] = pksollong + #print(ds2) + else: + # okay so its poor quality but lets try it anyway + mtch = subset + ds2 = copy.deepcopy(ds) + ds2['IAU_code'] = mtch[-1][3].strip() + ds2['name'] = mtch[-1][4].strip() + ds2['V'] = mtch[-1][12] + ds2['@id'] = mtch[-1][1] + ds2['RA'] = mtch[-1][8] + ds2['DE'] = mtch[-1][9] + + pksollong = float(mtch[-1][7]) + dt = datetime.datetime.now() + yr = dt.year + mth = dt.month + jd = sollon2jd(yr, mth, pksollong) + pkdt = jd2Date(jd, dt_obj=True) + ds2['peak'] = pkdt.strftime('%h %d') + # start/end pop idx, ZHR not available in the IAU data + ds2['start'] = (pkdt + datetime.timedelta(days=-2)).strftime('%h %d') + ds2['end'] = (pkdt + datetime.timedelta(days=2)).strftime('%h %d') + ds2['pksollon'] = pksollong + if useFull is False: + if 'pksollon' not in ds: + ds['pksollon'] = ds2['pksollon'] + elif ds['pksollon'] is None: + ds['pksollon'] = ds2['pksollon'] + if ds['peak'] is None: + ds['peak'] = ds2['peak'] + ds['@id'] = ds2['@id'] + return ds + else: + ds2['ZHR'] = ds['ZHR'] + ds2['r'] = ds['r'] + return ds2 + + def getStart(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + if shower['start'] is not None: + startdate = datetime.datetime.strptime(shower['start'], '%b %d') + else: + startdate = datetime.datetime.strptime(shower['peak'], '%b %d') + datetime.timedelta(days=-3) + if iaucode == 'QUA' and mth !=12: + # quadrantids straddle yearend + now = now - 1 + startdate = startdate.replace(year=now) + return startdate + + def getEnd(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + #print(shower) + if shower['end'] is not None: + enddate = datetime.datetime.strptime(shower['end'], '%b %d') + else: + enddate = datetime.datetime.strptime(shower['peak'], '%b %d') + datetime.timedelta(days=3) + if iaucode == 'QUA' and mth == 12: + # quadrantids straddle yearend + now = now + 1 + enddate = enddate.replace(year=now) + return enddate + + def getPeak(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + enddate = datetime.datetime.strptime(shower['peak'], '%b %d') + if iaucode == 'QUA' and mth == 12: + # quadrantids straddle yearend + now = now + 1 + enddate = enddate.replace(year=now) + return enddate + + def getRvalue(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['r'] + + def getName(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['name'] + + def getVelocity(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['V'] + + def getZHR(self, iaucode): + shower = self.getShowerByCode(iaucode) + zhr = shower['ZHR'] + if zhr is None: + return -1 + else: + return int(zhr) + + def getRaDec(self, iaucode): + shower = self.getShowerByCode(iaucode) + return float(shower['RA']), float(shower['DE']) + + def getActiveShowers(self, datetotest, majorOnly=False, inclMinor=False): + activelist = [] + for shower in self.showerlist: + shwname = shower['IAU_code'] + if shwname == 'ANT': #skip the anthelion source, its not a real shower + continue + start = self.getStart(shwname, datetotest.strftime('%Y%m%d')) + #print(shwname, start,shower) + end = self.getEnd(shwname, datetotest.strftime('%Y%m%d')) + datetime.timedelta(days=3) + if datetotest > start and datetotest < end: + if majorOnly is False or (majorOnly is True and shwname in majorlist): + activelist.append(shwname) + elif inclMinor is True and shwname in minorlist: + activelist.append(shwname) + return activelist + + def getMajorShowers(self, includeSpo=False, stringFmt=False): + majlist = majorlist + if includeSpo is True: + majlist.append('spo') + if stringFmt is True: + tmplist = '' + for shwr in majlist: + tmplist = tmplist + shwr + ' ' + majlist = tmplist + return majlist diff --git a/archive/ukmon_pylib/utils/kmlHandlers.py b/archive/ukmon_pylib/utils/kmlHandlers.py new file mode 100644 index 000000000..60f958b24 --- /dev/null +++ b/archive/ukmon_pylib/utils/kmlHandlers.py @@ -0,0 +1,149 @@ +import xmltodict +from shapely.geometry import Polygon +import csv +import simplekml +import numpy as np +import pandas as pd +import os + + +def readCameraKML(kmlFilename, return_poly=False): + """ Load a KML file and return either a list of lats and longs, or a Shapely polygon + + Arguments: + kmlFilename: [string] full path to the KML file to consume + return_poly: [bool] return a Shapely polygon? Default False + + + Returns: + if return_poly is false, returns a tuple of (cameraname, lats, longs) where lats and longs are lists of the + latitudes and longitudes in the KML file. + + If return_poly is true, returns a tuple of (cameranamem, shapely Polygon) + """ + + with open(kmlFilename) as fd: + x = xmltodict.parse(fd.read()) + cname = x['kml']['Folder']['name'] + coords = x['kml']['Folder']['Placemark']['MultiGeometry']['Polygon']['outerBoundaryIs']['LinearRing']['coordinates'] + coords = coords.split('\n') + if return_poly is False: + lats = [] + lngs = [] + for lin in coords: + s = lin.split(',') + lngs.append(float(s[0])) + lats.append(float(s[1])) + return cname, lats, lngs + else: + ptsarr=[] + for lin in coords: + s = lin.split(',') + ptsarr.append((float(s[0]), float(s[1]))) + polyg = Polygon(ptsarr) + return cname, polyg + + +def trackCsvtoKML(trackcsvfile, trackdata=None, saveOutput=True, outdir=None): + """ + Either reads a CSV file containing lat, long, height of an event and + creates a 3d KML file from it or, if trackdata is populated, converts a Pandas dataframe containing + the same data. Output is written to disk unless saveOutput is false. + + Arguments: + trackcsvfile: [string] full path to the file to read from + trackdata: [array] pandas dataframe containing the data. Default None + saveOutput: [bool] write the KML file to disk. Default true + outdir: [string] where to save the file. Default same folder as the source file + + Returns: + the KML file as a tuple + """ + kml=simplekml.Kml() + kml.document.name = trackcsvfile + if trackdata is None: + inputfile = csv.reader(open(trackcsvfile)) + for row in inputfile: + #columns are lat, long, height, times + kml.newpoint(name='', coords=[(row[1], row[0], row[2])]) + else: + for i,r in trackdata.iterrows(): + kml.newpoint(name=f'{r[3]:.5f}', coords=[(r[1], r[0], r[2])], extrude=1, altitudemode='absolute') + if 'csv' in trackcsvfile: + outname = trackcsvfile.replace('.csv','.kml') + else: + outname = f'{trackcsvfile}.kml' + if saveOutput: + if outdir is None: + outdir, _ = os.path.split(trackcsvfile) + os.makedirs(outdir, exist_ok=True) + outname = os.path.join(outdir, outname) + kml.save(outname) + return kml + + +def trackKMLtoCsv(kmlfile, kmldata = None, saveOutput=True, outdir=None): + """ convert a track KML retrieved by ukmondb.trajectoryKML to a 3 dimensional CSV file + containing coordinates of points on the trajectory. + + Arguments: + kmlfile [string] full path to the KML file to read from. + kmldata [kml] the kml data if available. + saveOutput [bool] Default True, save the output to file. + outdir [string] where to save to if saveOutput is True. + + Note: if kmldata is supplied, then kmlfile is ignored. + + Returns: + a Pandas dataframe containing the lat, long, alt and time of each + point on the trajectory, sorted by time. + + """ + with open(kmlfile) as fd: + x = xmltodict.parse(fd.read()) + placemarks=x['kml']['Document']['Placemark'] + lats = [] + lons = [] + alts = [] + tims = [] + for pm in placemarks: + tims.append(float(pm['name'])) + coords = pm['Point']['coordinates'].split(',') + lons.append(float(coords[0])) + lats.append(float(coords[1])) + alts.append(float(coords[2])) + df = pd.DataFrame({"lats": lats, "lons": lons, "alts": alts, "times": tims}) + df = df.sort_values(by=['times', 'lats']) + if saveOutput: + fname = kmlfile + if outdir is None: + outdir, fname = os.path.split(kmlfile) + outf = os.path.join(outdir, fname).replace('.kml', '.csv').replace('.KML','.csv') + df.to_csv(outf, index=False) + return df + + +def getTrackDetails(traj): + """ Get track details from a WMPL trajectory object + + Arguments: + traj: a WMPL trajectory object containing observations + + Returns: + a Pandas dataframe containing the lat, long, alt and time of each point on the trajectory, sorted by time. + """ + lats = [] + lons = [] + alts = [] + lens = [] + # Go through observation from all stations + for obs in traj.observations: + # Go through all observed points + for i in range(obs.kmeas): + lats.append(np.degrees(obs.model_lat[i])) + lons.append(np.degrees(obs.model_lon[i])) + alts.append(obs.model_ht[i]) + lens.append(obs.time_data[i]) + df = pd.DataFrame({"lats": lats, "lons": lons, "alts": alts, "times": lens}) + df = df.sort_values(by=['times', 'lats']) + return df diff --git a/archive/ukmon_pylib/analysis/meteorFlux.py b/archive/ukmon_pylib/utils/meteorFlux.py similarity index 100% rename from archive/ukmon_pylib/analysis/meteorFlux.py rename to archive/ukmon_pylib/utils/meteorFlux.py diff --git a/archive/ukmon_pylib/utils/sendAnEmail.py b/archive/ukmon_pylib/utils/sendAnEmail.py new file mode 100644 index 000000000..a7d215fba --- /dev/null +++ b/archive/ukmon_pylib/utils/sendAnEmail.py @@ -0,0 +1,192 @@ +# Copyright (C) 2018-2023 Mark McIntyre +import os +import platform +import base64 +import email +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart + +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from google_auth_oauthlib.flow import InstalledAppFlow +from googleapiclient.discovery import build + +SCOPES =['https://mail.google.com/'] + + +def _getGmailCreds(tokfile=None, crdfile=None): + creds = None + # The token stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first + # time. + if tokfile is None: + tokfile = '~/.ssh/gmailtoken.json' + if crdfile is None: + crdfile = '~/.ssh/gmailcreds.json' + tokfile = os.path.expanduser(tokfile) + crdfile = os.path.expanduser(crdfile) + if os.path.exists(tokfile): + creds = Credentials.from_authorized_user_file(tokfile, SCOPES) + + # If there are no (valid) credentials available, ask the user to log in interactively. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + if not os.path.isfile(crdfile): + print(f'to use this function you must have stored your Google OAUTH2 secret json file in {crdfile}') + print('To set up OAUTH2 go to your google cloud console, select APIs then Credentials, and add an OAUTH2 desktop client ID.') + return None + flow = InstalledAppFlow.from_client_secrets_file(crdfile, SCOPES) + creds = flow.run_local_server(port=0) + # Save the credentials for the next run + with open(tokfile, 'w') as token: + token.write(creds.to_json()) + return creds + + +def _refreshCreds(tokfile=None, crdfile=None): + creds = None + if tokfile is None: + tokfile = '~/.ssh/gmailtoken.json' + if crdfile is None: + crdfile = '~/.ssh/gmailcreds.json' + tokfile = os.path.expanduser(tokfile) + crdfile = os.path.expanduser(crdfile) + if os.path.exists(tokfile): + creds = Credentials.from_authorized_user_file(tokfile, SCOPES) + + # If there are no (valid) credentials available, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + try: + creds.refresh(Request()) + except: + flow = InstalledAppFlow.from_client_secrets_file(crdfile, SCOPES) + creds = flow.run_local_server(port=0) + # Save the credentials for the next run + if creds.valid: + with open(tokfile, 'w') as token: + token.write(creds.to_json()) + else: + print('credentials not valid, try again') + return creds + + +def _create_message(sender, to, subject, message_text, msg_text_html=None): + if msg_text_html is None: + msg = MIMEText(message_text) + else: + msg = MIMEMultipart('alternative') + msg.attach(MIMEText(message_text, 'plain')) + msg.attach(MIMEText(msg_text_html, 'html')) # html must be the last part + msg['to'] = to + msg['from'] = sender + msg['subject'] = subject + return {'raw': base64.urlsafe_b64encode(msg.as_string().encode('utf-8')).decode('utf-8')} + + + +def sendAnEmail(mailrecip, message, subject, mailfrom, files=None, tokfile=None, crdfile=None, msg_html=None): + """ sends an email using gmail. + + Arguments: + mailrecip: [string] email address of recipient. + message: [string] the message to send. + subject: [string] Subject line. + mailfrom: [string] email address of sender. + Keyword Args: + files: [list] list of files to attach, not currently implemented. + tokfile: [string] full path to token file, if not ~/.ssh/gmailtoken.json + crdfile: [string] full path to credentials file, if not ~/.ssh/gmailcreds.json + msg_html: [string] HTML version of the message body, if any + + Returns: + Nothing, though a message is printed onscreen. + + Notes: + You must have gmail OAUTH2 set up. The gmail credentials default to gmailtoken.json and + gmailcreds.json in the $HOME/.ssh folder. + """ + + if subject is None: + subject = platform.uname()[1] + + # email a summary to the mailrecip + creds = _getGmailCreds(tokfile, crdfile) + if not creds: + return + service = build('gmail', 'v1', credentials=creds) + + subj = subject + mailmsg = _create_message(mailfrom, mailrecip, subj, message, msg_text_html=msg_html) + + try: + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + print('Message Id: %s' % retval['id']) + except: + print('An error occurred sending the message') + + +def forwardAnEmail(reciplist, msgid=None, tokfile=None, crdfile=None): + """ Forward email from a gmail account to a list of recipients + + Arguments: + recplist: [list of strings] list of addresses to forward to + + Keyword arguments: + msgid: [string] gmail message id. If None, then all unread mail is forwarded + tokfile: [string] full path to a gmail oauth2 json token file + crdfile: [string] full path to a gmail oauth2 json credentials file for initial authorization + + To obtain the oauth2 credentials file, go to the google cloud console, select APIs and enable gmail. Then + go to Credentials and create an OAUTH2 Client ID for Desktop and download the JSON credentials file. + Upon first run, you'll be taken to the gmail authorisation screen and the token file will be created. + Subsequent runs will use the token file. The creds file will be used to reauthorise periodically. + """ + creds = _getGmailCreds(tokfile, crdfile) + if not creds: + return + service = build('gmail', 'v1', credentials=creds) + userid = 'me' + if msgid is None: + try: + msglist = (service.users().messages().list(userId=userid, labelIds=['INBOX','UNREAD']).execute()) + for msg in msglist['messages']: + msgid = msg['id'] + # retrieve raw mail + message = (service.users().messages().get(userId='me', id=msgid, format='raw').execute()) + # decode it from base64 + decmsg = base64.urlsafe_b64decode(message['raw']) + newmsg = email.message_from_bytes(decmsg) + newmsg.add_header('Reply-To',newmsg['from']) + newmsg.replace_header('Subject','Fwd: ' + newmsg['subject']) + # Send it + for recip in reciplist: + newmsg.replace_header('To', recip) + mailmsg = {'raw': base64.urlsafe_b64encode(newmsg.as_string().encode('utf-8')).decode('utf-8')} + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + # This will mark the messagea as read + service.users().messages().modify(userId=userid, id=msgid, body={'removeLabelIds': ['UNREAD']}).execute() + print(retval) + except Exception: + print('Nothing to forward') + else: + try: + message = (service.users().messages().get(userId='me', id=msgid, format='raw').execute()) + # decode it from base64 + decmsg = base64.urlsafe_b64decode(message['raw']) + newmsg = email.message_from_bytes(decmsg) + newmsg.add_header('Reply-To',newmsg['from']) + newmsg.replace_header('Subject','Fwd: ' + newmsg['subject']) + # Send it + for recip in reciplist: + newmsg.replace_header('To', recip) + mailmsg = {'raw': base64.urlsafe_b64encode(newmsg.as_string().encode('utf-8')).decode('utf-8')} + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + # This will mark the messagea as read + service.users().messages().modify(userId=userid, id=msgid, body={'removeLabelIds': ['UNREAD']}).execute() + print(retval) + except Exception: + print('Nothing to forward') + return diff --git a/archive/utils/README.md b/archive/utils/README.md index e179ca9f5..d1317bbe9 100644 --- a/archive/utils/README.md +++ b/archive/utils/README.md @@ -3,27 +3,36 @@ This folder contains some utility functions useful for managing the environment. ## Used by the Batch +* cleanupDeletedTrajs.sh - checks for and removes any logically-deleted trajectories eg duplicates +* getCostmetrics.sh - extract costs data from AWS and push to the website +* loadMatchCsvMDB.sh - loads the match data into MariaDB +* loadSingleCsvMDB.sh - loads the single-station data into MariaDB +* loadBrightCsvMDB.sh - loads brightness data into MariaDB * clearSpace.sh - used by the batch to delete old logs etc -* clearCaches.sh - clears the in-memory cache. Not used much. +* clearCaches.sh - clears the in-memory cache before and after running memory-intensive jobs. + +## Other routine maintenance +* checkAndRollKeys.sh - rolls station AWS Key/Secret pairs periodically to avoid stale keys +* statsToMqtt.sh - posts server space/memory etc statistics to MQ +* userAudit.sh - performs a user audit and emails a report ## User tools to maintain data * deleteOrbit.sh - remove an orbit from the database and website * updateFireballFlag.sh - marks or unmarks a match as a Fireball * updateFireballImage.sh - updates the image shown for a fireball -* stopstart-calcengine.sh - stops/starts the calculation engine server +* stopstartCalcengine.sh - stops/starts the calculation engine server ## Used by the deployment tool * makeConfig.sh - used by the deployment process to make the config file ## Used in case one of the Lambdas fails -* rerunconsolidateFTPdetect.sh - reruns the lambda -* rerunFTPtoUkMONlambra.sh - reruns the lambda -* rerun GetExtraFiles.sh - reruns the lambda +* rerunFTPtoUkMONlambra.sh - reruns FTPtoUKMON lambda +* rerunGetExtraFiles.sh - reruns GetExtraFiles lambda * cftpd_templ.json - template used by the above ## The following are work in progress -* monthlyClearDown.sh - should be run monthly to clear down old data * createTestDataSet.sh - work in progress +* getGmnData.sh - pulls GMN datasets from the GMN server ## Copyright All code Copyright (C) 2018-2023 Mark McIntyre diff --git a/archive/utils/checkCalcServerRunTime.sh b/archive/utils/checkCalcServerRunTime.sh deleted file mode 100644 index 8380d69f3..000000000 --- a/archive/utils/checkCalcServerRunTime.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# script to extract the start/stop times of the Calculation Engine server from the logs -# I have a spreadsheet that consumes this and works out the cost for various models of server (c8g.2xlarge, c8g.4xlarge etc) - -cd ~/prod/logs - -csl=/tmp/calcserver_timings.txt -csl2=/tmp/csl.txt -[ -f $csl ] && rm $csl -touch $csl - -ls -1tr nightly*.gz | while read i -do - mfz=matchJob${i:10:50} - gunzip $i - gunzip $mfz - nf=$(basename $i .gz) - mf=$(basename $mfz .gz) - egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl - gzip $nf - gzip $mf -done -ls -1tr nightlyJob* | grep -v gz | while read nf -do - mf=matchJob${nf:10:50} - egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl -done -[ -f $csl2 ] && rm $csl2 -touch $csl2 -cat $csl | while read i -do - parta=$(echo $i | awk -F ">" '{print $2}') - partb=$(echo $parta | awk -F " " '{printf("%s, %s, %s\n",$1,$2,$3)}') - echo $partb >> $csl2 -done -[ -f $csl ] && rm $csl -python << EOD -lis = open('$csl2','r').readlines() -data = [x.strip().split(',') for x in lis] -currdt = None -results = [] -res = [] -col = 0 -with open('/tmp/computeservertimes.csv','w') as outf: - for i in range(len(data)): - mth = data[i][0] - dy = data[i][1] - tim = data[i][2] - dtstr = f'{mth} {dy}' - if currdt is None: - currdt = dtstr - if currdt != dtstr: - outf.write(','.join(res) + '\n') - res = [] - col = 0 - if col == 0: - res.append(dtstr) - res.append(tim) - else: - res.append(tim) - col += 1 - currdt = dtstr - outf.write(','.join(res) + '\n') -EOD -[ -f $csl2 ] && rm $csl2 diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh index bd0836b9a..70774dd1d 100644 --- a/archive/utils/cleanupDeletedTrajs.sh +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -14,50 +14,56 @@ cd ${DATADIR}/distrib startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") -echo "checking main storage, website and csvfiles" +logger -s -t cleanupDeletedTrajs "starting: checking main storage and website" sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do - export moved=0 - trajdir=$(dirname $traj) - #echo $trajdir - srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir - trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir - aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet - export moved=1 - done - yr=${trajdir:13:4} - trajpth=$(basename $trajdir) - webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth - newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth - aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv $webloc/$fname $newloc/$fname --quiet - export moved=1 - done - - trajpth1=${trajdir:34:19} - csvname=$(echo $trajpth1 | sed 's/_/-/g') - csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv - newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} - aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do - aws s3 mv $csvloc/$fname $newloc/$fname --quiet - export moved=1 - done - csvloc=$DATADIR/orbits/${yr}/fullcsv/processed - ls -1 $csvloc/ | grep $csvname | while read fname; do - aws s3 mv $csvloc/$fname $newloc/$fname --quiet - export moved=1 - done - [ $moved == 1 ] && echo "moved $trajpth" + # check if there are two trajectories with the same folder - we don't want to delete the active one + cnt=$(sqlite3 $DATADIR/distrib/trajectories.db "select count(traj_id) from trajectories where traj_file_path='$traj';") + moved=0 + if [ $cnt == 1 ] ; then + trajdir=$(dirname $traj) + srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir + trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir + aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet + moved=1 + done + yr=${trajdir:13:4} + trajpth=$(basename $trajdir) + webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth + aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv $webloc/$fname $newloc/$fname --quiet + moved=1 + done + [ $moved == 1 ] && echo "moved $trajpth" + else + echo skipping $traj + fi done -echo "checking consolidated matches" +logger -s -t cleanupDeletedTrajs "checking raw fullcsv data files" +yr=${startdt:0:4} +csvloc=matches/${yr}/fullcsv +newloc=matches/duplicates/csvs +python -c "from maintenance.dataMaintenance import removeDeletedTrajCsv;removeDeletedTrajCsv('$csvloc', '$newloc')" + +# cater for year-end +newyr=$(date +%Y) +if [ $newyr != $yr ] ; then + csvloc=matches/${newyr}/fullcsv + newloc=matches/duplicates/csvs + python -c "from maintenance.dataMaintenance import removeDeletedTrajCsv;removeDeletedTrajCsv('$csvloc', '$newloc')" + +fi + +logger -s -t cleanupDeletedTrajs "checking consolidated matches" lasttraj=$(sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | tail -1) trajdir=$(dirname $lasttraj) yr=${trajdir:13:4} trajpth=$(basename $trajdir) matchcsv=$DATADIR/matched/matches-full-${yr}.csv -grep $trajpth $matchcsv +grep $trajpth $matchcsv > /dev/null if [ $? == 0 ] ; then python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$matchcsv')" python -m converters.toParquet $matchcsv @@ -70,7 +76,6 @@ if [ $? == 0 ] ; then aws s3 sync $DATADIR/searchidx/ $WEBSITEBUCKET/search/indexes/ --exclude "*" --include "*allevents.csv" --quiet fi -export AWS_PROFILE=ukmonshared # needed for mariadb connection details -echo "checking Mariadb Database" -python -c "from maintenance.dataMaintenance import removeDelTrajFromDb;removeDelTrajFromDb()" -unset AWS_PROFILE +# no need to check the SQL database in mariadb, as it is populated from the CSV file that +# we cleaned up above +logger -s -t cleanupDeletedTrajs "finished cleanupDeletedTrajs" diff --git a/archive/utils/liveevent.template b/archive/utils/liveevent.template deleted file mode 100644 index 02af78a63..000000000 --- a/archive/utils/liveevent.template +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Records": [ - { - "s3": { - "bucket": { - "name": "ukmda-live", - "arn": "arn:aws:s3:::ukmda-live" - }, - "object": { - "key": "KEYHERE" - } - } - } - ] -} diff --git a/archive/utils/updateDb.sh b/archive/utils/loadBrightnessCsvMDB.sh similarity index 69% rename from archive/utils/updateDb.sh rename to archive/utils/loadBrightnessCsvMDB.sh index 501497f0b..b38b95c39 100644 --- a/archive/utils/updateDb.sh +++ b/archive/utils/loadBrightnessCsvMDB.sh @@ -11,8 +11,10 @@ else rundt=$1 fi -cd ~/prod/data/brightness -mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD + +cd $DATADIR/brightness +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value| sed 's/"//g') +mysql -p$passwd -ubatch -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; load data local infile './CaptureNight_${rundt}.csv' diff --git a/archive/utils/loadMatchCsvMDB.sh b/archive/utils/loadMatchCsvMDB.sh index b2fd845f7..4dfaccb51 100644 --- a/archive/utils/loadMatchCsvMDB.sh +++ b/archive/utils/loadMatchCsvMDB.sh @@ -17,8 +17,9 @@ if [ ! -f $csvname ] ; then echo "$csvname not found" ; exit ; fi echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch -passwd=$(cat ~/.ssh/db_batch.passwd ) -mysql -u${user} -p${passwd} << EOD +# leading space to prevent being inserted into environment +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') +mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.matches; LOAD DATA LOCAL INFILE '$csvname' INTO TABLE ukmon.matches diff --git a/archive/utils/loadSingleCsvMDB.sh b/archive/utils/loadSingleCsvMDB.sh index 09fdaaaab..6d7b7b02b 100644 --- a/archive/utils/loadSingleCsvMDB.sh +++ b/archive/utils/loadSingleCsvMDB.sh @@ -17,8 +17,9 @@ if [ ! -f $csvname ] ; then echo "$csvname not found" ; exit ; fi echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch -passwd=$(cat ~/.ssh/db_batch.passwd ) -mysql -u${user} -p${passwd} << EOD +# leading space to prevent being inserted into environment +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') +mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.singles; LOAD DATA LOCAL INFILE '$csvname' INTO TABLE ukmon.singles diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index a0aa9d567..d68f3f9cb 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -8,7 +8,7 @@ fi RUNTIME_ENV=$1 envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') -export AWS_PROFILE=default +#export AWS_PROFILE=default # read from AWS SSM Parameterset SRC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_srcdir --query Parameters[0].Value | tr -d '"') @@ -19,23 +19,21 @@ UKMONLIVEBUCKET=s3://$(aws ssm get-parameters --region eu-west-2 --names ${envna RMS_LOC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_rmshome --query Parameters[0].Value | tr -d '"') WMPL_LOC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_wmplhome --query Parameters[0].Value | tr -d '"') SERVERINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcinstance --query Parameters[0].Value | tr -d '"') -BKPINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_backupinstance --query Parameters[0].Value | tr -d '"') SERVERSSHKEY=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_sshkey --query Parameters[0].Value | tr -d '"') NJLOGGRP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_batchloggroup --query Parameters[0].Value | tr -d '"') SERVERUSERID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcuser --query Parameters[0].Value | tr -d '"') +CALCSERVERIP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcserverip --query Parameters[0].Value | tr -d '"') unset AWS_PROFILE # hardcoded PYLIB=$SRC/ukmon_pylib TEMPLATES=$SRC/website/templates -RCODEDIR=$SRC/R DATADIR=$SRC/data AWS_DEFAULT_REGION=eu-west-2 MATCHSTART=2 MATCHEND=0 RMS_ENV=RMS WMPL_ENV=wmpl -APIKEY=$(cat ~/.ssh/gmapsapikey) KMLTEMPLATE=*70km.kml # variables that can be used in scripts and python files to ensure test data doesnt overwrite live data @@ -60,11 +58,10 @@ echo "UKMONSHAREDBUCKET=${UKMONSHAREDBUCKET}" >> ${CFGFILE} echo "UKMONLIVEBUCKET=${UKMONLIVEBUCKET}" >> ${CFGFILE} echo "PYLIB=${PYLIB}" >> ${CFGFILE} echo "TEMPLATES=${TEMPLATES}" >> ${CFGFILE} -echo "RCODEDIR=${RCODEDIR}" >> ${CFGFILE} echo "DATADIR=${DATADIR}" >> ${CFGFILE} echo "SERVERINSTANCEID=${SERVERINSTANCEID}" >> ${CFGFILE} echo "SERVERUSERID=${SERVERUSERID}" >> ${CFGFILE} -echo "BKPINSTANCEID=${BKPINSTANCEID}" >> ${CFGFILE} +echo "CALCSERVERIP=${CALCSERVERIP}" >> ${CFGFILE} echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> ${CFGFILE} echo "RMS_ENV=${RMS_ENV}" >> ${CFGFILE} echo "WMPL_ENV=${WMPL_ENV}" >> ${CFGFILE} @@ -73,7 +70,6 @@ echo "WMPL_LOC=${WMPL_LOC}" >> ${CFGFILE} echo "MATCHSTART=${MATCHSTART}" >> ${CFGFILE} echo "MATCHEND=${MATCHEND}" >> ${CFGFILE} echo "SERVERSSHKEY=${SERVERSSHKEY}" >> ${CFGFILE} -echo "APIKEY=${APIKEY}" >> ${CFGFILE} echo "KMLTEMPLATE=${KMLTEMPLATE}" >> ${CFGFILE} echo "NJLOGGRP=${NJLOGGRP}" >> ${CFGFILE} echo "TESTMODE=${TESTMODE}" >> ${CFGFILE} @@ -81,11 +77,11 @@ echo "TESTSUFF=${TESTSUFF}" >> ${CFGFILE} echo "" >> ${CFGFILE} echo "export RUNTIME_ENV SRC SITEURL" >> ${CFGFILE} echo "export WEBSITEBUCKET UKMONSHAREDBUCKET" >> ${CFGFILE} -echo "export PYLIB TEMPLATES RCODEDIR DATADIR BKPINSTANCEID AWS_DEFAULT_REGION" >> ${CFGFILE} +echo "export PYLIB TEMPLATES DATADIR AWS_DEFAULT_REGION" >> ${CFGFILE} echo "export RMS_ENV RMS_LOC WMPL_ENV WMPL_LOC" >> ${CFGFILE} echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFILE} echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} -echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID SERVERUSERID NJLOGGRP" >> ${CFGFILE} +echo "export KMLTEMPLATE SERVERINSTANCEID SERVERUSERID CALCSERVERIP NJLOGGRP" >> ${CFGFILE} echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib" >> ${CFGFILE} echo "export TESTMODE TESTSUFF" >> ${CFGFILE} @@ -93,8 +89,6 @@ echo "export TESTMODE TESTSUFF" >> ${CFGFILE} echo "source ~/.condaon" >> ${CFGFILE} # function to log to cloudwatch echo 'function log2cw() { ' >> ${CFGFILE} -echo 'aws logs put-log-events --log-group-name $1 --log-stream-name $2 --log-events "[{\"timestamp\": $(date +%s%3N), \"message\": \"$3\"}]" --profile ukmonshared > /dev/null ' >> ${CFGFILE} +echo 'aws logs put-log-events --log-group-name $1 --log-stream-name $2 --log-events "[{\"timestamp\": $(date +%s%3N), \"message\": \"$3\"}]" > /dev/null ' >> ${CFGFILE} echo 'logger -s -t $4 RUNTIME $SECONDS $3' >> ${CFGFILE} echo '}' >> ${CFGFILE} - - diff --git a/archive/utils/monthlyCleardown.sh b/archive/utils/monthlyCleardown.sh deleted file mode 100644 index 89ca8f7bf..000000000 --- a/archive/utils/monthlyCleardown.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# script to backup then clear down orbits from the local disk. - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -what=$1 -if [ "$what" == "" ] ; then - echo "usage: ./monthlyCleardown.sh single|raw|matched|reports|search|other" - exit -fi - -cyr=$(date +%Y) -cmt=$(date +%m) -lyr=$(date --date='last year' +%Y) -lym=$(date --date='last month' +%Y%m) - -mkdir -p $DATADIR/olddata - -# SINGLE STATION DATA -case "$what" in -single) - logger -s -t monthlyClearDown "single-station data" - cd $DATADIR/single/new/processed - # backup then remove last year's data if still present - if compgen -G "$DATADIR/single/new/processed/ukmon*_${lyr}????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-singlecsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-singlecsv.tar ukmon*_${lyr}????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-singlecsv.tar ukmon*_${lyr}????_*.csv - fi - rm -f $DATADIR/single/new/processed/ukmon*_${lyr}????_*.csv - else - echo "no ukmon-csv files from ${lyr} to archive" - fi - # backup last month's data - if compgen -G "$DATADIR/single/new/processed/ukmon*_${lym}??_*.csv" > /dev/null ; then - if [ $cmt -ne 1 ] ; then - if [ ! -f $DATADIR/olddata/${lym}-singlecsv.tar ] ; then - tar cvf $DATADIR/olddata/${lym}-singlecsv.tar ukmon*_${lym}??_*.csv - else - tar uvf $DATADIR/olddata/${lym}-singlecsv.tar ukmon*_${lym}??_*.csv - fi - fi - rm -f $DATADIR/single/new/processed/ukmon*_${lym}??_*.csv - else - echo "no ukmon-csv files from ${lym} to archive" - fi - ;; -raw) - logger -s -t monthlyClearDown "rawcsv data" - # raw CSV files from cameras - cd $DATADIR/single/rawcsvs - # backup then remove last year's data if still present - if compgen -G "$DATADIR/single/rawcsvs/*${lyr}????_??????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-rawcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lyr}????_??????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lyr}????_??????_*.csv - fi - find . -maxdepth 1 -name "*${lyr}????_??????_*.csv" -print0 | xargs -0 rm -f - else - echo "no rawcsv files from ${lyr} to archive" - fi - # backup last month's data - if compgen -G "$DATADIR/single/rawcsvs/*${lym}??_??????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-rawcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lym}??_??????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lym}??_??????_*.csv - fi - rm -f $DATADIR/single/rawcsvs/*${lym}??_??????_*.csv - else - echo "no rawcsv files from ${lym} to archive" - fi - ;; -matched) - logger -s -t monthlyClearDown "match data" - # MATCHES - # last year's fullcsv files - cd $DATADIR/orbits/$lyr/fullcsv/processed - if compgen -G "$DATADIR/orbits/${lyr}/fullcsv/processed/${lyr}*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-matchfullcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-matchfullcsv.tar ${lyr}*.csv - else - tar uvf $DATADIR/olddata/${lyr}-matchfullcsv.tar ${lyr}*.csv - fi - rm -f $DATADIR/orbits/${lyr}/fullcsv/processed/${lyr}*.csv - else - echo "no fullcsv files from ${lyr} to archive" - fi - - # and now last months fullcsv data - cd $DATADIR/orbits/$cyr/fullcsv/processed - if compgen -G "$DATADIR/orbits/${cyr}/fullcsv/processed/${lym}??-*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lym}-matchfullcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lym}-matchfullcsv.tar ${lym}??-*.csv - else - tar uvf $DATADIR/olddata/${lym}-matchfullcsv.tar ${lym}??-*.csv - fi - rm -f $DATADIR/orbits/${cyr}/fullcsv/processed/${lym}??-*.csv - else - echo "no fullcsv files from ${lym} to archive" - fi - ;; -reports) - logger -s -t monthlyClearDown "reports and so forth" - # SHOWER, MONTHLY AND ANNUAL REPORTS FROM PREVIOUS YEAR - cd $DATADIR/reports/${lyr} - nf=$(ls -1 | wc -l) - if [ $nf -ne 0 ] ; then - if [ ! -f $DATADIR/olddata/${lyr}reports.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-reports.tar * - else - tar uvf $DATADIR/olddata/${lyr}-reports.tar * - fi - rm -Rf * - else - echo "no report files from ${lyr} to archive" - fi - ;; -search) - logger -s -t monthlyClearDown "old search indexes" - # SEARCH INDEXES FOR PRIOR YEAR - cd $DATADIR/searchidx - if compgen -G "$DATADIR/searchidx/${lyr}-allevents.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-searchidx.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-searchidx.tar ${lyr}-allevents.csv - else - tar uvf $DATADIR/olddata/${lyr}-searchidx.tar ${lyr}-allevents.csv - fi - rm -Rf $DATADIR/searchidx/${lyr}-allevents.csv - else - echo "no search indexes from ${lyr} to archive" - fi - ;; -ftpdata) - logger -s -t monthlyClearDown "old ftpdetect and platepars_all files" - # cleardown the raw input data. This is already backed up elsewhere - inputym=$(date --date='3 months ago' +%Y%m) - #python $PYLIB/maintenance/dataMaintenance.py $inputym - ;; -*) - echo missing argument - ;; -esac -# now push it all to the offline backup -logger -s -t monthlyClearDown "syncing to backup bucket" -aws s3 sync $DATADIR/olddata/ s3://ukmda-admin/backups/ --exclude "*" --include "${lyr}*.tar" --quiet -rm -f $DATADIR/olddata/${lyr}*.tar - -aws s3 sync $DATADIR/olddata/ s3://ukmda-admin/backups/ --exclude "*" --include "${lym}*.tar" --quiet -# dont do this in case data arrives late -#rm -f $DATADIR/olddata/${lym}*.tar diff --git a/archive/utils/rerunFTPtoUKMONlambda.sh b/archive/utils/rerunFTPtoUKMONlambda.sh index 9c768d024..38d6b1f63 100644 --- a/archive/utils/rerunFTPtoUKMONlambda.sh +++ b/archive/utils/rerunFTPtoUKMONlambda.sh @@ -22,6 +22,6 @@ cat /tmp/ftp2ukmon.txt | while read i do fullname=${i} cat $SRC/utils/cftpd_templ.json | sed "s|KEYGOESHERE|${fullname}|g" > ./tmp.json - aws lambda invoke --profile ukmonshared --function-name ftpToUkmon --log-type Tail --payload file://./tmp.json --region eu-west-2 --cli-binary-format raw-in-base64-out res.log + aws lambda invoke --function-name ftpToUkmon --log-type Tail --payload file://./tmp.json --region eu-west-2 --cli-binary-format raw-in-base64-out res.log done rm ./tmp.json \ No newline at end of file diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh index e2334bfee..170ebb0bf 100644 --- a/archive/utils/statsToMqtt.sh +++ b/archive/utils/statsToMqtt.sh @@ -10,7 +10,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then msg="$(hostname) diskspace was ${diskpct}%" subj="Diskspace alert for $(hostname)" hn="$(hostname)@aws" - python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" + python -c "from utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi elif [ "$(hostname)" == "ukmcalcserver" ] ; then source $HOME/venvs/wmpl/bin/activate @@ -31,6 +31,6 @@ else msg="$(hostname) diskspace is ${diskpct}%" subj="Diskspace alert for $(hostname)" hn="$(hostname)@aws" - python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" + python -c "from utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi fi diff --git a/archive/utils/stopstart-calcengine.sh b/archive/utils/stopstartCalcengine.sh similarity index 90% rename from archive/utils/stopstart-calcengine.sh rename to archive/utils/stopstartCalcengine.sh index 85d31f29c..366ed507f 100644 --- a/archive/utils/stopstart-calcengine.sh +++ b/archive/utils/stopstartCalcengine.sh @@ -5,4 +5,4 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 -aws ec2 $1-instances --instance-ids $SERVERINSTANCEID --region eu-west-2 --profile ukmonshared +aws ec2 $1-instances --instance-ids $SERVERINSTANCEID --region eu-west-2 diff --git a/archive/utils/userAudit.sh b/archive/utils/userAudit.sh index 205f7e209..98e7123f0 100644 --- a/archive/utils/userAudit.sh +++ b/archive/utils/userAudit.sh @@ -8,22 +8,24 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -cd $DATADIR -echo "{\"Subject\": {\"Data\": \"User Audit\",\"Charset\": \"utf-8\"}," > auditmsg.json -echo -n "\"Body\": {\"Text\": {\"Data\": \"" >> auditmsg.json +echo "{\"Subject\": {\"Data\": \"User Audit\",\"Charset\": \"utf-8\"}," > $DATADIR/admin/auditmsg.json +echo -n "\"Body\": {\"Text\": {\"Data\": \"" >> $DATADIR/admin/auditmsg.json # general audit of all users export AWS_PROFILE=ukmonshared -python -m maintenance.getUserAndKeyInfo audit > $DATADIR/useraudit.txt +python -m maintenance.getUserAndKeyInfo audit > $DATADIR/admin/useraudit.txt # accounts not used for 30 days or more export MAXAGE=30 -python -m maintenance.getUserAndKeyInfo dormant >> $DATADIR/useraudit.txt +python -m maintenance.getUserAndKeyInfo dormant >> $DATADIR/admin/useraudit.txt -awk -v ORS='\\n' '1' useraudit.txt >> auditmsg.json -echo "\",\"Charset\": \"utf-8\"}}}" >> auditmsg.json +awk -v ORS='\\n' '1' $DATADIR/admin/useraudit.txt >> $DATADIR/admin/auditmsg.json +echo "\",\"Charset\": \"utf-8\"}}}" >> $DATADIR/admin/auditmsg.json + +pushd $DATADIR/admin aws ses send-email --destination ToAddresses="markmcintyre99@googlemail.com" --message file://auditmsg.json --from "noreply@ukmeteors.co.uk" --region eu-west-2 +popd +rm $DATADIR/admin/auditmsg.json unset MAXAGE unset AWS_PROFILE -rm auditmsg.json diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh index 537a7b2fe..385fb0353 100644 --- a/archive/website/cameraStatusReport.sh +++ b/archive/website/cameraStatusReport.sh @@ -27,9 +27,28 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} export PYTHONPATH=$PYLIB logger -s -t cameraStatusReport "starting" -sudo grep publickey /var/log/secure* | grep -v ec2-user | awk '{printf("%s-%s, %s, %s\n", $1, $2, $3,$9)}' | awk -F ":" '{print $2$3$4}' > $DATADIR/reports/lastlogins.txt +echo TEMPORARY HACK REMOVE CODE PERTAINING TO OLD SERVER +platform=$(grep "^NAME" /etc/os-release | awk -F'"' '{print $2}') +if [ "$platform" == "Ubuntu" ] ; then + authlog=/var/log/auth.log + batchuser=ubuntu +else + authlog=/var/log/secure + batchuser=ec2-user +fi +sudo grep publickey $authlog* | grep -v $batchuser > $DATADIR/reports/lastlogins.txt + +if [ "$platform" == "Ubuntu" ] +then + ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | grep -v ec2-user" > $DATADIR/reports/lastlogins_old.txt +fi +cat $DATADIR/reports/lastlogins_old.txt >> $DATADIR/reports/lastlogins.txt +echo TO HERE python -m metrics.camMetrics $rundate +rm -f $DATADIR/reports/lastlogins.txt +rm -f $DATADIR/reports/lastlogins_old.txt + aws s3 cp $DATADIR/reports/stationlogins.txt $WEBSITEBUCKET/reports/stationlogins.txt --region eu-west-2 --quiet diff --git a/archive/website/costReport.sh b/archive/website/costReport.sh index cd60ec06f..1ad8efa04 100644 --- a/archive/website/costReport.sh +++ b/archive/website/costReport.sh @@ -27,10 +27,8 @@ if [ $# -gt 1 ] ; then numdays = $1 ; fi mkdir -p $DATADIR/costs > /dev/null -export AWS_PROFILE=ukmonshared logger -s -t costReport "getting data for $numdays for $AWS_PROFILE" python -m metrics.costMetrics $DATADIR/costs eu-west-2 $numdays -unset AWS_PROFILE tod=$(date +%d) if [ "$tod" == "03" ] ; then @@ -74,8 +72,8 @@ done echo "var outer_div = document.getElementById(\"docindex\"); outer_div.appendChild(table); })" >> $frjs logger -s -t costReport "publishing data" -aws s3 sync $DATADIR/costs/ $WEBSITEBUCKET/docs/financial/raw/ --exclude "*" --include "costs-18*.csv" --quiet --profile ukmonshared -aws s3 cp $costfile $WEBSITEBUCKET/reports/ --quiet --profile ukmonshared -aws s3 cp $DATADIR/$imgfile3 $WEBSITEBUCKET/reports/ --quiet --profile ukmonshared -aws s3 cp $frjs $WEBSITEBUCKET/docs/ --quiet --profile ukmonshared +aws s3 sync $DATADIR/costs/ $WEBSITEBUCKET/docs/financial/raw/ --exclude "*" --include "costs-18*.csv" --quiet +aws s3 cp $costfile $WEBSITEBUCKET/reports/ --quiet +aws s3 cp $DATADIR/$imgfile3 $WEBSITEBUCKET/reports/ --quiet +aws s3 cp $frjs $WEBSITEBUCKET/docs/ --quiet logger -s -t costReport "done" diff --git a/archive/website/createReportIndex.sh b/archive/website/createReportIndex.sh index edf24c4f8..f758ae1d2 100644 --- a/archive/website/createReportIndex.sh +++ b/archive/website/createReportIndex.sh @@ -60,7 +60,7 @@ if [ -f $curryr/tmp.txt ] ; then rm -f $curryr/tmp.txt ; fi aws s3 ls $WEBSITEBUCKET/reports/$curryr/ | grep PRE | egrep -v "ALL|orbits|stations|fireballs|showers|.js|.html" | awk '{print $2 }' | while read j do - python -c "from meteortools.utils import getShowerDets ; x=getShowerDets('${j:0:3}', True); print(x)" >> $curryr/tmp.txt + python -c "from utils.getActiveShowers import getShowerDets ; x=getShowerDets('${j:0:3}', True); print(x)" >> $curryr/tmp.txt done sort -n $curryr/tmp.txt > $curryr/shwrs.txt rm -f $curryr/tmp.txt diff --git a/archive/website/createShwrExtracts.sh b/archive/website/createShwrExtracts.sh index beb015b48..d90f0463e 100644 --- a/archive/website/createShwrExtracts.sh +++ b/archive/website/createShwrExtracts.sh @@ -44,7 +44,7 @@ cd $DATADIR/browse/showers/ # get a list of files on the website aws s3 ls $WEBSITEBUCKET/browse/showers/ | awk '{ print $4 }' | grep csv > /tmp/browseshwr.txt -shwrs=$(python -c "from meteortools.utils.getActiveShowers import getActiveShowersStr ; getActiveShowersStr('${ymd}')") +shwrs=$(python -c "from utils.getActiveShowers import getActiveShowersStr ; getActiveShowersStr('${ymd}')") for shwr in $shwrs do now=$(date '+%Y-%m-%d %H:%M:%S') diff --git a/archive/website/createSummaryTable.sh b/archive/website/createSummaryTable.sh index fab46348f..caa0fd83b 100644 --- a/archive/website/createSummaryTable.sh +++ b/archive/website/createSummaryTable.sh @@ -9,6 +9,7 @@ # Consumes # the single station and matched station data files in $DATADIR/single and $DATADIR/matched # the most recent match run logfile +# information about the number of active cameras # # Produces # a webpage showing the current summary - the site homepage @@ -32,37 +33,37 @@ python -c "from reports.createSummaryTable import createSummaryTable; createSumm logger -s -t createSummaryTable "create a coverage map from the kmls" # make sure correct version of GEOS and PROJ4 available for mapping routines -LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib -export LD_LIBRARY_PATH +#LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib +#export LD_LIBRARY_PATH mkdir -p $DATADIR/kmls aws s3 sync $WEBSITEBUCKET/img/kmls/ $DATADIR/kmls --quiet export KMLTEMPLATE="*25km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} export KMLTEMPLATE="*70km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} export KMLTEMPLATE="*100km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} -python -c "from reports.makeCoverageMap import createCoveragePage as ccp ; ccp() ;" +python -c "from reports.makeCoverageMap import createCoveragePage as ccp ; ccp('reports/${yr}') ;" logger -s -t createSummaryTable "create year-to-date barchart" -pushd $DATADIR python -c "from reports.createAnnualBarChart import createBarChart; createBarChart('${DATADIR}','${yr}')" -popd # update index page -numcams=$(cat $DATADIR/activecamcount.txt) -cat $TEMPLATES/frontpage.html | sed "s/#NUMCAMS#/$numcams/g" > $DATADIR/newindex.html +numcams=$(cat $DATADIR/consolidated/activecamcount.txt) +cat $TEMPLATES/frontpage.html | sed "s/#NUMCAMS#/$numcams/g" > $DATADIR/reports/${yr}/newindex.html logger -s -t createSummaryTable "copying to website" -aws s3 cp $DATADIR/latest/coverage-maps.html $WEBSITEBUCKET/latest/ --quiet -aws s3 cp $DATADIR/summarytable.js $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-100km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/coverage.html --quiet -aws s3 cp $DATADIR/coverage-25km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/Annual-${yr}.jpg $WEBSITEBUCKET/YearToDate.jpg --quiet -aws s3 cp $DATADIR/newindex.html $WEBSITEBUCKET/index.html --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-maps.html $WEBSITEBUCKET/latest/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-100km.html $WEBSITEBUCKET/data/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-70km.html $WEBSITEBUCKET/data/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-70km.html $WEBSITEBUCKET/data/coverage.html --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-25km.html $WEBSITEBUCKET/data/ --quiet + +aws s3 cp $DATADIR/reports/${yr}/Annual-${yr}.jpg $WEBSITEBUCKET/YearToDate.jpg --quiet + +aws s3 cp $DATADIR/reports/${yr}/newindex.html $WEBSITEBUCKET/index.html --quiet +aws s3 cp $DATADIR/reports/${yr}/summarytable.js $WEBSITEBUCKET/data/ --quiet aws s3 cp $SRC/website/templates/header.html $WEBSITEBUCKET/templates/ --quiet aws s3 cp $SRC/website/templates/footer.html $WEBSITEBUCKET/templates/ --quiet diff --git a/archive/website/updateCameraDets.sh b/archive/website/updateCameraDets.sh new file mode 100644 index 000000000..c620bde50 --- /dev/null +++ b/archive/website/updateCameraDets.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre +# create various html and js files of camera info for the website +# +# Parameters +# none +# +# Consumes +# +# +# Produces +# camera locations file cameraLocs.json +# camera details file camera-details.csv +# three html files used by the search page + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# load the configuration +source $here/../config.ini >/dev/null 2>&1 +conda activate ${WMPL_ENV} + +logger -s -t updateCameraDets "starting" + +python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();" +aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --quiet +aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --quiet +aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --quiet + +# create the CSV file of camera info and the html versions for search functions on the website + +python -c "from reports.CameraDetails import createCDCsv; createCDCsv('consolidated','searchidx');" +aws s3 cp $DATADIR/consolidated/camera-details.csv $UKMONSHAREDBUCKET/consolidated/ --quiet +aws s3 cp $DATADIR/searchidx/statopts.html $WEBSITEBUCKET/search/ --quiet +aws s3 cp $DATADIR/searchidx/activestatopts.html $WEBSITEBUCKET/search/ --quiet +aws s3 cp $DATADIR/searchidx/activestatlocs.html $WEBSITEBUCKET/search/ --quiet + +logger -s -t updateCameraDets "finished" diff --git a/bumpver.toml b/bumpver.toml index f8ef5f0f4..17d501fc7 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2026.04.1" +current_version = "2026.05.0" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/install_or_update.sh b/install_or_update.sh index 8efa03f02..83713e346 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -10,18 +10,63 @@ envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -mkdir -p ~/${envname} - +echo "Updating codebase..." cd $here/archive +git pull -for loc in analysis ukmon_pylib website cronjobs utils share static_content +echo "Updating code..." +mkdir -p ~/${envname} +[ -d ~/$envname/data ] && msg="upgrade" || msg="install" +for loc in analysis ukmon_pylib website cronjobs utils static_content do - rsync -avz $loc/ ~/${envname}/$loc + rsync -a --delete $loc/ ~/${envname}/$loc chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 done +rsync -a share/ ~/${envname}/share + +echo "Creating data folders..." DATADIR=~/$envname/data -mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} +mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls,manualuploads} mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs +mkdir -p ~/.logrotate +mkdir -p ~/.aws +echo "Checking conda environment..." +if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] +then + pushd $here + source ~/.condaon + conda activate wmpl + pip install -r archive/ukmon_pylib/additional_requirements.txt | grep -v "already satisfied" + echo "" + popd +fi + +echo "Updating meteor shower tables..." +# update the IMO and GMN meteor shower tables if missing +if [ ! -f ~/${envname}/share/IMO_Working_Meteor_Shower_List.xml ] +then + ~/$envname/cronjobs/getImoWSfile.sh + echo "" +fi +echo "" +read -n 1 -p "Update $envname configuration files? (y/N) " yesno +if [[ "$yesno" == "y" || "$yesno" == "Y" ]] +then + echo "" + echo "Updating config for $envname" + ~/$envname/utils/makeConfig.sh $RUNTIME_ENV + for fil in .bashrc .bash_aliases .vimrc .condaon + do + rsync -a server_setup/$fil ~ + done + mkdir -p ~/.logrotate + rsync -a server_setup/logs.conf ~/.logrotate +else + echo "" + echo skipping config and bashrc +fi +echo "" +echo "$msg complete" diff --git a/using_vault.sh b/using_vault.sh deleted file mode 100644 index b87bd80d5..000000000 --- a/using_vault.sh +++ /dev/null @@ -1,49 +0,0 @@ -# testing vault - -# set the default to JSON exports -export VAULT_FORMAT="json" - -# create an approle thats bound to your cidr blocks - vault write auth/approle/role/usermaint \ - secret_id_ttl=10m \ - token_num_uses=10 \ - token_ttl=20m \ - token_max_ttl=30m \ - secret_id_num_uses=40 \ - secret_id_bound_cidrs="192.168.1.0/24" - -# a policy must exist to allow the role to access the secret -echo 'path "secret/test" { capabilities = ["list"]}' > usermaint.hcl -echo 'path "secret/test" { capabilities = ["read"] }' >> usermaint.hcl -vault policy write usermaint usermaint.hcl -vault write auth/approle/role/usermaint policies=usermaint - -# a token must exist for the server to initially authenticate. -# This needs a policy which grants access the approle -# in a corporate environment this step would be done with LDAP or AD authentication - -echo 'path "auth/approle/role" { capabilities = ["list"]}' > server1.hcl -echo 'path "auth/approle/role/usermaint/*" { capabilities = ["read", "update"] }' >> server1.hcl -vault policy write server1 server1.hcl -servertoken=$(vault token create -policy=server1 -format=json | jq -r .auth.client_token) - - -# now on the target server we can use this unprivileged token to retrieve a more privileged one -export VAULT_TOKEN=$servertoken -# get the role ID - this could be statically stored, its not considered Secret -vault_role_id=$(vault read -format=json auth/approle/role/usermaint/role-id | jq -r .data.role_id) - -# then either get a wrapped token and unwrap it to get a secret -# this is more secure because the wrapped token is single-use and expires after 120s -wraptoken=$(vault write -format=json -wrap-ttl=120s -f auth/approle/role/usermaint/secret-id | jq -r .wrap_info.token) -vault_secret_id=$(vault unwrap -format=json $wraptoken | jq -r .data.secret_id) - -# or just straight up get the secret. -vault_secret_id=$(vault write -format=json -force auth/approle/role/usermaint/secret-id | jq -r .data.secret_id) - -# then login to get a token with permission to read the actual secret -vault_token=$(vault write -format=json auth/approle/login role_id=$vault_role_id secret_id=$vault_secret_id | jq -r .auth.client_token) - -# now you can use vault_token in subsequent calls eg -VAULT_TOKEN=$vault_token vault kv get -format=json -mount=secret test - diff --git a/utils/reimageBatchServer.ps1 b/utils/reimageBatchServer.ps1 new file mode 100644 index 000000000..8f513e4df --- /dev/null +++ b/utils/reimageBatchServer.ps1 @@ -0,0 +1,9 @@ +# script to re-image the UKMON calc server +# copyright Mark McIntyre, 2026- + +$dtstr=(get-date -uformat "%Y%m%d") + +$instdetails=(aws ec2 describe-instances --filters Name="tag:Name",Values="batchserver" --profile ukmonshared --region eu-west-2) +$instanceid= (Write-Output $instdetails| convertfrom-json)[0].reservations.instances.instanceid + +aws ec2 create-image --instance-id $instanceid --name "batchserver_${dtstr}" --description "Latest Batchserver image" --profile ukmonshared --region eu-west-2 \ No newline at end of file