diff --git a/.gitignore b/.gitignore index e897bccc6..d266d9591 100644 --- a/.gitignore +++ b/.gitignore @@ -531,13 +531,13 @@ StandaloneCurator/.installed analysis/orbitSolver/testing analysis/FormatConverters/testing */test_data/* -archive/samfunctions*/pytz* +archive/lambdas*/pytz* archive/static_content/data/searchdialog.js ToDo* archive/costs/**/*.csv archive/costs/**/*.json archive/costs/plots/*.png -archive/samfunctions*/.aws-sam/build.toml +archive/lambdas*/.aws-sam/build.toml archive/costs/S3-size-empireelements.xlsx archive/ukmon_pylib/share/maps/BM*.jpg archive/ukmon_pylib/share/maps/BM*.png @@ -556,11 +556,10 @@ archive/unused/** ukmon_pylib/tests/20220217_032206.832_UK/* unused/* *tfstate.lock.info -archive/samfunctions*/getExtraFilesV2/pythoncode/WesternMeteorPyLib/* -archive/samfunctions*/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/* -archive/containers/trajsolver/WesternMeteorPyLib/* +archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/** +archive/containers/trajsolver/WesternMeteorPyLib/** archive/containers/trajsolver/awskeys -archive/containers/trajsolvertest/WesternMeteorPyLib/* +archive/containers/trajsolvertest/WesternMeteorPyLib/** archive/containers/trajsolvertest/awskeys #ukmon_pylib/traj/clusdetails.txt **/*.pickle @@ -568,11 +567,11 @@ archive/containers/gui/awskeys archive/terraform/*/ukmonreadonly.key .cache_ggshield replacements.txt -archive/samfunctions*/apis-mm.txt -archive/samfunctions*/apis-ee.txt +archive/lambdas/apis-mm.txt +archive/lambdas/apis-ee.txt usermgmt/windows/createUserKeyAWS.ps1 -archive/samfunctions*/matchPickle/pythoncode/WesternMeteorPyLib/** -archive/samfunctions*/matchPickle/pythoncode/*.npy +archive/lambdas/matchPickle/pythoncode/WesternMeteorPyLib/** +archive/lambdas/matchPickle/pythoncode/*.npy .terraform/ archive/utils/tmp.json usermgmt/windows/caminfo/camera-details-new.csv @@ -589,8 +588,8 @@ ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M002.ecsv ukmon_pylib/tests/usertools/2021-07-17T02_41_05_05_M004.ecsv ukmon_pylib/tests/usertools/20230202_014115.kml archive/containers*/simpleUI/awskeys -archive/samfunctions*/camDetails/pythoncode/camDetails.py -archive/samfunctions*/camDetails/pythoncode/awskeys +archive/lambdas*/camDetails/pythoncode/camDetails.py +archive/lambdas*/camDetails/pythoncode/awskeys archive/containers/RMS-docker/rms_docker.tar archive/containers/RMS-docker/Dockerfile_orig.txt @@ -609,11 +608,11 @@ usermgmt/server/ukmon*.json **/awskeys.test **/trajsolver_old/* archive/tmp/* -servercopy/files/* +servercopy/* tests/testing/RMS/ tests/testing/WesternMeteorPyLib/ archive/ukmon_pylib/tests/testdata.tar.gz -servercopybkp/* +servercopydev/* fbcollector/config.ini usermgmt/windows/stationmaint.ini usermgmt/windows/README.md diff --git a/README.md b/README.md index 226593df5..595de7c54 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2024.04.2 +version: 2026.04.1 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index 9fefc6338..108c7fbf6 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,5 +1,8 @@ Data Processing and Flows ========================== + +version: 2026.04.1 + This diagram shows the overall flow of data from Cameras to websites and out to the public. ```mermaid diff --git a/archive/analysis/README.md b/archive/analysis/README.md index a260be3d5..a20d70e2b 100644 --- a/archive/analysis/README.md +++ b/archive/analysis/README.md @@ -1,17 +1,20 @@ # analysis -This folder contains the batch scripts that perform various analyses of the data. They're mostly triggere +This folder contains the batch scripts that perform various analyses of the data. They're mostly triggered from the nightly batch but can also be triggered manually as needed. +## findAllMatches +The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. + +## runDistrib +Called by findAllMatches to execute the distributed processing engine. This file is designed to be run standalone so you can rerun distributed matching. + ## consolidateOutput Collects all single station and match data and consolidates it into two files in parquet format. These are used for all detailed analysis. ## createSearchable Creates a single file for the search engine, by consolidating the required information from the match and single station data. -## findAllMatches -The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. - ## getBadStations Checks for stations that failed quality tests such as too many detections, not uploaded for a few days etc. @@ -24,9 +27,6 @@ Creates a UFO-analyser compatible version of the RMS single-station detections. ## reportActiveShowers Creates a shower report for any active showers, by calling showerReport for each active shower year-to-date. -## runDistrib -Called by findAllMatches to execute the distributed processing engine. - ## showerReport Creates a report for one or more showers. @@ -34,4 +34,4 @@ Creates a report for one or more showers. Creates a report of data for one or all stations, for a month or year to date, which is then pushed to the website. ## Copyright -All code Copyright (C) 2018-2023 Mark McIntyre# \ No newline at end of file +All code Copyright (C) 2018- Mark McIntyre# \ No newline at end of file diff --git a/archive/analysis/consolidateOutput.sh b/archive/analysis/consolidateOutput.sh index 34743434c..51aa0418f 100644 --- a/archive/analysis/consolidateOutput.sh +++ b/archive/analysis/consolidateOutput.sh @@ -46,12 +46,12 @@ do typ=${bn:0:3} if [ "$typ" != "M20" ] ; then pref="P" - yr=${bn:7:4} + snglyr=${bn:7:4} else pref="M" - yr=${bn:1:4} + snglyr=${bn:1:4} fi - mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv + mrgfile=${DATADIR}/consolidated/${pref}_${snglyr}-unified.csv if [ ! -f $mrgfile ] ; then cat $csvf >> $mrgfile else @@ -62,6 +62,10 @@ do fi done +logger -s -t consolidateOutput "purging older raw data which is on S3 anyway" +find ${DATADIR}/single/rawcsvs -mtime +180 -exec rm -f {} \; + + logger -s -t consolidateOutput "pushing consolidated information back" aws s3 sync ${DATADIR}/consolidated ${UKMONSHAREDBUCKET}/consolidated/ --exclude 'UKMON*' --quiet @@ -90,6 +94,12 @@ cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/matched/matches-full-$y cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/searchidx/matches-full-$yr-new.csv mv ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv ${DATADIR}/orbits/${yr}/fullcsv/processed +logger -s -t consolidateOutput "purging older raw data" +find ${DATADIR}/orbits/${yr}/fullcsv/processed -mtime +180 -exec rm -f {} \; +# and last year, because the data is split across dated folders +lyr=$(date -d 'last year' +%Y) +find ${DATADIR}/orbits/${lyr}/fullcsv/processed -mtime +180 -exec rm -f {} \; + python << EOD3 import pandas as pd df = pd.read_csv('${DATADIR}/matched/matches-full-${yr}.csv', skipinitialspace=True) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index db2b35cd6..2fba436f8 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -29,7 +29,8 @@ if [ "$NJLOGSTREAM" == "" ]; then aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared fi log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" findAllMatches -rundate=$(cat $DATADIR/rundate.txt) + +[ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d) # read start/end dates from commandline if rerunning for historical date if [ $# -gt 0 ] ; then @@ -47,15 +48,7 @@ if [ $# -gt 0 ] ; then fi # folder for logs -mkdir -p $SRC/logs > /dev/null 2>&1 - -log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" findAllMatches -# this creates the parquet table for Athena -$SRC/analysis/getRMSSingleData.sh - -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" findAllMatches -yr=$(date +%Y) -$SRC/analysis/createSearchable.sh $yr singles +mkdir -p $SRC/logs/distrib > /dev/null 2>&1 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000') @@ -64,6 +57,9 @@ log2cw $NJLOGGRP $NJLOGSTREAM "solving for ${startdt} to ${enddt}" findAllMatche log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches $SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND +log2cw $NJLOGGRP $NJLOGSTREAM "clean duplicate/deleted trajs" findAllMatches +$SRC/utils/cleanupDeletedTrajs.sh + log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches success=$(grep "Total run time:" $SRC/logs/matchJob.log) @@ -79,28 +75,15 @@ python -m maintenance.rerunFailedLambdas cd $here log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches -python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate processed_trajectories.json - -log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches -dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) -trajlist=$(cat $dailyrep | awk -F, '{print $2}') - matchlog=${SRC}/logs/matchJob.log -vals=$(python -m metrics.getMatchStats $matchlog ) -evts=$(echo $vals | awk '{print $2}') -trajs=$(echo $vals | awk '{print $6}') -matches=$(wc -l $dailyrep | awk '{print $1}') -rtim=$(echo $vals | awk '{print $7}') -echo $(basename $dailyrep) $evts $trajs $matches $rtim >> $DATADIR/dailyreports/stats.txt +python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate +python -m metrics.getMatchStats $matchlog # copy stats to S3 so the daily report can run if [ "$RUNTIME_ENV" == "PROD" ] ; then aws s3 sync $DATADIR/dailyreports/ $UKMONSHAREDBUCKET/matches/RMSCorrelate/dailyreports/ --quiet fi -log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" findAllMatches -$SRC/website/updateIndexPages.sh $dailyrep - log2cw $NJLOGGRP $NJLOGSTREAM "start purgeLogs" findAllMatches find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \; find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \; diff --git a/archive/analysis/getRMSSingleData.sh b/archive/analysis/getRMSSingleData.sh index 4879e4b36..0174a815c 100644 --- a/archive/analysis/getRMSSingleData.sh +++ b/archive/analysis/getRMSSingleData.sh @@ -56,7 +56,10 @@ fi # push to S3 bucket for future use by AWS tools logger -s -t getRMSSingleData "copy to S3 bucket" -aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --quiet +aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --exclude "rawcsvs/*" --exclude "used/*" --quiet aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/singlepq/ --exclude "*" --include "*.parquet.snap" --exclude "*new.parquet.snap" --quiet +logger -s -t getRMSSingleData "purge processed data" +find $outdir/processed -mtime +180 -exec rm -f {} \; + logger -s -t getRMSSingleData "finished" \ No newline at end of file diff --git a/archive/analysis/onlyConsolDistrib.sh b/archive/analysis/onlyConsolDistrib.sh deleted file mode 100644 index b37254253..000000000 --- a/archive/analysis/onlyConsolDistrib.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# just consolidate solutions of distributed candidates -# -# Used if consolidating a bunch of candidates that were solved outside the normal process -# -# consumes: the trajdb and solution json files -# creates: an updated trajdb and hopefully additional orbits -# - -""" -To use this function, first create the candidate pickle files, then copy them to the candidates -folder on the calcserver and submit them to the distributed processing engine with the following: - -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} -export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib -export AWS_PROFILE=ukmonshared -cd /home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates -time python -m traj.distributeCandidates 20230113 /home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates $UKMONSHAREDBUCKET/matches/distrib - -when this completes, logoff the calcserver and shut it down again -On the ukmonhelper run this script to wait for the distrib processing to finish and then -merge in the data - - -""" - - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -logger -s -t onlyConsolDistrib "starting onlyConsolDistrib" - -# load the configuration -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -# set the profile to the EE account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared - - -if [ "$1" == "" ] ; then - rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -else - rundate=$1 -fi -logger -s -t onlyConsolDistrib "consolidating for $rundate" - -python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}'); " - -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) -while [ "$privip" == "" ] ; do - sleep 5 - echo "getting ipaddress" - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) -done - -if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then - aws s3 sync $UKMONSHAREDBUCKET/matches/distrib/ $DATADIR/distrib/ --exclude "*" --include "*.json" --quiet - cp -f $DATADIR/distrib/processed_trajectories.json $DATADIR/distrib/prev_processed_trajectories.json - - numtoconsol=$(ls -1 $DATADIR/distrib/${rundate}*.json | wc -l) - if [ $numtoconsol -gt 5 ] ; then - logger -s -t onlyConsolDistrib "restarting calcserver to consolidate results" - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text --profile ukmonshared) - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - fi - logger -s -t onlyConsolDistrib "waiting for the server to be ready" - while [ "$stat" -ne 16 ]; do - sleep 30 - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - fi - echo "checking - status is ${stat}" - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text --profile ukmonshared) - done - - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - while [ $? -ne 0 ] ; do - # in case the server isn't responding to ssh sessions yet - sleep 10 - echo "server not responding yet, retrying" - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - done - scp -i $SERVERSSHKEY $DATADIR/distrib/${rundate}*.json ec2-user@$privip:data/distrib - - echo "#!/bin/bash" > /tmp/execConsol.sh - echo "export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib" >> /tmp/execConsol.sh - echo "python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json ${rundate}" >> /tmp/execConsol.sh - chmod +x /tmp/execConsol.sh - scp -i $SERVERSSHKEY /tmp/execConsol.sh ec2-user@$privip:data/distrib - ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/execConsol.sh" - - scp -i $SERVERSSHKEY ec2-user@$privip:data/distrib/processed_trajectories.json $DATADIR/distrib - - ssh -i $SERVERSSHKEY ec2-user@$privip "rm -f data/distrib/*.json" - - logger -s -t runDistrib "stopping calcserver again" - aws ec2 stop-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - - python -c "from traj.consolidateDistTraj import patchTrajDB ; patchTrajDB('$DATADIR/distrib/processed_trajectories.json','/home/ec2-user/ukmon-shared/matches/RMSCorrelate', '/home/ec2-user/data/distrib');" - else - python -m traj.consolidateDistTraj $DATADIR/distrib $DATADIR/distrib/processed_trajectories.json $rundate - fi - # push the updated traj db to the S3 bucket - aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - - logger -s -t onlyConsolDistrib "compressing the procssed data" - gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz - aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet - tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib/done/ --quiet - rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 rm $UKMONSHAREDBUCKET/matches/distrib/ --exclude "*" --include "${rundate}*.json" --exclude "test/*" --recursive -else - echo "trajectory database is size zero... not proceeding with copy" -fi -python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate processed_trajectories.json -dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) -$SRC/website/updateIndexPages.sh $dailyrep -logger -s -t onlyConsolDistrib "finished" diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 468f02987..9350e5db3 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -27,7 +27,7 @@ if [ "$NJLOGSTREAM" == "" ]; then fi log2cw $NJLOGGRP $NJLOGSTREAM "starting runDistrib" runDistrib -# set the profile to the EE account so we can run the server and monitor progress +# set the profile to the UKMDA account so we can run the server and monitor progress export AWS_PROFILE=ukmonshared if [ $# -gt 0 ] ; then @@ -46,11 +46,9 @@ begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" runDistrib -stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) -if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID -fi +aws ec2 start-instances --instance-ids $SERVERINSTANCEID +stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) while [ "$stat" -ne 16 ]; do sleep 5 log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" runDistrib @@ -58,12 +56,14 @@ while [ "$stat" -ne 16 ]; do done log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate}" runDistrib + conda activate $HOME/miniconda3/envs/${WMPL_ENV} log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" runDistrib + execdist=execdistrib.sh execMatchingsh=/tmp/$execdist -python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh +python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE chmod +x $execMatchingsh log2cw $NJLOGGRP $NJLOGSTREAM "get server details" runDistrib @@ -76,100 +76,99 @@ done log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" runDistrib -scp -i $SERVERSSHKEY $execMatchingsh ec2-user@$privip:data/distrib/$execdist +scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $execMatchingsh ec2-user@$privip:data/distrib/$execdist + scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist done -# push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner.json ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py ec2-user@$privip:src/ukmon_pylib/traj +# push the python code and ECS templates required +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj # now run the script log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib -ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execdist" +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist" + +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates + +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib aws ec2 stop-instances --instance-ids $SERVERINSTANCEID log2cw $NJLOGGRP $NJLOGSTREAM "monitoring and waiting for completion" runDistrib -python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}'); " +python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}', '${TESTMODE}'); " -log2cw $NJLOGGRP $NJLOGSTREAM "merging in the new json files" runDistrib mkdir -p $DATADIR/distrib cd $DATADIR/distrib -# make sure the database isn't corrupt before overwriting it !! -if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then - aws s3 sync $UKMONSHAREDBUCKET/matches/distrib/ $DATADIR/distrib/ --exclude "*" --include "*.json" --quiet - cp -f $DATADIR/distrib/processed_trajectories.json $DATADIR/distrib/prev_processed_trajectories.json - - log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib +stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +if [ $stat -eq 80 ]; then + aws ec2 start-instances --instance-ids $SERVERINSTANCEID +fi +log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib +while [ "$stat" -ne 16 ]; do + sleep 30 if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi - log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib - while [ "$stat" -ne 16 ]; do - sleep 30 - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID - fi - log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) - done - - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - while [ $? -ne 0 ] ; do - # in case the server isn't responding to ssh sessions yet - sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - done - scp -i $SERVERSSHKEY $DATADIR/distrib/${rundate}*.json ec2-user@$privip:data/distrib - - execcons=execconsol.sh - execConsolsh=/tmp/$execcons - python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh')" - chmod +x $execConsolsh - - log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib - scp -i $SERVERSSHKEY $execConsolsh ec2-user@$privip:data/distrib/$execcons - ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execcons" - - log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation" runDistrib - scp -i $SERVERSSHKEY ec2-user@$privip:data/distrib/processed_trajectories.json $DATADIR/distrib - - ssh -i $SERVERSSHKEY ec2-user@$privip "rm -f data/distrib/*.json /tmp/processed_trajectories.json" - # remote temporary files - ssh -i $SERVERSSHKEY ec2-user@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" - # prune trajdb folder on calcserver - ssh -i $SERVERSSHKEY ec2-user@$privip "find ~/ukmon-shared/matches/RMSCorrelate/trajdb -maxdepth 1 -name "*.json*" -mtime +30 -exec rm -f {} \;" - - log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib - aws ec2 stop-instances --instance-ids $SERVERINSTANCEID - - python -c "from traj.consolidateDistTraj import patchTrajDB ; patchTrajDB('$DATADIR/distrib/processed_trajectories.json','/home/ec2-user/ukmon-shared/matches/RMSCorrelate', '/home/ec2-user/data/distrib');" - - # archive older data then push the updated traj db to the S3 bucket - python -m traj.jsonDbMaintenance $DATADIR/distrib/ - aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - - log2cw $NJLOGGRP $NJLOGSTREAM "compressing the procssed data" runDistrib - gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz - aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet - tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib/done/ --quiet - rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 rm $UKMONSHAREDBUCKET/matches/distrib/ --exclude "*" --include "${rundate}*.json" --exclude "test/*" --recursive -else - log2cw $NJLOGGRP $NJLOGSTREAM "trajectory database is size zero... not proceeding with copy" runDistrib -fi + log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +done +privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) + +execcons=execconsol.sh +execConsolsh=/tmp/$execcons +python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', '$TESTMODE')" +chmod +x $execConsolsh + +log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib +scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$privip:data/distrib/$execcons +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execcons" + +log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation, copying databases" runDistrib +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib + +# remote temporary files +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" + +log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib +aws ec2 stop-instances --instance-ids $SERVERINSTANCEID + +# grab a copy of the indvidual container dbs so we can get a list of new solutions +rm -Rf $DATADIR/latest/contdbs/ +mkdir -p $DATADIR/latest/contdbs/ +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/contdbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --exclude "test/*" --quiet +aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "*${rundate}*.db" --exclude "test/*" --exclude "dbs/*" --recursive --quiet +aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATADIR/distrib --quiet + +# grab a copy of the indvidual container logs - duplicated in monitorProgress, but never mind +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $SRC/logs/distrib/ --exclude "*" --include "correl*.log" --exclude "logs/" --quiet +aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "correl*.log" --exclude "logs/" --recursive --quiet + +log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib + +mkdir -p $DATADIR/trajdb +tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db +mkdir -p $DATADIR/distrib/containers +tar czvf $DATADIR/distrib/containers/contdbs_${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle +aws s3 cp $DATADIR/distrib/containers/contdbs_${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet + +tar czvf $DATADIR/distrib/containers/contlogs_${rundate}.tgz $SRC/logs/distrib/correlator_${rundate}*.log $SRC/logs/distrib/${rundate}_*.log + +find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \; +find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \; +rm -f $DATADIR/distrib/${rundate}.pickle + + # and then clear the profile again unset AWS_PROFILE log2cw $NJLOGGRP $NJLOGSTREAM "finished runDistrib" runDistrib diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index 948846cb8..beed33373 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -59,7 +59,7 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" updatePlotsAndDetStatus execrerun=execreplot.sh execrerunsh=/tmp/$execrerun -python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh')" +python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh', '$TESTMODE')" chmod +x $execrerunsh log2cw $NJLOGGRP $NJLOGSTREAM "get server details" updatePlotsAndDetStatus @@ -72,18 +72,18 @@ done log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" updatePlotsAndDetStatus -scp -i $SERVERSSHKEY $execrerunsh ec2-user@$privip:data/distrib/$execrerun +scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" updatePlotsAndDetStatus - scp -i $SERVERSSHKEY $execrerunsh ec2-user@$privip:data/distrib/$execrerun + scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun done # push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py ec2-user@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj # now run the script -ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execrerun" +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execrerun" log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" updatePlotsAndDetStatus aws ec2 stop-instances --instance-ids $SERVERINSTANCEID diff --git a/archive/containers/trajsolver/Dockerfile b/archive/containers/trajsolver/Dockerfile index ff8b6ff5b..47f4ac3fa 100644 --- a/archive/containers/trajsolver/Dockerfile +++ b/archive/containers/trajsolver/Dockerfile @@ -11,6 +11,8 @@ RUN conda install -y matplotlib RUN pip install boto3 RUN apt-get update && apt-get -y upgrade RUN conda install -y -c conda-forge numba +RUN conda install -y -c conda-forge libsqlite sqlite +RUN conda install -y -c conda-forge paramiko COPY WesternMeteorPyLib/ ./WesternMeteorPyLib ENV PYTHONPATH=./WesternMeteorPyLib diff --git a/archive/containers/trajsolver/build.ps1 b/archive/containers/trajsolver/build.ps1 index 4a4dac048..c5bb2a3f5 100644 --- a/archive/containers/trajsolver/build.ps1 +++ b/archive/containers/trajsolver/build.ps1 @@ -15,7 +15,7 @@ $repo = "calcengine/${imagename}" write-output "building $imagename in $accid" $yn=read-host -prompt "update WMPL?" -if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } +if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } if ($env -eq "test") { move-item awskeys.test awskeys} diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py index 2d42835f7..e9fdd9a9a 100644 --- a/archive/containers/trajsolver/trajsolver.py +++ b/archive/containers/trajsolver/trajsolver.py @@ -6,6 +6,8 @@ import boto3 import datetime import tempfile +import glob +import logging from wmpl.Trajectory.CorrelateRMS import RMSDataHandle from wmpl.Utils.Math import generateDatetimeBins @@ -15,8 +17,10 @@ from wmpl.Trajectory.CorrelateRMS import TrajectoryReduced, DatabaseJSON # noqa: F401 from wmpl.Trajectory.CorrelateRMS import MeteorObsRMS, PlateparDummy, MeteorPointRMS # noqa: F401 +log = logging.getLogger('traj_correlator') # must be the same name as the WMPL logger -def runCorrelator(dir_path, time_beg, time_end): + +def runCorrelator(dir_path, time_beg, time_end, max_stns=9999): # Init trajectory constraints maxtoffset = 10.0 maxstationdist = 600.0 @@ -25,7 +29,7 @@ def runCorrelator(dir_path, time_beg, time_end): saveplots = True velpart = 0.40 uncerttime = False - distribute = 2 + distribute = 3 trajectory_constraints = TrajectoryConstraints() trajectory_constraints.max_toffset = maxtoffset @@ -34,6 +38,7 @@ def runCorrelator(dir_path, time_beg, time_end): trajectory_constraints.run_mc = not disablemc trajectory_constraints.save_plots = saveplots trajectory_constraints.geometric_uncert = not uncerttime + trajectory_constraints.max_stations = max_stns # Clock for measuring script time t1 = datetime.datetime.now(datetime.timezone.utc) @@ -42,8 +47,8 @@ def runCorrelator(dir_path, time_beg, time_end): event_time_range = None # Extract time range - dt_beg = datetime.datetime.strptime(time_beg, "%Y%m%d-%H%M%S") - dt_end = datetime.datetime.strptime(time_end, "%Y%m%d-%H%M%S") + dt_beg = datetime.datetime.strptime(time_beg, "%Y%m%d-%H%M%S").replace(tzinfo=datetime.timezone.utc) + dt_end = datetime.datetime.strptime(time_end, "%Y%m%d-%H%M%S").replace(tzinfo=datetime.timezone.utc) print("Custom time range:") print(" BEG: {:s}".format(str(dt_beg))) @@ -52,17 +57,7 @@ def runCorrelator(dir_path, time_beg, time_end): event_time_range = [dt_beg, dt_end] # Init the data handle - dh = RMSDataHandle(dir_path, event_time_range) - - # If there is nothing to process, stop, unless we're in distributed - # processing mode 2 - if not dh.processing_list and distribute !=2: - print() - print("Nothing to process!") - print("Probably everything is already processed.") - print("Exiting...") - sys.exit() - + dh = RMSDataHandle(dir_path, dt_range=event_time_range, mcmode=distribute) ### GENERATE MONTHLY TIME BINS ### @@ -108,25 +103,26 @@ def runCorrelator(dir_path, time_beg, time_end): print("-----------------------------") print() - # Load data of unprocessed observations - dh.unpaired_observations = dh.loadUnpairedObservations(dh.processing_list, dt_range=(bin_beg, bin_end)) - # Run the trajectory correlator - tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, distribute=distribute, enableOSM=True) - tc.run(event_time_range=event_time_range) + tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, enableOSM=True) + tc.run(event_time_range=event_time_range, mcmode=3) + dh.closeObservationsDatabase() + dh.closeTrajectoryDatabase() + print("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) + log.info("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) return # read the source bucket + folder and target buckets + folders from the environment def getSourceAndTargets(): - srcpth = os.getenv('SRCPATH', default='s3://ukmda-shared/matches/distrib') + srcpth = os.getenv('SRCPATH', default='s3://ukmda-shared/test/matches/distrib') srcpth = srcpth[5:] srcbucket = srcpth[:srcpth.find('/')] srcpth = srcpth[srcpth.find('/')+1:] - outpth = os.getenv('OUTPATH', default='s3://ukmda-shared/matches/distrib') + outpth = os.getenv('OUTPATH', default='s3://ukmda-shared/test/matches/RMSCorrelate') outpth = outpth[5:] outbucket = outpth[:outpth.find('/')] outpth = outpth[outpth.find('/')+1:] @@ -136,7 +132,9 @@ def getSourceAndTargets(): webbucket = webpth[:webpth.find('/')] webpth = webpth[webpth.find('/')+1:] - return srcbucket, srcpth, outbucket, outpth, webbucket, webpth + max_stns = int(os.getenv('MAX_STNS', default='9999')) + + return srcbucket, srcpth, outbucket, outpth, webbucket, webpth, max_stns # extra args for setting the MIME type when uploading to S3. @@ -162,6 +160,8 @@ def getExtraArgs(fname): ctyp = 'application/json' elif file_ext=='.zip': ctyp = 'application/zip' + elif file_ext=='.db': + ctyp = 'application/vnd.sqlite3' extraargs = {'ContentType': ctyp} return extraargs @@ -233,15 +233,37 @@ def getS3Client(): # starting point for the process def startup(srcfldr, startdt, enddt, isTest=False): + + localfldr = tempfile.mkdtemp() + + # Init the logger + log.setLevel(logging.DEBUG) + + # Init the log formatter + log_formatter = logging.Formatter( + fmt='%(asctime)s-%(levelname)-5s-%(module)-15s:%(lineno)-5d- %(message)s', + datefmt='%Y/%m/%d %H:%M:%S') + + # Init the file handler + datasetname = os.path.split(srcfldr)[-1] + log_file = os.path.join(localfldr, f'{datasetname}.log') + file_handler = logging.handlers.TimedRotatingFileHandler(log_file, when="midnight", backupCount=7) + file_handler.setFormatter(log_formatter) + log.addHandler(file_handler) + + # Init the console handler (i.e. print to console) + console_handler = logging.StreamHandler() + console_handler.setFormatter(log_formatter) + log.addHandler(console_handler) + print(f'processing {srcfldr}') print(f"Starting at {datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}") - localfldr = tempfile.mkdtemp() canddir = os.path.join(localfldr,'candidates') os.makedirs(canddir, exist_ok = True) s3 = getS3Client() - srcbucket, srcpth, outbucket, outpth, webbucket, webpth = getSourceAndTargets() + srcbucket, srcpth, outbucket, outpth, webbucket, webpth, max_stns = getSourceAndTargets() if isTest is True: outpth = os.path.join(outpth, 'test') @@ -249,7 +271,7 @@ def startup(srcfldr, startdt, enddt, isTest=False): print(f'fetching data from {srcbucket}/{srckey} saving to {outbucket} and {webbucket}') objlist = s3.meta.client.list_objects_v2(Bucket=srcbucket,Prefix=srckey) - print(objlist) + #print(objlist) if objlist['KeyCount'] > 0: keys = objlist['Contents'] for k in keys: @@ -257,24 +279,45 @@ def startup(srcfldr, startdt, enddt, isTest=False): if '.pickle' in fname: _, locfname = os.path.split(fname) targfile = os.path.join(canddir, locfname) - print(f'downloading {locfname}') + print(f'downloading {locfname} to {targfile}') s3.meta.client.download_file(srcbucket, fname, targfile) - runCorrelator(localfldr, startdt, enddt) + runCorrelator(localfldr, startdt, enddt, max_stns = max_stns) + + # flush and close the logs + file_handler.flush() + console_handler.flush() + logging.shutdown() print('uploading data to website') trajfldr = os.path.join(localfldr,'trajectories') - # reacquire tokens just in case the 1hour time limit on chained roles is exceeded + # reacquire tokens just in case the 1 hour time limit on chained roles is exceeded s3 = getS3Client() pushToWebsite(s3, trajfldr, webbucket, webpth, outbucket, outpth) - fname = f'{srcfldr}.json' - jsonfile = os.path.join(localfldr, 'processed_trajectories.json') - targkey = f'{srcpth}/{fname}' - print(f'uploading {jsonfile} to {srcbucket}/{srcpth}') - s3.meta.client.upload_file(jsonfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + + # databases get pushed back to source location not target + if srcfldr[:5] == 'test/': + srcfldr = srcfldr[5:] + for dbname in ['observations', 'trajectories', 'candidates']: + + fname = f'{dbname}_{srcfldr}.db' + localfile = os.path.join(localfldr, f'{dbname}.db') + if os.path.isfile(localfile): + targkey = f'{srcpth}/{fname}' + print(f'uploading {localfile} to {srcbucket}/{targkey}') + s3.meta.client.upload_file(localfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + + lognames = glob.glob(os.path.join(localfldr, '*.log')) + if len(lognames)> 0: + fname = f'correlator_{srcfldr}.log' + targkey = f'{srcpth}/{fname}' + print(f'uploading {lognames[0]} to {srcbucket}/{targkey}') + s3.meta.client.upload_file(lognames[0], srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + else: print('no files found') + print(f"Finished at {datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}") return diff --git a/archive/containers/trajsolver/update_wmpl.sh b/archive/containers/trajsolver/update_wmpl.sh index 5fa92c4b2..1be9f5dc5 100644 --- a/archive/containers/trajsolver/update_wmpl.sh +++ b/archive/containers/trajsolver/update_wmpl.sh @@ -8,9 +8,9 @@ else fi cd /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive/containers/$targ/ -ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout forcontainer && git pull" +ssh ukmonhelper2 "cd ~/src/wmpldev && git stash && git checkout distrib_processing && git pull" rsync -avz ukmonhelper2:src/wmpldev/* ./WesternMeteorPyLib/ --exclude "build/" --exclude "*.egg*" --exclude "dist/" --exclude ".git/" --exclude "__pycache__/" -ssh ukmonhelper2 "cd src/wmpldev && git checkout - && git stash apply" +ssh ukmonhelper2 "cd ~/src/wmpldev && git checkout - && git stash apply" # remove modules and code that don't work in a container because theres no GUI rm -Rf ./WesternMeteorPyLib/wmpl/MetSim diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index 7f184a5fd..1c9eb1204 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -47,18 +47,40 @@ if [ -f $SRC/logs/$matchlog ] ; then suff=$(stat matchJob.log -c %X) mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff fi + +log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" nightlyJob +# this creates the parquet table for Athena +$SRC/analysis/getRMSSingleData.sh +if [ "$(date +%m%d)" == "0101" ] ; then + # catch any data uploaded on 01/01 that is for 31/12 the previous year + $SRC/analysis/getRMSSingleData.sh $(date -d 'last year' +%Y) +fi + +log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" nightlyJob + +$SRC/analysis/createSearchable.sh $yr singles + # Run the match process - run this only once as it scoops up all unprocessed data ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 -# from here down, we're creating reports +log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" nightlyJob +$SRC/website/updateIndexPages.sh + +# FIRST CONSOLIDATE THE DATA AND CREATE SEARCH INDEXES # consolidate the output of the match process for further analysos log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob $SRC/analysis/consolidateOutput.sh ${yr} +if [ "$(date +%m%d)" == "0101" ] ; then + # catch any data uploaded on 01/01 that is for 31/12 the previous year + $SRC/analysis/consolidateOutput.sh $(date -d 'last year' +%Y) +fi # create the search indexes used on the website log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 2" nightlyJob $SRC/analysis/createSearchable.sh $yr matches +# FROM HERE DOWN WE'RE CREATING REPORTS + # add daily report to the website log2cw $NJLOGGRP $NJLOGSTREAM "start publishDailyReport" nightlyJob $SRC/website/publishDailyReport.sh diff --git a/archive/deployment/analysis.yml b/archive/deployment/analysis.yml index 8e217f41d..f65068693 100644 --- a/archive/deployment/analysis.yml +++ b/archive/deployment/analysis.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -25,7 +25,6 @@ - {src: '{{srcdir}}/analysis/getBadStations.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/getLogData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/getRMSSingleData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/onlyConsolDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/reportActiveShowers.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/runDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/showerReport.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } diff --git a/archive/deployment/config.yml b/archive/deployment/config.yml index b9823e58a..88ad1408a 100644 --- a/archive/deployment/config.yml +++ b/archive/deployment/config.yml @@ -2,7 +2,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: dev-vars.yml diff --git a/archive/deployment/database.yml b/archive/deployment/database.yml index 1b75b1f5f..3abf9d787 100644 --- a/archive/deployment/database.yml +++ b/archive/deployment/database.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/deploy-usermgmt.yml b/archive/deployment/deploy-usermgmt.yml deleted file mode 100644 index 8429780d3..000000000 --- a/archive/deployment/deploy-usermgmt.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive/usermgmt" - destdir: /home/ec2-user/keymgmt - tasks: - - name: Ensures {{destdir}} exists - file: path={{destdir}} state=directory - - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - with_items: - - {src: '{{srcdir}}/addSftpUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/updateAwsKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/addAdminUser.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } - - {src: '{{srcdir}}/addFBApiKey.sh', dest: '{{destdir}}/', mode: '744', backup: yes, directory_mode: no } diff --git a/archive/deployment/dev-vars.yml b/archive/deployment/dev-vars.yml index fb60fbb1d..86dfcf575 100644 --- a/archive/deployment/dev-vars.yml +++ b/archive/deployment/dev-vars.yml @@ -1,6 +1,3 @@ destdir: /home/ec2-user/dev env: DEV -websitebucket: mjmm-ukmonarchive.co.uk -#api_id: 0zbnc358p0 -#api_svc: test s3profile: default diff --git a/archive/deployment/prod-vars.yml b/archive/deployment/prod-vars.yml index d26be9990..20171b912 100644 --- a/archive/deployment/prod-vars.yml +++ b/archive/deployment/prod-vars.yml @@ -1,6 +1,3 @@ destdir: /home/ec2-user/prod env: PROD -websitebucket: ukmda-website -#api_id: 40luvfh1od -#api_svc: Prod s3profile: ukmda_admin \ No newline at end of file diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml index 46a4afc71..313173390 100644 --- a/archive/deployment/pylib.yml +++ b/archive/deployment/pylib.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -97,6 +97,7 @@ - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mm.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } + - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetailstest-ukmda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/consolidateDistTraj.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/createDistribMatchingSh.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } @@ -107,4 +108,5 @@ - {src: '{{srcdir}}/ukmon_pylib/traj/plotOSMGroundTrack.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/ShowerAssociation.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } + - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner_test.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/README.md', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } diff --git a/archive/deployment/server_setup.yml b/archive/deployment/server_setup.yml index f621dc278..f53ea3a7b 100644 --- a/archive/deployment/server_setup.yml +++ b/archive/deployment/server_setup.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -24,10 +24,10 @@ - {src: '{{srcdir}}/server_setup/install-proj4.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - {src: '{{srcdir}}/server_setup/install-sqlite3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - {src: '{{srcdir}}/server_setup/install_opencv.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - {src: '{{srcdir}}/server_setup/migratingBatchServer.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } - {src: '{{srcdir}}/server_setup/newserver.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - {src: '{{srcdir}}/server_setup/remount-s3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } diff --git a/archive/deployment/shwrinfo.yml b/archive/deployment/shwrinfo.yml index 17a6110ee..c7bbfd11f 100644 --- a/archive/deployment/shwrinfo.yml +++ b/archive/deployment/shwrinfo.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/utils.yml b/archive/deployment/utils.yml index f122d7cb5..73dd5f02f 100644 --- a/archive/deployment/utils.yml +++ b/archive/deployment/utils.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml index 1ee42364e..78956ddaa 100644 --- a/archive/deployment/website.yml +++ b/archive/deployment/website.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -37,6 +37,5 @@ - {src: '{{srcdir}}/website/templates/frontpage.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/header.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/reportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/searchdialog.js', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/shwrcsvindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/statreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } diff --git a/archive/deployment/website_static.yml b/archive/deployment/website_static.yml index cbe146679..c15044f48 100644 --- a/archive/deployment/website_static.yml +++ b/archive/deployment/website_static.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -27,7 +27,6 @@ - {src: '{{srcdir}}/static_content/js/', dest: '{{destdir}}/static_content/js', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/latest/', dest: '{{destdir}}/static_content/latest', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/live/', dest: '{{destdir}}/static_content/live', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/reports/', dest: '{{destdir}}/static_content/reports', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/search/', dest: '{{destdir}}/static_content/search', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/templates/', dest: '{{destdir}}/static_content/templates', mode: '754', backup: no, directory_mode: yes } diff --git a/archive/samfunctions/README.md b/archive/lambdas/README.md similarity index 100% rename from archive/samfunctions/README.md rename to archive/lambdas/README.md diff --git a/archive/samfunctions/camDetails/camDetails.py b/archive/lambdas/camDetails/camDetails.py similarity index 100% rename from archive/samfunctions/camDetails/camDetails.py rename to archive/lambdas/camDetails/camDetails.py diff --git a/archive/samfunctions/camDetails/samconfig.toml b/archive/lambdas/camDetails/samconfig.toml similarity index 100% rename from archive/samfunctions/camDetails/samconfig.toml rename to archive/lambdas/camDetails/samconfig.toml diff --git a/archive/samfunctions/camDetails/template.yml b/archive/lambdas/camDetails/template.yml similarity index 100% rename from archive/samfunctions/camDetails/template.yml rename to archive/lambdas/camDetails/template.yml diff --git a/archive/samfunctions/fetchECSV/Math.py b/archive/lambdas/fetchECSV/Math.py similarity index 100% rename from archive/samfunctions/fetchECSV/Math.py rename to archive/lambdas/fetchECSV/Math.py diff --git a/archive/samfunctions/fetchECSV/MeteorObservation.py b/archive/lambdas/fetchECSV/MeteorObservation.py similarity index 100% rename from archive/samfunctions/fetchECSV/MeteorObservation.py rename to archive/lambdas/fetchECSV/MeteorObservation.py diff --git a/archive/samfunctions/fetchECSV/fetchECSV.py b/archive/lambdas/fetchECSV/fetchECSV.py similarity index 100% rename from archive/samfunctions/fetchECSV/fetchECSV.py rename to archive/lambdas/fetchECSV/fetchECSV.py diff --git a/archive/samfunctions/fetchECSV/ftpDetectInfo.py b/archive/lambdas/fetchECSV/ftpDetectInfo.py similarity index 100% rename from archive/samfunctions/fetchECSV/ftpDetectInfo.py rename to archive/lambdas/fetchECSV/ftpDetectInfo.py diff --git a/archive/samfunctions/fetchECSV/requirements.txt b/archive/lambdas/fetchECSV/requirements.txt similarity index 100% rename from archive/samfunctions/fetchECSV/requirements.txt rename to archive/lambdas/fetchECSV/requirements.txt diff --git a/archive/samfunctions/fetchECSV/samconfig.toml b/archive/lambdas/fetchECSV/samconfig.toml similarity index 100% rename from archive/samfunctions/fetchECSV/samconfig.toml rename to archive/lambdas/fetchECSV/samconfig.toml diff --git a/archive/samfunctions/fetchECSV/template.yml b/archive/lambdas/fetchECSV/template.yml similarity index 100% rename from archive/samfunctions/fetchECSV/template.yml rename to archive/lambdas/fetchECSV/template.yml diff --git a/archive/samfunctions/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt b/archive/lambdas/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt similarity index 100% rename from archive/samfunctions/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt rename to archive/lambdas/fetchECSV/tests/FTPdetectinfo_UK005U_20240809_201258_058797.txt diff --git a/archive/samfunctions/fetchECSV/tests/platepars_all_recalibrated.json b/archive/lambdas/fetchECSV/tests/platepars_all_recalibrated.json similarity index 100% rename from archive/samfunctions/fetchECSV/tests/platepars_all_recalibrated.json rename to archive/lambdas/fetchECSV/tests/platepars_all_recalibrated.json diff --git a/archive/samfunctions/fireballApi/getFireballFiles.py b/archive/lambdas/fireballApi/getFireballFiles.py similarity index 100% rename from archive/samfunctions/fireballApi/getFireballFiles.py rename to archive/lambdas/fireballApi/getFireballFiles.py diff --git a/archive/samfunctions/fireballApi/requirements.txt b/archive/lambdas/fireballApi/requirements.txt similarity index 100% rename from archive/samfunctions/fireballApi/requirements.txt rename to archive/lambdas/fireballApi/requirements.txt diff --git a/archive/samfunctions/fireballApi/samconfig.toml b/archive/lambdas/fireballApi/samconfig.toml similarity index 100% rename from archive/samfunctions/fireballApi/samconfig.toml rename to archive/lambdas/fireballApi/samconfig.toml diff --git a/archive/samfunctions/fireballApi/template.yml b/archive/lambdas/fireballApi/template.yml similarity index 100% rename from archive/samfunctions/fireballApi/template.yml rename to archive/lambdas/fireballApi/template.yml diff --git a/archive/samfunctions/ftpToUkmon/.gitignore b/archive/lambdas/ftpToUkmon/.gitignore similarity index 100% rename from archive/samfunctions/ftpToUkmon/.gitignore rename to archive/lambdas/ftpToUkmon/.gitignore diff --git a/archive/samfunctions/ftpToUkmon/README.md b/archive/lambdas/ftpToUkmon/README.md similarity index 100% rename from archive/samfunctions/ftpToUkmon/README.md rename to archive/lambdas/ftpToUkmon/README.md diff --git a/archive/samfunctions/ftpToUkmon/__init__.py b/archive/lambdas/ftpToUkmon/__init__.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/__init__.py rename to archive/lambdas/ftpToUkmon/__init__.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/Dockerfile b/archive/lambdas/ftpToUkmon/pythoncode/Dockerfile similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/Dockerfile rename to archive/lambdas/ftpToUkmon/pythoncode/Dockerfile diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/ShowerAssociation.py b/archive/lambdas/ftpToUkmon/pythoncode/ShowerAssociation.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/ShowerAssociation.py rename to archive/lambdas/ftpToUkmon/pythoncode/ShowerAssociation.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/Showers.py b/archive/lambdas/ftpToUkmon/pythoncode/Showers.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/Showers.py rename to archive/lambdas/ftpToUkmon/pythoncode/Showers.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/__init__.py b/archive/lambdas/ftpToUkmon/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/__init__.py rename to archive/lambdas/ftpToUkmon/pythoncode/__init__.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/established_showers.csv b/archive/lambdas/ftpToUkmon/pythoncode/established_showers.csv similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/established_showers.csv rename to archive/lambdas/ftpToUkmon/pythoncode/established_showers.csv diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/ftpToUkmon1.py b/archive/lambdas/ftpToUkmon/pythoncode/ftpToUkmon1.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/ftpToUkmon1.py rename to archive/lambdas/ftpToUkmon/pythoncode/ftpToUkmon1.py diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/requirements.txt b/archive/lambdas/ftpToUkmon/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/requirements.txt rename to archive/lambdas/ftpToUkmon/pythoncode/requirements.txt diff --git a/archive/samfunctions/ftpToUkmon/pythoncode/supportFuncs.py b/archive/lambdas/ftpToUkmon/pythoncode/supportFuncs.py similarity index 100% rename from archive/samfunctions/ftpToUkmon/pythoncode/supportFuncs.py rename to archive/lambdas/ftpToUkmon/pythoncode/supportFuncs.py diff --git a/archive/samfunctions/ftpToUkmon/samconfig.toml b/archive/lambdas/ftpToUkmon/samconfig.toml similarity index 100% rename from archive/samfunctions/ftpToUkmon/samconfig.toml rename to archive/lambdas/ftpToUkmon/samconfig.toml diff --git a/archive/samfunctions/ftpToUkmon/template.yaml b/archive/lambdas/ftpToUkmon/template.yaml similarity index 100% rename from archive/samfunctions/ftpToUkmon/template.yaml rename to archive/lambdas/ftpToUkmon/template.yaml diff --git a/archive/samfunctions/ftpToUkmon/tests/expected_results.txt b/archive/lambdas/ftpToUkmon/tests/expected_results.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/expected_results.txt rename to archive/lambdas/ftpToUkmon/tests/expected_results.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/localTest.ps1 b/archive/lambdas/ftpToUkmon/tests/localTest.ps1 similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/localTest.ps1 rename to archive/lambdas/ftpToUkmon/tests/localTest.ps1 diff --git a/archive/samfunctions/ftpToUkmon/tests/new_data.txt b/archive/lambdas/ftpToUkmon/tests/new_data.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/new_data.txt rename to archive/lambdas/ftpToUkmon/tests/new_data.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/remoteResult.txt b/archive/lambdas/ftpToUkmon/tests/remoteResult.txt similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/remoteResult.txt rename to archive/lambdas/ftpToUkmon/tests/remoteResult.txt diff --git a/archive/samfunctions/ftpToUkmon/tests/remoteTest.ps1 b/archive/lambdas/ftpToUkmon/tests/remoteTest.ps1 similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/remoteTest.ps1 rename to archive/lambdas/ftpToUkmon/tests/remoteTest.ps1 diff --git a/archive/samfunctions/ftpToUkmon/tests/testEvent.json b/archive/lambdas/ftpToUkmon/tests/testEvent.json similarity index 100% rename from archive/samfunctions/ftpToUkmon/tests/testEvent.json rename to archive/lambdas/ftpToUkmon/tests/testEvent.json diff --git a/archive/samfunctions/getApiDetails.ps1 b/archive/lambdas/getApiDetails.ps1 similarity index 100% rename from archive/samfunctions/getApiDetails.ps1 rename to archive/lambdas/getApiDetails.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/.gitignore b/archive/lambdas/getExtraFilesV2/.gitignore similarity index 100% rename from archive/samfunctions/getExtraFilesV2/.gitignore rename to archive/lambdas/getExtraFilesV2/.gitignore diff --git a/archive/samfunctions/getExtraFilesV2/README.md b/archive/lambdas/getExtraFilesV2/README.md similarity index 100% rename from archive/samfunctions/getExtraFilesV2/README.md rename to archive/lambdas/getExtraFilesV2/README.md diff --git a/archive/samfunctions/getExtraFilesV2/__init__.py b/archive/lambdas/getExtraFilesV2/__init__.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/__init__.py rename to archive/lambdas/getExtraFilesV2/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile similarity index 87% rename from archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile rename to archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile index 26f9034fa..03a07a133 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile +++ b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile @@ -6,10 +6,11 @@ COPY requirements.txt ./ RUN python -m pip install -r requirements.txt -t . RUN python -m pip install numba watchdog basemap-data-hires -t . RUN python -m pip install cartopy -t . -#RUN yum update -y +RUN python -m pip install paramiko -t . + ENV LD_LIBRARY_PATH=./ COPY WesternMeteorPyLib/ ./WesternMeteorPyLib -#COPY *.npy ./WesternMeteorPyLib/wmpl/share/ + COPY *.py ./ ENV PYTHONPATH=./WesternMeteorPyLib ENV PROJ_LIB=./ diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/__init__.py b/archive/lambdas/getExtraFilesV2/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/__init__.py rename to archive/lambdas/getExtraFilesV2/pythoncode/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py b/archive/lambdas/getExtraFilesV2/pythoncode/createOrbitPageIndex.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py rename to archive/lambdas/getExtraFilesV2/pythoncode/createOrbitPageIndex.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py similarity index 85% rename from archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py rename to archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py index 8c71bd7b3..83fb956cc 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py +++ b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py @@ -36,10 +36,15 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): yr = orbname[:4] ym = orbname[:6] ymd= orbname[:8] + istest = False if int(yr) > 2020: webpth = f'reports/{yr}/orbits/{ym}/{ymd}/{orbname}/' else: webpth = f'reports/{yr}/orbits/{ym}/{orbname}/' + + if 'test' in key: + webpth = f'dummy/{yr}/orbits/{ym}/{ymd}/{orbname}/' + istest = True outdir = os.path.join(tmpdir, orbname) try: @@ -162,7 +167,7 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): pass print('pushing files back') - pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3) + pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3, istest=istest) print('updating the index page') createOrbitPageIndex(outdir, websitebucket, s3) @@ -241,7 +246,7 @@ def findOtherFiles(evtdt, archbucket, websitebucket, outdir, fldr, s3): return -def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): +def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3, istest=False): flist = os.listdir(outdir) _, pth =os.path.split(outdir) @@ -253,26 +258,32 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): else: webpth = f'reports/{yr}/orbits/{ym}/{pth}/' + if istest: + webpth = f'dummy/{yr}/orbits/{ym}/{ymd}/{pth}/' + + for f in flist: print(f) locfname = os.path.join(outdir, f) + extraargs = getExtraArgs(locfname) # some files need to be pushed to the website, some to the archive bucket if '3dtrack' in f or '2dtrack' in f: key = os.path.join(webpth, f) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, websitebucket, key, ExtraArgs=extraargs) elif '.lst' in f: - key = os.path.join(f'matches/RMSCorrelate/trajectories/{yr}/{ym}/{ymd}/{pth}', f) - extraargs = getExtraArgs(locfname) + if istest: + key = os.path.join(f'matches/distrib/test/trajectories/{yr}/{ym}/{ymd}/{pth}', f) + else: + key = os.path.join(f'matches/RMSCorrelate/trajectories/{yr}/{ym}/{ymd}/{pth}', f) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'summary' in f: key = os.path.join(fldr, f) - # print(locfname, key) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'orbit_full.csv' in f: - key = os.path.join(f'matches/{yr}/fullcsv', f) - extraargs = getExtraArgs(locfname) + if istest: + key = os.path.join(f'matches/distrib/test/{yr}/fullcsv', f) + else: + key = os.path.join(f'matches/{yr}/fullcsv', f) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) return @@ -345,38 +356,8 @@ def getExtraArgs(fname): def lambda_handler(event, context): - # NB: IN THE CONSOLE, ADD TRUST RELATIONSHIP - # "AWS": "arn:aws:iam::317976261112:root" - # on the target role (replace by this lambda's ARN). - - # Note: if the lambda is connected to a VPC you must - # create a private subnet for the lambda - # create a NAT gateway on a public subnet - # create a routing table on the VPC to send 0.0.0.0/0 to the NAT gw - # attach the new subnet to the NAT gw - # - - sts_client = boto3.client('sts') - response = sts_client.get_caller_identity()['Account'] - if response == '317976261112': - assumed_role_object=sts_client.assume_role( - RoleArn="arn:aws:iam::183798037734:role/service-role/S3FullAccess", - RoleSessionName="GetExtraFilesV2") - - credentials=assumed_role_object['Credentials'] - - # Use the temporary credentials that AssumeRole returns to connections - s3 = boto3.resource('s3', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) - ddb = boto3.resource('dynamodb', region_name='eu-west-2', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) - else: - s3 = boto3.resource('s3') - ddb = boto3.resource('dynamodb', region_name='eu-west-2') + s3 = boto3.resource('s3') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') websitebucket = os.getenv('WEBSITEBUCKET') if websitebucket[:3] == 's3:': diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/pickleAnalysis.py b/archive/lambdas/getExtraFilesV2/pythoncode/pickleAnalysis.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/pickleAnalysis.py rename to archive/lambdas/getExtraFilesV2/pythoncode/pickleAnalysis.py diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/requirements.txt b/archive/lambdas/getExtraFilesV2/pythoncode/requirements.txt similarity index 100% rename from archive/samfunctions/getExtraFilesV2/pythoncode/requirements.txt rename to archive/lambdas/getExtraFilesV2/pythoncode/requirements.txt diff --git a/archive/samfunctions/getExtraFilesV2/samconfig.toml b/archive/lambdas/getExtraFilesV2/samconfig.toml similarity index 100% rename from archive/samfunctions/getExtraFilesV2/samconfig.toml rename to archive/lambdas/getExtraFilesV2/samconfig.toml diff --git a/archive/samfunctions/getExtraFilesV2/template.yaml b/archive/lambdas/getExtraFilesV2/template.yaml similarity index 100% rename from archive/samfunctions/getExtraFilesV2/template.yaml rename to archive/lambdas/getExtraFilesV2/template.yaml diff --git a/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/localTest.ps1 similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 rename to archive/lambdas/getExtraFilesV2/tests/localTest.ps1 diff --git a/archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 similarity index 97% rename from archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 rename to archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 index 0a8098317..fa8c832dd 100644 --- a/archive/samfunctions/getExtraFilesV2/tests/remoteTest.ps1 +++ b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 @@ -15,4 +15,3 @@ else { aws lambda invoke --profile ukmonshared --function-name getExtraOrbitFilesV2 --log-type Tail --cli-binary-format raw-in-base64-out --payload file://tests/dummy.json --region eu-west-2 ./ftpdetect.log Remove-Item tests\dummy.json } - diff --git a/archive/samfunctions/getExtraFilesV2/tests/templateEvent.json b/archive/lambdas/getExtraFilesV2/tests/templateEvent.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/templateEvent.json rename to archive/lambdas/getExtraFilesV2/tests/templateEvent.json diff --git a/archive/samfunctions/getExtraFilesV2/tests/testEvent.json b/archive/lambdas/getExtraFilesV2/tests/testEvent.json similarity index 100% rename from archive/samfunctions/getExtraFilesV2/tests/testEvent.json rename to archive/lambdas/getExtraFilesV2/tests/testEvent.json diff --git a/archive/samfunctions/getExtraFilesV2/tests/testEvent2.json b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json similarity index 60% rename from archive/samfunctions/getExtraFilesV2/tests/testEvent2.json rename to archive/lambdas/getExtraFilesV2/tests/testEvent2.json index fd940082b..9aa49b7f2 100644 --- a/archive/samfunctions/getExtraFilesV2/tests/testEvent2.json +++ b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json @@ -7,7 +7,7 @@ "arn": "arn:aws:s3:::ukmda-shared" }, "object": { - "key": "matches/RMSCorrelate/trajectories/2024/202407/20240714/20240714_000041.319_UK/20240714_000041_trajectory.pickle" + "key": "matches/distrib/test/trajectories/2026/202604/20260412/20260412_232324.162_UK/20260412_232324_trajectory.pickle" } } } diff --git a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh b/archive/lambdas/getExtraFilesV2/update_wmpl.sh similarity index 68% rename from archive/samfunctions/getExtraFilesV2/update_wmpl.sh rename to archive/lambdas/getExtraFilesV2/update_wmpl.sh index 57d3fb6fa..002e1f93d 100644 --- a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh +++ b/archive/lambdas/getExtraFilesV2/update_wmpl.sh @@ -4,14 +4,14 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" cd $here/pythoncode -ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout forcontainer && git pull" +ssh ukmonhelper2 "cd ~/src/wmpldev && git stash && git checkout distrib_processing && git pull" rsync -avz ukmonhelper2:src/wmpldev/* ./WesternMeteorPyLib/ --exclude "build/" --exclude "*.egg*" --exclude "dist/" --exclude ".git/" --exclude "__pycache__/" ssh ukmonhelper2 "cd src/wmpldev && git checkout - && git stash apply" -mv -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so.0 ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so +rm -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so +cp -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so.0 ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so # remove modules and code that don't work in a container because theres no GUI rm -Rf ./WesternMeteorPyLib/wmpl/MetSim rm -Rf ./WesternMeteorPyLib/wmpl/CAMO rm -f ./WesternMeteorPyLib/wmpl/Utils/DynamicMassFit.py -rm -f ./WesternMeteorPyLib/wmpl/Utils/PlotMap_OSM.py -cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py +#cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py b/archive/lambdas/getExtraFilesV2/wmpl__init__.py_fixed.py similarity index 100% rename from archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py rename to archive/lambdas/getExtraFilesV2/wmpl__init__.py_fixed.py diff --git a/archive/samfunctions/getLiveImages/getLiveImages.py b/archive/lambdas/getLiveImages/getLiveImages.py similarity index 90% rename from archive/samfunctions/getLiveImages/getLiveImages.py rename to archive/lambdas/getLiveImages/getLiveImages.py index 164ecc605..5d589131d 100644 --- a/archive/samfunctions/getLiveImages/getLiveImages.py +++ b/archive/lambdas/getLiveImages/getLiveImages.py @@ -16,7 +16,10 @@ def getLiveImages(dtstr, ddb=None): resp = table.query(IndexName='month-image_name-index', KeyConditionExpression=Key('month').eq(dtstr[4:6]) & Key('image_name').begins_with(f'M{dtstr}'), ProjectionExpression='image_name') - return resp + resp2 = table.query(IndexName='month-image_name-index', + KeyConditionExpression=Key('month').eq(dtstr[4:6]) & Key('image_name').begins_with(f'FF_{dtstr}'), + ProjectionExpression='image_name') + return resp['Items'] + resp2['Items'] def filterImages(d1, d2, statid=None, maxitems=-1, ddb=None): @@ -103,10 +106,10 @@ def lambda_handler(event, context): patt = qs['pattern'] print(f'searching for {patt}') ecsvstr = getLiveImages(patt, ddb=ddb) - print(f"found {ecsvstr['Items']}") + print(f"found {len(ecsvstr)} items") return { 'statusCode': 200, - 'body': json.dumps(ecsvstr['Items']) + 'body': json.dumps(ecsvstr) } else: if 'dtstr' in qs: @@ -119,7 +122,7 @@ def lambda_handler(event, context): if 'enddtstr' in qs: dtstr2 = qs['enddtstr'] else: - dtstr2 = datetime.datetime.now().strftime('M%Y%m%d_%H') + dtstr2 = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.000Z') statid = None if 'statid' in qs: statid = qs['statid'] diff --git a/archive/samfunctions/getLiveImages/requirements.txt b/archive/lambdas/getLiveImages/requirements.txt similarity index 100% rename from archive/samfunctions/getLiveImages/requirements.txt rename to archive/lambdas/getLiveImages/requirements.txt diff --git a/archive/samfunctions/getLiveImages/samconfig.toml b/archive/lambdas/getLiveImages/samconfig.toml similarity index 100% rename from archive/samfunctions/getLiveImages/samconfig.toml rename to archive/lambdas/getLiveImages/samconfig.toml diff --git a/archive/samfunctions/getLiveImages/template.yml b/archive/lambdas/getLiveImages/template.yml similarity index 100% rename from archive/samfunctions/getLiveImages/template.yml rename to archive/lambdas/getLiveImages/template.yml diff --git a/archive/samfunctions/matchDataApi/localTest.ps1 b/archive/lambdas/matchDataApi/localTest.ps1 similarity index 100% rename from archive/samfunctions/matchDataApi/localTest.ps1 rename to archive/lambdas/matchDataApi/localTest.ps1 diff --git a/archive/samfunctions/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py similarity index 83% rename from archive/samfunctions/matchDataApi/matchDataApi.py rename to archive/lambdas/matchDataApi/matchDataApi.py index b7c2023ef..3000292de 100644 --- a/archive/samfunctions/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -9,6 +9,33 @@ import pymysql.cursors +def fileToJsonString(flis): + hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', + 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', + 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] + ptsarray='[' + gotpts = False + for fli in flis: + if 'Points' in fli: + gotpts = True + continue + elif '------' in fli or ' No' in fli: + continue + elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): + gotpts=False + break + elif gotpts is True: + spls = fli.split(',') + thisrow = '{' + for h, s in zip(hdr, spls): + thisrow = thisrow + f'"{h}": "{s.strip()}",' + thisrow = thisrow[:-1] + '},' + ptsarray = ptsarray + thisrow + ptsarray = ptsarray[:-1] + ']' + ptsarray = '{' + f'"points": {ptsarray}' + '}' + return ptsarray + + def getSqlLoginDetails(): # retrieve password and host from SSM. This allows me to manage them from Terraform ssm = boto3.client('ssm', region_name='eu-west-1') @@ -22,15 +49,25 @@ def getSqlLoginDetails(): return host, user, password, db -def getStationData(statid, dtstr): +def periodToSqlFragment(period): + if period == 'am': + frag = "and s._h_ut < 12" + elif period == 'pm': + frag = "and s._h_ut >= 12" + elif '-' in period: + sh,eh = period.split('-') + frag = f"and s._h_ut >={sh} and s._h_ut < {eh}" + return frag + + +def getStationData(statid, dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() - connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) try: + statfrag = f"and s.stations like '%{statid}%' " if statid is not None else "" + perfrag = periodToSqlFragment(period) if period is not None else "" with connection.cursor() as cursor: - if statid is None: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%'" - else: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' and s.stations like '%{statid}%'" + sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}" cursor.execute(sql) result = cursor.fetchall() finally: @@ -41,13 +78,14 @@ def getStationData(statid, dtstr): return res -def getSummaryData(dtstr): +def getSummaryData(dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() + perfrag = periodToSqlFragment(period) if period is not None else "" connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' - expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%'" + expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%' {perfrag}" result=[] try: with connection.cursor() as cursor: @@ -88,12 +126,10 @@ def lambda_handler(event, context): if qs is None: return { 'statusCode': 200, - 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1]' + 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1][&period=am|pm|h-h]' } reqtyp = qs['reqtyp'] - points = False - if 'points' in qs: - points = True + points = True if 'points' in qs else False if reqtyp == 'station': statid = qs['statid'] @@ -102,12 +138,14 @@ def lambda_handler(event, context): res = getStationData(statid, dtstr) elif reqtyp == 'summary': dtstr = qs['reqval'] + period = qs['period'] if 'period' in qs else None print(f'summary data requested for {dtstr}') - res = getSummaryData(dtstr) + res = getSummaryData(dtstr, period) elif reqtyp == 'matches': dtstr = qs['reqval'] + period = qs['period'] if 'period' in qs else None print(f'match data requested for {dtstr}') - res = getStationData(None, dtstr) + res = getStationData(None, dtstr, period) elif reqtyp == 'detail': orbname = qs['reqval'] print(f'detail requested for {orbname}') @@ -140,30 +178,3 @@ def lambda_handler(event, context): 'statusCode': 200, 'body': res } - - -def fileToJsonString(flis): - hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', - 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', - 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] - ptsarray='[' - gotpts = False - for fli in flis: - if 'Points' in fli: - gotpts = True - continue - elif '------' in fli or ' No' in fli: - continue - elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): - gotpts=False - break - elif gotpts is True: - spls = fli.split(',') - thisrow = '{' - for h, s in zip(hdr, spls): - thisrow = thisrow + f'"{h}": "{s.strip()}",' - thisrow = thisrow[:-1] + '},' - ptsarray = ptsarray + thisrow - ptsarray = ptsarray[:-1] + ']' - ptsarray = '{' + f'"points": {ptsarray}' + '}' - return ptsarray diff --git a/archive/samfunctions/matchDataApi/requirements.txt b/archive/lambdas/matchDataApi/requirements.txt similarity index 100% rename from archive/samfunctions/matchDataApi/requirements.txt rename to archive/lambdas/matchDataApi/requirements.txt diff --git a/archive/samfunctions/matchDataApi/samconfig.toml b/archive/lambdas/matchDataApi/samconfig.toml similarity index 100% rename from archive/samfunctions/matchDataApi/samconfig.toml rename to archive/lambdas/matchDataApi/samconfig.toml diff --git a/archive/samfunctions/matchDataApi/template.yml b/archive/lambdas/matchDataApi/template.yml similarity index 93% rename from archive/samfunctions/matchDataApi/template.yml rename to archive/lambdas/matchDataApi/template.yml index d07529b3a..83f98f961 100644 --- a/archive/samfunctions/matchDataApi/template.yml +++ b/archive/lambdas/matchDataApi/template.yml @@ -31,6 +31,8 @@ Resources: Required: false - method.request.querystring.statid: Required: false + - method.request.querystring.period: + Required: false Tags: billingtag: "ukmda" typetag: "api" diff --git a/archive/samfunctions/matchDataApi/testDetail.json b/archive/lambdas/matchDataApi/testDetail.json similarity index 100% rename from archive/samfunctions/matchDataApi/testDetail.json rename to archive/lambdas/matchDataApi/testDetail.json diff --git a/archive/samfunctions/matchDataApi/testMatches.json b/archive/lambdas/matchDataApi/testMatches.json similarity index 100% rename from archive/samfunctions/matchDataApi/testMatches.json rename to archive/lambdas/matchDataApi/testMatches.json diff --git a/archive/lambdas/matchDataApi/testMatchesPeriod.json b/archive/lambdas/matchDataApi/testMatchesPeriod.json new file mode 100644 index 000000000..6f61467d2 --- /dev/null +++ b/archive/lambdas/matchDataApi/testMatchesPeriod.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "matches", + "reqval": "20251213", + "period": "0-3" + } +} \ No newline at end of file diff --git a/archive/samfunctions/matchDataApi/testPoints.json b/archive/lambdas/matchDataApi/testPoints.json similarity index 100% rename from archive/samfunctions/matchDataApi/testPoints.json rename to archive/lambdas/matchDataApi/testPoints.json diff --git a/archive/samfunctions/matchDataApi/testStation.json b/archive/lambdas/matchDataApi/testStation.json similarity index 100% rename from archive/samfunctions/matchDataApi/testStation.json rename to archive/lambdas/matchDataApi/testStation.json diff --git a/archive/lambdas/matchDataApi/testSumPeriod.json b/archive/lambdas/matchDataApi/testSumPeriod.json new file mode 100644 index 000000000..81c4b4194 --- /dev/null +++ b/archive/lambdas/matchDataApi/testSumPeriod.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "summary", + "reqval": "20251213", + "period": "0-3" + } +} \ No newline at end of file diff --git a/archive/samfunctions/matchDataApi/testSummary.json b/archive/lambdas/matchDataApi/testSummary.json similarity index 100% rename from archive/samfunctions/matchDataApi/testSummary.json rename to archive/lambdas/matchDataApi/testSummary.json diff --git a/archive/samfunctions/matchPickle/__init__.py b/archive/lambdas/matchPickle/__init__.py similarity index 100% rename from archive/samfunctions/matchPickle/__init__.py rename to archive/lambdas/matchPickle/__init__.py diff --git a/archive/samfunctions/matchPickle/localTest.ps1 b/archive/lambdas/matchPickle/localTest.ps1 similarity index 100% rename from archive/samfunctions/matchPickle/localTest.ps1 rename to archive/lambdas/matchPickle/localTest.ps1 diff --git a/archive/samfunctions/matchPickle/pythoncode/Dockerfile b/archive/lambdas/matchPickle/pythoncode/Dockerfile similarity index 75% rename from archive/samfunctions/matchPickle/pythoncode/Dockerfile rename to archive/lambdas/matchPickle/pythoncode/Dockerfile index a7caae450..cb6745d50 100644 --- a/archive/samfunctions/matchPickle/pythoncode/Dockerfile +++ b/archive/lambdas/matchPickle/pythoncode/Dockerfile @@ -3,13 +3,13 @@ FROM public.ecr.aws/lambda/python:3.8 COPY requirements.txt ./ RUN python3.8 -m pip install -r requirements.txt -t . -ENV LD_LIBRARY_PATH ./ +ENV LD_LIBRARY_PATH=./ # WMPL required even tho only one function called, because # pickle files reference structures defined in the library COPY WesternMeteorPyLib/ ./WesternMeteorPyLib -COPY *.npy ./WesternMeteorPyLib/wmpl/share/ +#COPY *.npy ./WesternMeteorPyLib/wmpl/share/ COPY *.py ./ -ENV PYTHONPATH ./WesternMeteorPyLib -ENV PROJ_LIB ./ +ENV PYTHONPATH=./WesternMeteorPyLib +ENV PROJ_LIB=./ CMD ["matchPickleApi.lambda_handler"] diff --git a/archive/samfunctions/matchPickle/pythoncode/__init__.py b/archive/lambdas/matchPickle/pythoncode/__init__.py similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/__init__.py rename to archive/lambdas/matchPickle/pythoncode/__init__.py diff --git a/archive/samfunctions/matchPickle/pythoncode/matchPickleApi.py b/archive/lambdas/matchPickle/pythoncode/matchPickleApi.py similarity index 100% rename from archive/samfunctions/matchPickle/pythoncode/matchPickleApi.py rename to archive/lambdas/matchPickle/pythoncode/matchPickleApi.py diff --git a/archive/samfunctions/matchPickle/pythoncode/requirements.txt b/archive/lambdas/matchPickle/pythoncode/requirements.txt similarity index 94% rename from archive/samfunctions/matchPickle/pythoncode/requirements.txt rename to archive/lambdas/matchPickle/pythoncode/requirements.txt index 58aa7aa9c..92834776d 100644 --- a/archive/samfunctions/matchPickle/pythoncode/requirements.txt +++ b/archive/lambdas/matchPickle/pythoncode/requirements.txt @@ -12,4 +12,5 @@ PyQt5 ; platform_machine != 'aarch64' pyyaml pyswarms keras -pytz \ No newline at end of file +pytz +paramiko \ No newline at end of file diff --git a/archive/lambdas/matchPickle/remoteTest.ps1 b/archive/lambdas/matchPickle/remoteTest.ps1 new file mode 100644 index 000000000..861de301a --- /dev/null +++ b/archive/lambdas/matchPickle/remoteTest.ps1 @@ -0,0 +1,5 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# test + + curl "https://api.ukmeteors.co.uk/pickle/getpickle?reqval=20260421_000017.759_UK" > ./testresult.json + diff --git a/archive/samfunctions/matchPickle/samconfig.toml b/archive/lambdas/matchPickle/samconfig.toml similarity index 100% rename from archive/samfunctions/matchPickle/samconfig.toml rename to archive/lambdas/matchPickle/samconfig.toml diff --git a/archive/samfunctions/matchPickle/template.yml b/archive/lambdas/matchPickle/template.yml similarity index 100% rename from archive/samfunctions/matchPickle/template.yml rename to archive/lambdas/matchPickle/template.yml diff --git a/archive/samfunctions/matchPickle/testEvent.json b/archive/lambdas/matchPickle/testEvent.json similarity index 100% rename from archive/samfunctions/matchPickle/testEvent.json rename to archive/lambdas/matchPickle/testEvent.json diff --git a/archive/lambdas/matchPickle/testresult.json b/archive/lambdas/matchPickle/testresult.json new file mode 100644 index 000000000..7c1230d27 --- /dev/null +++ b/archive/lambdas/matchPickle/testresult.json @@ -0,0 +1,4157 @@ +{ + "jdt_ref": 2461151.5002055545, + "meastype": 2, + "output_dir": ".", + "max_toffset": 10.0, + "verbose": false, + "v_init_part": 0.4, + "v_init_ht": null, + "fixed_time_offsets": {}, + "fixed_time_offsets_copy": {}, + "estimate_timing_vel": true, + "fixed_times": null, + "monte_carlo": true, + "mc_runs": 10, + "mc_runs_max": 20, + "mc_pick_multiplier": 1, + "mc_noise_std": 1.0, + "enable_OSM_plot": true, + "geometric_uncert": true, + "filter_picks": false, + "calc_orbit": true, + "show_plots": false, + "show_jacchia": false, + "save_results": false, + "gravity_correction": true, + "gravity_factor": 1.0, + "plot_all_spatial_residuals": false, + "plot_file_type": "png", + "traj_id": "20260421000017_W5MFS", + "reject_n_sigma_outliers": 2, + "mc_cores": 2, + "file_name": "20260421_000017", + "meas_count": 19, + "observations": [ + { + "ObservedPoints.UK002Z": { + "meas1": [ + 2.3968813154069024, + 2.398794550755332, + 2.401040877732498, + 2.403248309562171, + 2.405262341887118, + 2.407007598336504, + 2.4083661435454724, + 2.4104380988088447, + 2.4124495492369737, + 2.414010003498822, + 2.4151889792750527, + 2.4171917886793013, + 2.417909396727613, + 2.42033600533781, + 2.4223435953587114, + 2.427942162403076, + 2.4300097550164326, + 2.4318363974955406, + 2.433899657791362, + 2.436516452224417, + 2.437238417883184 + ], + "meas2": [ + 0.44034695311399874, + 0.4392383275873981, + 0.4382869800294502, + 0.43791610200467623, + 0.43703226417084373, + 0.4362760437170148, + 0.4357724143366082, + 0.4343759101176623, + 0.4336724668163452, + 0.4324539154277791, + 0.4328902971191608, + 0.43031636173280585, + 0.43144285337433486, + 0.43049915328211963, + 0.4295252455056575, + 0.42636260506048096, + 0.4263728887871292, + 0.4259131817698758, + 0.4244646349323693, + 0.42476734916852354, + 0.4238170894671063 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 0.2905716440411789, + 0.33053564404117924, + 0.41047964404117904, + 0.4504116440411789, + 0.4903636440411791, + 0.5703196440411791, + 0.6102876440411793, + 0.6502516440411792, + 0.690195644041179, + 0.7301756440411791, + 0.7701196440411788, + 0.8101196440411789, + 0.850059644041179, + 0.889999644041179, + 0.929951644041179, + 1.0498356440411787, + 1.0897596440411788, + 1.169703644041179, + 1.209671644041179, + 1.249563644041179, + 1.2895636440411786 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 21, + "JD_data": [ + 2461151.500208891, + 2461151.5002093534, + 2461151.5002102787, + 2461151.500210741, + 2461151.5002112035, + 2461151.5002121287, + 2461151.500212591, + 2461151.500213054, + 2461151.500213516, + 2461151.500213979, + 2461151.500214441, + 2461151.500214904, + 2461151.5002153665, + 2461151.500215829, + 2461151.5002162913, + 2461151.5002176785, + 2461151.500218141, + 2461151.500219066, + 2461151.5002195286, + 2461151.5002199905, + 2461151.5002204534 + ], + "lat": 0.9344075681486413, + "lon": -0.006680073179083097, + "ele": 34.0, + "station_id": "UK002Z", + "azim_data": [ + 4.737799374816502, + 4.735428543040449, + 4.732911263205364, + 4.730867396557772, + 4.728583356038361, + 4.726610236803339, + 4.725138590050284, + 4.722411691527732, + 4.720262091239325, + 4.718078528987862, + 4.717466569845322, + 4.713889250782277, + 4.7141741061193105, + 4.711500673159142, + 4.709139283371754, + 4.702204122090407, + 4.700540864881191, + 4.698711574889019, + 4.695932771463546, + 4.69404324715463, + 4.692730245169177 + ], + "elev_data": [ + 0.5397212284763535, + 0.540021754973832, + 0.5406386453085916, + 0.5416703853261442, + 0.5422000352665981, + 0.5426661854062742, + 0.5430931842237807, + 0.5432695876107108, + 0.5439333938247444, + 0.5439396765993363, + 0.5449702160484455, + 0.5442139525051555, + 0.54549253181179, + 0.5462208350060278, + 0.5466770226216804, + 0.5476093934732065, + 0.5488458699002796, + 0.549583029220384, + 0.5497111435989721, + 0.5514955495795086, + 0.5512040280201354 + ], + "ra_data": [ + 2.396881315406902, + 2.3987945507553325, + 2.401040877732498, + 2.403248309562171, + 2.4052623418871177, + 2.407007598336504, + 2.4083661435454733, + 2.4104380988088443, + 2.4124495492369737, + 2.4140100034988214, + 2.4151889792750527, + 2.417191788679302, + 2.417909396727613, + 2.4203360053378105, + 2.422343595358711, + 2.427942162403076, + 2.430009755016433, + 2.431836397495541, + 2.433899657791363, + 2.4365164522244163, + 2.4372384178831847 + ], + "dec_data": [ + 0.44034695311399874, + 0.43923832758739784, + 0.4382869800294501, + 0.43791610200467607, + 0.43703226417084373, + 0.4362760437170146, + 0.43577241433660796, + 0.4343759101176626, + 0.4336724668163451, + 0.43245391542777895, + 0.432890297119161, + 0.43031636173280596, + 0.43144285337433513, + 0.4304991532821195, + 0.4295252455056574, + 0.42636260506048085, + 0.42637288878712926, + 0.42591318176987575, + 0.4244646349323693, + 0.4247673491685236, + 0.4238170894671061 + ], + "magnitudes": [ + 3.6, + 5.02, + 3.44, + 3.24, + 4.28, + 5.1, + 5.72, + 3.45, + 3.98, + 5.66, + 4.45, + 5.81, + 3.55, + 4.2, + 5.66, + 4.01, + 4.53, + 5.21, + 3.72, + 5.72, + 6.0 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK002Z_20260421-000018.239614_8851", + "comment": "{\"ff_name\": \"FF_UK002Z_20260421_000016_194_0375808.fits\"}", + "incident_angle": 0.5002051863784102, + "weight": 0.23002152819250413, + "h_residuals": [ + -3.8052913962245976, + 20.398349033358095, + 20.60146887676896, + -17.142668686892375, + -12.648791991966329, + -13.029467986359393, + -17.831075981704, + 20.861748677458234, + 11.816255498445493, + 51.257765767545706, + -14.989881371214677, + 106.20898562360125, + 5.1376366928819195, + 0.8473289069201014, + 9.980312274554471, + 63.8895794707902, + 3.432361870992362, + -21.53507262422365, + 17.085715282877228, + -78.20881756637142, + -36.77494843909438 + ], + "h_res_rms": 29.722498774935328, + "v_residuals": [ + 6.309529854796034, + -33.877011893083605, + -34.22656685546831, + 28.4211657875748, + 20.979912654651937, + 21.618339676041263, + 29.581808972747485, + -34.69571283606128, + -19.644738726302496, + -85.4334144263602, + 24.889885886707884, + -177.67230920382528, + -8.544025956103757, + -1.4090186969998264, + -16.60820767834652, + -106.71982725101019, + -5.713621282471512, + 35.80745108911481, + -28.480033854767836, + 129.65104400143866, + 61.12507390508546 + ], + "v_res_rms": 49.431213522476874, + "velocities": [ + 0.0, + 16249.106425374774, + 8967.863527321228, + 15682.132184664548, + 15881.2387686525, + 6875.364245819561, + 10338.63052400254, + 17524.6942293953, + 14943.888606668643, + 13556.02313277404, + 6169.26635208947, + 20213.790509050847, + 1179.9960627586386, + 17843.383954063283, + 15237.361660009743, + 14427.322364972191, + 12382.078647459066, + 6167.611210861438, + 16369.803445691245, + 14542.207740346299, + 6974.687234953151 + ], + "velocities_prev_point": [ + 11765.84594020067, + 11765.84594020067, + 11765.84594020067, + 11765.84594020067, + 12515.334991192343, + 11548.839904206608, + 11165.164595519736, + 11379.517422838991, + 11657.45887779439, + 11886.958654011225, + 11826.65374047157, + 12038.458636794907, + 11892.690809748638, + 11932.525875899712, + 12027.59652074097, + 12255.042346574837, + 12396.814201326875, + 12294.978224650129, + 12284.761091501927, + 12304.68522942524, + 12262.39262157771 + ], + "length": [ + 0.0, + 649.3792891836829, + 1366.3061710118493, + 1992.5250734098718, + 2627.0123246950798, + 3176.7389483338284, + 3589.953333117164, + 4290.310213300716, + 4887.228899805485, + 5429.198704653792, + 5675.623879821653, + 6484.175500183687, + 6531.304542930267, + 7243.9692980555565, + 7852.7323710962655, + 9582.337485498587, + 10076.679593419743, + 10569.743104060852, + 11224.01140817824, + 11804.129159356135, + 12083.116648754258 + ], + "state_vect_dist": [ + 3587.0536516011, + 4236.432940784783, + 4953.35982261295, + 5579.5787250109715, + 6214.0659762961795, + 6763.792599934928, + 7177.006984718264, + 7877.363864901816, + 8474.282551406584, + 9016.252356254892, + 9262.677531422753, + 10071.229151784788, + 10118.358194531367, + 10831.022949656655, + 11439.786022697366, + 13169.391137099687, + 13663.733245020843, + 14156.796755661951, + 14811.06505977934, + 15391.182810957234, + 15670.170300355358 + ], + "lag": [ + 63.37447151468132, + 194.5118468950268, + -125.25258238414244, + -16.860626788108675, + 99.54032331926737, + -387.5799767748267, + -492.65937666999525, + -310.54441028977544, + -231.60828321249573, + -208.08787566819592, + -479.64525992782546, + -189.8023912456356, + -660.6040370513838, + -465.86997047842124, + -375.1931986155396, + -200.2100838728802, + -223.59118100337764, + -767.2189814696103, + -631.2444620307251, + -568.4349489031392, + -808.1562111848561 + ], + "lag_line": [ + 11370.947134066559, + -3165.029756068875 + ], + "v_init": 11370.947134066559, + "v_init_stddev": null, + "jacchia_fit": [ + 63.688001193520016, + 1.8900846614547042 + ], + "model_ra": [ + 2.396881593167252, + 2.3989304099368014, + 2.4011844016151613, + 2.4031860875196687, + 2.4052274845571198, + 2.406976087071311, + 2.4083103572784346, + 2.4106035391564897, + 2.412566930909117, + 2.4143572664202093, + 2.41515920660715, + 2.417869111447515, + 2.418000831463778, + 2.420405613794107, + 2.4224695120777024, + 2.428399440764061, + 2.430109452127189, + 2.4317929026796112, + 2.4340906148854335, + 2.4361376027592625, + 2.4371099642448937 + ], + "model_dec": [ + 0.440302678909872, + 0.4394764728451123, + 0.43852840306686475, + 0.4377147344407611, + 0.43688315067561767, + 0.4361219869381554, + 0.4355611737515261, + 0.4346241691042197, + 0.4338134534871204, + 0.4330683131149763, + 0.43271070454145943, + 0.431599113939899, + 0.43150463507581566, + 0.4305093179638242, + 0.4296461972321007, + 0.4271459610813783, + 0.4264149242154197, + 0.42564839310551755, + 0.42467559658948373, + 0.42380137844967836, + 0.4233614958813678 + ], + "model_azim": [ + 4.73778196828684, + 4.735521732614828, + 4.7330053965160985, + 4.730788912250429, + 4.728525435294195, + 4.726550570784113, + 4.725056923942933, + 4.722507072008819, + 4.720316125854996, + 4.7183126731808445, + 4.717397902721562, + 4.714373400873156, + 4.714197584063138, + 4.711504522278045, + 4.709184901171225, + 4.702495517776749, + 4.700556524882403, + 4.69861306303917, + 4.696010730582082, + 4.6936850389188365, + 4.692562119253269 + ], + "model_elev": [ + 0.5396755224600758, + 0.5402679008661397, + 0.5408884116776695, + 0.5414620307648099, + 0.5420456131089348, + 0.5425065260896258, + 0.5428741613485486, + 0.5435273909554157, + 0.5440798903382158, + 0.5445789635092166, + 0.5447834940136214, + 0.5455517621083557, + 0.5455568612650257, + 0.5462314278769935, + 0.5468031898818537, + 0.548429156889144, + 0.5488898437745016, + 0.5493059467086585, + 0.549932237410441, + 0.5504839878814398, + 0.5507264098873919 + ], + "model_fit1": [ + 4.73778196828684, + 4.735521732614828, + 4.7330053965160985, + 4.730788912250429, + 4.728525435294195, + 4.726550570784113, + 4.725056923942933, + 4.722507072008819, + 4.720316125854996, + 4.7183126731808445, + 4.717397902721562, + 4.714373400873156, + 4.714197584063138, + 4.711504522278045, + 4.709184901171225, + 4.702495517776749, + 4.700556524882403, + 4.69861306303917, + 4.696010730582082, + 4.6936850389188365, + 4.692562119253269 + ], + "model_fit2": [ + 0.5396755224600758, + 0.5402679008661397, + 0.5408884116776695, + 0.5414620307648099, + 0.5420456131089348, + 0.5425065260896258, + 0.5428741613485486, + 0.5435273909554157, + 0.5440798903382158, + 0.5445789635092166, + 0.5447834940136214, + 0.5455517621083557, + 0.5455568612650257, + 0.5462314278769935, + 0.5468031898818537, + 0.548429156889144, + 0.5488898437745016, + 0.5493059467086585, + 0.549932237410441, + 0.5504839878814398, + 0.5507264098873919 + ], + "meas_eci": [ + [ + -0.6651403376230054, + 0.6131039197684128, + 0.42625334583010854 + ], + [ + -0.6666597964124698, + 0.6121494659158085, + 0.42525021719721046 + ], + [ + -0.6683315036211338, + 0.610923065694228, + 0.4243889832103093 + ], + [ + -0.6697948071375551, + 0.6095521783058309, + 0.424053131404759 + ], + [ + -0.6712985427876046, + 0.6084534187936955, + 0.42325252935984004 + ], + [ + -0.6725967633254724, + 0.6074952687408567, + 0.42256726378361675 + ], + [ + -0.6735794940210694, + 0.6067233106956108, + 0.4221107550073526 + ], + [ + -0.6752733035008325, + 0.6057194052358751, + 0.42084435091821104 + ], + [ + -0.6767109144101443, + 0.6045569835569059, + 0.42020613031078136 + ], + [ + -0.6780353559270645, + 0.6038403657657955, + 0.41910007013205 + ], + [ + -0.6786100121852274, + 0.6029190303333297, + 0.41949623862900476 + ], + [ + -0.6806225551669195, + 0.602272242271424, + 0.41715834354283843 + ], + [ + -0.6807019943374223, + 0.6014721244956676, + 0.4181818723470973 + ], + [ + -0.6824555864862213, + 0.6000788818635572, + 0.4173244637151418 + ], + [ + -0.6839643602217045, + 0.5989750663284349, + 0.4164392198909243 + ], + [ + -0.6882992447918803, + 0.5959956375357374, + 0.41356178457069387 + ], + [ + -0.6895268279492358, + 0.5945684648394516, + 0.41357114763571556 + ], + [ + -0.6907558791802513, + 0.5934317844296477, + 0.4131525536730498 + ], + [ + -0.6924328423375028, + 0.592393747301069, + 0.41183298436502674 + ], + [ + -0.6938856684688789, + 0.5904989435368124, + 0.4121088166678862 + ], + [ + -0.6946099128507072, + 0.5902511457747254, + 0.41124281620613995 + ] + ], + "meas_eci_los": [ + [ + -0.6651532201448609, + 0.6130899435642051, + 0.42625334583010854 + ], + [ + -0.6666744422176303, + 0.6121335155564811, + 0.42525021719721046 + ], + [ + -0.668349681390559, + 0.6109031791657809, + 0.4243889832103093 + ], + [ + -0.669814719865365, + 0.6095302968659562, + 0.424053131404759 + ], + [ + -0.6713201921678998, + 0.6084295324671147, + 0.42325252935984004 + ], + [ + -0.6726219198904363, + 0.6074674151436691, + 0.42256726378361675 + ], + [ + -0.6736063861027676, + 0.6066934539853648, + 0.4221107550073526 + ], + [ + -0.6753019174198893, + 0.6056875041052527, + 0.42084435091821104 + ], + [ + -0.6767412328073986, + 0.6045230449433344, + 0.42020613031078136 + ], + [ + -0.6780673992284806, + 0.6038043833219787, + 0.41910007013205 + ], + [ + -0.6786437629715607, + 0.602881040302245, + 0.41949623862900476 + ], + [ + -0.6806580260024921, + 0.6022321546139711, + 0.41715834354283843 + ], + [ + -0.6807391702046832, + 0.6014300489577449, + 0.4181818723470973 + ], + [ + -0.6824944243246932, + 0.6000347096215143, + 0.4173244637151418 + ], + [ + -0.6840048714786812, + 0.598928803723841, + 0.4164392198909243 + ], + [ + -0.6883447630503693, + 0.5959430656730842, + 0.41356178457069387 + ], + [ + -0.6895739692035289, + 0.5945137902859511, + 0.41357114763571556 + ], + [ + -0.6908063894172288, + 0.5933729853420778, + 0.4131525536730498 + ], + [ + -0.6924849898642252, + 0.5923327880522399, + 0.41183298436502674 + ], + [ + -0.693939367574386, + 0.5904358367808104, + 0.4121088166678862 + ], + [ + -0.6946653105480682, + 0.590185947342021, + 0.41124281620613995 + ] + ], + "model_eci": [ + [ + -3431648.7433502977, + -1734673.2501258308, + 5171746.249168466 + ], + [ + -3431502.168458378, + -1735198.4026242748, + 5171393.320457905 + ], + [ + -3431340.2471023854, + -1735778.206261865, + 5171003.513213945 + ], + [ + -3431198.8672727705, + -1736284.6365797631, + 5170663.117429066 + ], + [ + -3431055.612694895, + -1736797.753523634, + 5170318.214222535 + ], + [ + -3430931.3319215793, + -1737242.3632842037, + 5170019.113061614 + ], + [ + -3430837.965680444, + -1737576.5501712172, + 5169794.374518736 + ], + [ + -3430679.813194559, + -1738142.9361567015, + 5169413.623255734 + ], + [ + -3430544.9854288357, + -1738625.679263341, + 5169089.050559572 + ], + [ + -3430422.545034539, + -1739063.9846602748, + 5168794.316972965 + ], + [ + -3430366.765328334, + -1739263.305532724, + 5168660.1248029135 + ], + [ + -3430184.1694901455, + -1739917.1765316478, + 5168220.538002146 + ], + [ + -3430173.328761385, + -1739955.342499328, + 5168194.58388284 + ], + [ + -3430012.3423658106, + -1740531.6864444455, + 5167807.052084736 + ], + [ + -3429874.786286485, + -1741024.0084887384, + 5167475.953263732 + ], + [ + -3429483.8746772027, + -1742422.7843117185, + 5166535.0961491605 + ], + [ + -3429372.090910218, + -1742822.589670693, + 5166266.09232802 + ], + [ + -3429260.304079202, + -1743221.4092456542, + 5165997.30074232 + ], + [ + -3429112.4197078072, + -1743750.5190035156, + 5165641.380819524 + ], + [ + -3428981.246454044, + -1744219.694618384, + 5165325.712035401 + ], + [ + -3428918.0015465165, + -1744445.3384067302, + 5165173.638196536 + ] + ], + "meas_lat": [ + 0.9346450225936334, + 0.9345970660398534, + 0.9345465778299221, + 0.9345060097444198, + 0.9344609020518903, + 0.9344222008936756, + 0.9343935294446254, + 0.9343405561033522, + 0.9342993156178634, + 0.9342574203614202, + 0.93424620996589, + 0.9341778905737628, + 0.9341839571465814, + 0.9341340643559439, + 0.9340902381901774, + 0.9339630599390059, + 0.9339337530240859, + 0.9339012154522863, + 0.933851362550423, + 0.9338192346503082, + 0.9337956204733615 + ], + "meas_lon": [ + -0.04088550365419589, + -0.04075548711489885, + -0.04060781279524608, + -0.04046702814824051, + -0.04033520755003031, + -0.04022319461679362, + -0.040136396159376767, + -0.03999918021151192, + -0.03987197780910561, + -0.03976888838620386, + -0.03970217229821398, + -0.03956286802268136, + -0.03952989329711795, + -0.03937914973331329, + -0.039254062049537145, + -0.03890628729167373, + -0.0387879885582501, + -0.03868188951868613, + -0.03855458929929464, + -0.038409166927020906, + -0.03836284386104872 + ], + "meas_ht": [ + 80166.50920069027, + 79907.89847155106, + 79663.6709640296, + 79509.71227376115, + 79287.06050564267, + 79100.56625424455, + 78967.5446651115, + 78669.14774349949, + 78480.50679384157, + 78234.48391237574, + 78254.36663415054, + 77789.32418704791, + 77932.08594644774, + 77696.87267580134, + 77475.89917794727, + 76804.02455018043, + 76731.24392918951, + 76602.42998524354, + 76320.02753907315, + 76272.04281635238, + 76112.63523459999 + ], + "meas_range": [ + 153339.33047024682, + 152779.23373854544, + 152169.97299907313, + 151631.42779261048, + 151086.25079201206, + 150624.91058615095, + 150273.74436083622, + 149672.74252612054, + 149162.45846822963, + 148700.46758205735, + 148495.59961464474, + 147802.94181198507, + 147771.35775808938, + 147162.96362779627, + 146645.26441739433, + 145179.17570669635, + 144762.42956183938, + 144356.17304021947, + 143802.8283496073, + 143313.7343373524, + 143083.5198313615 + ], + "model_lat": [ + 0.9346446667677631, + 0.9345989704981107, + 0.9345485008239762, + 0.9345044075468767, + 0.9344597200105587, + 0.9344209834416608, + 0.9343918634192264, + 0.9343425020668983, + 0.9343004177428308, + 0.9342621973971849, + 0.9342448099303645, + 0.9341877726487242, + 0.9341844358943037, + 0.9341341428363436, + 0.9340911683984181, + 0.9339690073179128, + 0.9339340724672882, + 0.9338992059698612, + 0.9338529539754011, + 0.9338119290348905, + 0.9337921890458436 + ], + "model_lon": [ + -0.04088647065639903, + -0.0407503038625055, + -0.04060257766467196, + -0.040471381075913575, + -0.0403384201294265, + -0.04022650428958335, + -0.04014092463279356, + -0.03999387866505028, + -0.039868976417382226, + -0.03975584834373117, + -0.03970598017255938, + -0.03953580064279074, + -0.03952858935807945, + -0.0393789360113246, + -0.03925152725986469, + -0.03889002516887339, + -0.038787118004688446, + -0.038687359650227564, + -0.03855024718026426, + -0.03842899481740237, + -0.038372180120937446 + ], + "model_ht": [ + 80160.57129329837, + 79939.75564689818, + 79695.8571107965, + 79482.97408082303, + 79267.32029777888, + 79080.2240190833, + 78939.71062025194, + 78701.77767818491, + 78498.9785627082, + 78314.84618101093, + 78230.94224464067, + 77956.46774587498, + 77940.11468171058, + 77698.18875523485, + 77491.51556920519, + 76904.43232644908, + 76736.61042515974, + 76568.72510363306, + 76346.81825810629, + 76150.02295057768, + 76055.10224458351 + ], + "model_range": [ + 153339.33064750832, + 152779.23885477215, + 152169.9782410282, + 151631.43142680972, + 151086.25277939526, + 150624.9127025781, + 150273.74833255954, + 149672.74799874792, + 149162.46022825968, + 148700.50095141708, + 148495.6024593428, + 147803.08674493877, + 147771.3580936107, + 147162.9636368488, + 146645.26569589667, + 145179.22897841557, + 144762.42971472227, + 144356.17909111048, + 143802.8321819995, + 143313.81433522768, + 143083.53761941093 + ], + "rbeg_lat": 0.9346446667677631, + "rbeg_lon": -0.04088647065639903, + "rbeg_ele": 80160.57129329837, + "rbeg_jd": 2461151.500208891, + "rend_lat": 0.9337921890458436, + "rend_lon": -0.038372180120937446, + "rend_ele": 76055.10224458351, + "rend_jd": 2461151.5002204534, + "htmin_lat": 0.9337921890458436, + "htmin_lon": -0.038372180120937446, + "htmin_ele": 76055.10224458351, + "htmin_jd": 2461151.5002204534, + "absolute_magnitudes": [ + 2.671732184227931, + 4.099678289776929, + 2.528355107417953, + 2.3360538259627734, + 3.3838752499018385, + 4.210515958921451, + 4.835584403455401, + 2.5742863380034136, + 3.111702310827387, + 4.798437841987752, + 3.591432036646971, + 4.961582479831281, + 2.7020486763676725, + 3.3610073736334165, + 4.828659766224398, + 3.200477571944522, + 3.726720681188872, + 4.412823108180669, + 2.931162802418613, + 4.938559724900953, + 5.222051654133499 + ], + "ra_data_los": [ + 2.396902327617516, + 2.3988184762780778, + 2.401070632813468, + 2.403280977947147, + 2.4052979235842242, + 2.4070490095918355, + 2.408410468104811, + 2.4104853396142762, + 2.4124997004205735, + 2.414063070928515, + 2.415244960008752, + 2.417250685659096, + 2.4179712070195376, + 2.420400728941866, + 2.42241123226677, + 2.42801853924753, + 2.43008904516489, + 2.431921517202225, + 2.433987690810178, + 2.4366073956132723, + 2.4373322775181343 + ], + "dec_data_los": [ + 0.44034695311399874, + 0.43923832758739784, + 0.4382869800294501, + 0.43791610200467607, + 0.43703226417084373, + 0.4362760437170146, + 0.43577241433660796, + 0.4343759101176626, + 0.4336724668163451, + 0.43245391542777895, + 0.432890297119161, + 0.43031636173280596, + 0.43144285337433513, + 0.4304991532821195, + 0.4295252455056574, + 0.42636260506048085, + 0.42637288878712926, + 0.42591318176987575, + 0.4244646349323693, + 0.4247673491685236, + 0.4238170894671061 + ], + "x_eci": [ + -0.6651403376230054, + -0.6666597964124698, + -0.6683315036211338, + -0.6697948071375551, + -0.6712985427876046, + -0.6725967633254724, + -0.6735794940210694, + -0.6752733035008325, + -0.6767109144101443, + -0.6780353559270645, + -0.6786100121852274, + -0.6806225551669195, + -0.6807019943374223, + -0.6824555864862213, + -0.6839643602217045, + -0.6882992447918803, + -0.6895268279492358, + -0.6907558791802513, + -0.6924328423375028, + -0.6938856684688789, + -0.6946099128507072 + ], + "y_eci": [ + 0.6131039197684128, + 0.6121494659158085, + 0.610923065694228, + 0.6095521783058309, + 0.6084534187936955, + 0.6074952687408567, + 0.6067233106956108, + 0.6057194052358751, + 0.6045569835569059, + 0.6038403657657955, + 0.6029190303333297, + 0.602272242271424, + 0.6014721244956676, + 0.6000788818635572, + 0.5989750663284349, + 0.5959956375357374, + 0.5945684648394516, + 0.5934317844296477, + 0.592393747301069, + 0.5904989435368124, + 0.5902511457747254 + ], + "z_eci": [ + 0.42625334583010854, + 0.42525021719721046, + 0.4243889832103093, + 0.424053131404759, + 0.42325252935984004, + 0.42256726378361675, + 0.4221107550073526, + 0.42084435091821104, + 0.42020613031078136, + 0.41910007013205, + 0.41949623862900476, + 0.41715834354283843, + 0.4181818723470973, + 0.4173244637151418, + 0.4164392198909243, + 0.41356178457069387, + 0.41357114763571556, + 0.4131525536730498, + 0.41183298436502674, + 0.4121088166678862, + 0.41124281620613995 + ], + "x_eci_los": [ + -0.6651532201448609, + -0.6666744422176303, + -0.668349681390559, + -0.669814719865365, + -0.6713201921678998, + -0.6726219198904363, + -0.6736063861027676, + -0.6753019174198893, + -0.6767412328073986, + -0.6780673992284806, + -0.6786437629715607, + -0.6806580260024921, + -0.6807391702046832, + -0.6824944243246932, + -0.6840048714786812, + -0.6883447630503693, + -0.6895739692035289, + -0.6908063894172288, + -0.6924849898642252, + -0.693939367574386, + -0.6946653105480682 + ], + "y_eci_los": [ + 0.6130899435642051, + 0.6121335155564811, + 0.6109031791657809, + 0.6095302968659562, + 0.6084295324671147, + 0.6074674151436691, + 0.6066934539853648, + 0.6056875041052527, + 0.6045230449433344, + 0.6038043833219787, + 0.602881040302245, + 0.6022321546139711, + 0.6014300489577449, + 0.6000347096215143, + 0.598928803723841, + 0.5959430656730842, + 0.5945137902859511, + 0.5933729853420778, + 0.5923327880522399, + 0.5904358367808104, + 0.590185947342021 + ], + "z_eci_los": [ + 0.42625334583010854, + 0.42525021719721046, + 0.4243889832103093, + 0.424053131404759, + 0.42325252935984004, + 0.42256726378361675, + 0.4221107550073526, + 0.42084435091821104, + 0.42020613031078136, + 0.41910007013205, + 0.41949623862900476, + 0.41715834354283843, + 0.4181818723470973, + 0.4173244637151418, + 0.4164392198909243, + 0.41356178457069387, + 0.41357114763571556, + 0.4131525536730498, + 0.41183298436502674, + 0.4121088166678862, + 0.41124281620613995 + ], + "x_stat": -3329692.8394425996, + "y_stat": -1828618.1638918526, + "z_stat": 5106390.987831098, + "stat_eci": [ + -3329692.8394425996, + -1828618.1638918526, + 5106390.987831098 + ], + "stat_eci_los": [ + [ + -3329654.415397559, + -1828688.127695391, + 5106390.987831098 + ], + [ + -3329649.0878441236, + -1828697.8280102296, + 5106390.987831098 + ], + [ + -3329638.4272870813, + -1828717.2383623847, + 5106390.987831098 + ], + [ + -3329633.099663696, + -1828726.9386035914, + 5106390.987831098 + ], + [ + -3329627.771997192, + -1828736.6388563307, + 5106390.987831098 + ], + [ + -3329617.1112139, + -1828756.049084223, + 5106390.987831098 + ], + [ + -3329611.783477447, + -1828765.7492633294, + 5106390.987831098 + ], + [ + -3329606.450332315, + -1828775.4592229126, + 5106390.987831098 + ], + [ + -3329601.1278900444, + -1828785.1496290863, + 5106390.987831098 + ], + [ + -3329595.7946883356, + -1828794.859557595, + 5106390.987831098 + ], + [ + -3329590.466838816, + -1828804.5596746, + 5106390.987831098 + ], + [ + -3329585.1335805035, + -1828814.2695720182, + 5106390.987831098 + ], + [ + -3329579.805659577, + -1828823.9696850183, + 5106390.987831098 + ], + [ + -3329574.4777103914, + -1828833.6697824954, + 5106390.987831098 + ], + [ + -3329569.1497478094, + -1828843.369837391, + 5106390.987831098 + ], + [ + -3329553.1656459146, + -1828872.4699901317, + 5106390.987831098 + ], + [ + -3329547.837570296, + -1828882.1699829372, + 5106390.987831098 + ], + [ + -3329537.17593862, + -1828901.5797448508, + 5106390.987831098 + ], + [ + -3329531.847763332, + -1828911.2797181348, + 5106390.987831098 + ], + [ + -3329526.5249406365, + -1828920.9698801264, + 5106390.987831098 + ], + [ + -3329521.191342843, + -1828930.679591074, + 5106390.987831098 + ] + ], + "plane_N": [ + -0.010758941735242446, + 0.5626841701025131, + -0.8266019416192922 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 4.808344044228582e-05, + 0.00025879609182125133, + 0.0002624805684568251, + 0.0002189398121328159, + 0.00016219719514936625, + 0.00016763637746942077, + 0.0002299126917867094, + 0.0002704217170759464, + 0.0001536191301664089, + 0.000669935090213715, + 0.0001957385228524681, + 0.0014004153194447633, + 6.738763283318817e-05, + 1.1091740167342496e-05, + 0.00013204797495846036, + 0.0008566651982668407, + 4.595851975374786e-05, + 0.0002895389345591846, + 0.0002308693235996496, + 0.0010565990502203683, + 0.0004986368036220028 + ], + "ang_res_std": 0.0003960746824252553 + } + }, + { + "ObservedPoints.UK00AS": { + "meas1": [ + 4.2684960666937455, + 4.275705660567851, + 4.282316120282617, + 4.287776816608948, + 4.295439857062819, + 4.301929096125738, + 4.308588575262349, + 4.314596261320367, + 4.3215806860608525, + 4.327444662401953, + 4.331822069286927, + 4.335799177502339 + ], + "meas2": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.0734864670857704, + 1.0676655293254065, + 1.0625353098743915, + 1.0568295338648426, + 1.0530700757470863, + 1.047988416812209, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 0.8142981686851195, + 0.8543421686851195, + 0.8943901686851182, + 0.934414168685119, + 0.97446616868512, + 1.0145101686851197, + 1.054550168685119, + 1.09461416868512, + 1.1346821686851198, + 1.1747101686851193, + 1.2147341686851183, + 1.2547741686851195 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 12, + "JD_data": [ + 2461151.500214981, + 2461151.5002154447, + 2461151.500215908, + 2461151.5002163714, + 2461151.5002168347, + 2461151.5002172985, + 2461151.500217762, + 2461151.5002182256, + 2461151.5002186894, + 2461151.5002191523, + 2461151.5002196156, + 2461151.500220079 + ], + "lat": 0.9311541522498763, + "lon": -0.045808173347230974, + "ele": 57.0, + "station_id": "UK00AS", + "azim_data": [ + 0.8832534008354802, + 0.8974325902214364, + 0.9104409422636692, + 0.9232847050629855, + 0.9391342537659964, + 0.9528309723365957, + 0.9677775061336478, + 0.9774560061825399, + 0.9902964441331861, + 1.0053988251477577, + 1.01598915152872, + 1.0237734767338456 + ], + "elev_data": [ + 1.191408858155571, + 1.1885973861086219, + 1.1859092541919156, + 1.1836589415935101, + 1.1802988113469086, + 1.1773284999232583, + 1.1741538069154087, + 1.1711813003457707, + 1.1676161578412807, + 1.164451512475476, + 1.162008573079885, + 1.1597749824215797 + ], + "ra_data": [ + 4.2684960666937455, + 4.275705660567851, + 4.282316120282618, + 4.2877768166089485, + 4.295439857062819, + 4.301929096125738, + 4.308588575262349, + 4.314596261320368, + 4.3215806860608525, + 4.327444662401954, + 4.331822069286927, + 4.33579917750234 + ], + "dec_data": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.07348646708577, + 1.0676655293254065, + 1.0625353098743917, + 1.0568295338648421, + 1.0530700757470863, + 1.0479884168122084, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "magnitudes": [ + 2.07, + 2.55, + 2.28, + 2.28, + 2.54, + 2.07, + 2.44, + 2.91, + 1.92, + 2.53, + 4.48, + 3.45 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK00AS_20260421-000018.455603_8295", + "comment": "{\"ff_name\": \"FF_UK00AS_20260421_000008_555_0363264.fits\"}", + "incident_angle": 1.265480375315784, + "weight": 0.909642928791794, + "h_residuals": [ + 5.292367312510447, + -1.2325180959965676, + -8.34854165501151, + 18.532634348422874, + 21.298045268951483, + 26.66314727338721, + 45.7974867531929, + 1.1143548526059945, + -25.281726091833455, + 20.538142904874615, + 42.62245203257627, + 32.87553355400295 + ], + "h_res_rms": 25.272385757689367, + "v_residuals": [ + 1.948174626316228, + -0.453757025563914, + -3.0739954328321524, + 6.815581499143073, + 7.830632268485128, + 9.7997429292699, + 16.81714105751497, + 0.40987265024026254, + -9.307426018837527, + 7.546207273811804, + 15.64451435403657, + 12.069949587749356 + ], + "v_res_rms": 9.28371695850286, + "velocities": [ + 0.0, + 12969.297700466464, + 12063.810480597225, + 11535.301400186596, + 14810.349609688103, + 12948.52617841113, + 14077.237634374165, + 10248.745079827027, + 13145.40515860947, + 14305.426366502534, + 10388.296265044806, + 8232.259675064872 + ], + "velocities_prev_point": [ + 12177.028722194334, + 12177.028722194334, + 12177.028722194334, + 12177.028722194334, + 12635.927988514766, + 12811.691915261166, + 13010.049718796889, + 12862.45411623616, + 12812.459983391418, + 12868.413677161267, + 12786.266891221863, + 12563.748843977468 + ], + "length": [ + 0.0, + 519.3425571174787, + 1002.4740392444213, + 1464.162942485498, + 2057.347065052742, + 2575.8578473410325, + 3139.510442221366, + 3550.116165099566, + 4076.826258994727, + 4649.4438655930835, + 5065.225035305226, + 5394.844712694833 + ], + "state_vect_dist": [ + 9921.354086256408, + 10440.696643373885, + 10923.828125500828, + 11385.517028741906, + 11978.70115130915, + 12497.21193359744, + 13060.864528477774, + 13471.470251355973, + 13998.180345251134, + 14570.79795184949, + 14986.579121561634, + 15316.19879895124 + ], + "lag": [ + -393.8633893220631, + -393.8001635112796, + -429.99988356618087, + -487.3309572559665, + -413.52990774576574, + -414.29845676416335, + -369.87332231534674, + -478.8062851196937, + -471.68674778223067, + -418.1409889898896, + -521.3797962085864, + -710.987579250519 + ], + "lag_line": [ + 12179.051046767432, + -9899.353326496881 + ], + "v_init": 12179.051046767432, + "v_init_stddev": null, + "jacchia_fit": [ + 169.25151905025842, + 0.9545719833694175 + ], + "model_ra": [ + 4.268675306470282, + 4.275740112640083, + 4.282193626963838, + 4.288259757151979, + 4.295985212896367, + 4.302594155354585, + 4.3096754275247084, + 4.3147005165367975, + 4.321109894501091, + 4.327977157888565, + 4.332834122852471, + 4.336600836612544 + ], + "model_dec": [ + 1.0877269738198503, + 1.0827457055324, + 1.078088202406111, + 1.073615583501862, + 1.067812547679924, + 1.0627178988374508, + 1.0571404830083215, + 1.0530775705703026, + 1.0478192667242059, + 1.0420648001571293, + 1.0378881555178725, + 1.0345818657151975 + ], + "model_azim": [ + 0.8831700010505981, + 0.8974524277636791, + 0.9105774389431591, + 0.9229769805934369, + 0.9387740337613688, + 0.9523731955673052, + 0.9669789957103925, + 0.9774364487803786, + 0.9907468066797422, + 1.0050282483732862, + 1.0152133154617018, + 1.0231714421513614 + ], + "model_elev": [ + 1.1913491255580793, + 1.188611241077182, + 1.1860024765005819, + 1.1834532326767415, + 1.1800641024878358, + 1.1770365570460057, + 1.1736559939503042, + 1.171169280976094, + 1.1678878621525661, + 1.1642326384892834, + 1.161556949829732, + 1.1594281762863772 + ], + "model_fit1": [ + 0.8831700010505981, + 0.8974524277636791, + 0.9105774389431591, + 0.9229769805934369, + 0.9387740337613688, + 0.9523731955673052, + 0.9669789957103925, + 0.9774364487803786, + 0.9907468066797422, + 1.0050282483732862, + 1.0152133154617018, + 1.0231714421513614 + ], + "model_fit2": [ + 1.1913491255580793, + 1.188611241077182, + 1.1860024765005819, + 1.1834532326767415, + 1.1800641024878358, + 1.1770365570460057, + 1.1736559939503042, + 1.171169280976094, + 1.1678878621525661, + 1.1642326384892834, + 1.161556949829732, + 1.1594281762863772 + ], + "meas_eci": [ + [ + -0.19949752969882315, + -0.41951346273949935, + 0.885555865106421 + ], + [ + -0.19831370967000816, + -0.42489567883123996, + 0.8832527014775893 + ], + [ + -0.19719537224748762, + -0.42989189519454285, + 0.8810828244894001 + ], + [ + -0.19653440624741567, + -0.43469911459331356, + 0.8788691068257918 + ], + [ + -0.19526604393742353, + -0.44086253314794654, + 0.8760772790977996 + ], + [ + -0.19419198311969488, + -0.44623568023084337, + 0.8735921195735208 + ], + [ + -0.1931713762708322, + -0.45209532151521414, + 0.8708011481697114 + ], + [ + -0.19171870682741307, + -0.45626258010862597, + 0.8689467161138501 + ], + [ + -0.19020697856453836, + -0.4616673137779947, + 0.8664205657152649 + ], + [ + -0.18946461302037865, + -0.4676323484404185, + 0.8633789128216748 + ], + [ + -0.18879990840657096, + -0.4719169246748595, + 0.8611904613911451 + ], + [ + -0.18794699339414994, + -0.4752570683607163, + 0.8595386242905415 + ] + ], + "meas_eci_los": [ + [ + -0.19947261818984635, + -0.41952530837615254, + 0.885555865106421 + ], + [ + -0.19828723693595823, + -0.4249080335800251, + 0.8832527014775893 + ], + [ + -0.1971673332669236, + -0.42990475582643994, + 0.8810828244894001 + ], + [ + -0.19650478472685406, + -0.4347125057399289, + 0.8788691068257918 + ], + [ + -0.19523471543822826, + -0.44087640777926657, + 0.8760772790977996 + ], + [ + -0.19415896881448527, + -0.4462500458800414, + 0.8735921195735208 + ], + [ + -0.19313660866544666, + -0.4521101754434669, + 0.8708011481697114 + ], + [ + -0.19168228546062463, + -0.4562778824308138, + 0.8689467161138501 + ], + [ + -0.19016877668854015, + -0.4616830511058688, + 0.8664205657152649 + ], + [ + -0.18942455378532386, + -0.4676485767305336, + 0.8633789128216748 + ], + [ + -0.18875810448810454, + -0.4719336470299235, + 0.8611904613911451 + ], + [ + -0.18790350619325405, + -0.4752742636762536, + 0.8595386242905415 + ] + ], + "model_eci": [ + [ + -3430218.4459287277, + -1739794.238366333, + 5168303.099138084 + ], + [ + -3430101.130186622, + -1740214.0687953206, + 5168020.730054632 + ], + [ + -3429991.976360051, + -1740604.6083564344, + 5167758.024153548 + ], + [ + -3429887.652948811, + -1740977.7996632662, + 5167506.958303962 + ], + [ + -3429753.6593147092, + -1741457.3125123843, + 5167184.447316998 + ], + [ + -3429636.5043168305, + -1741876.432179816, + 5166902.494989177 + ], + [ + -3429509.159525174, + -1742332.044553453, + 5166596.010709642 + ], + [ + -3429416.332842804, + -1742663.8893094896, + 5166372.661447595 + ], + [ + -3429297.3084989507, + -1743089.6082505668, + 5166086.229687476 + ], + [ + -3429167.922097079, + -1743552.438223741, + 5165774.850941512 + ], + [ + -3429073.9084734865, + -1743888.437613092, + 5165548.665302157 + ], + [ + -3428999.3243624857, + -1744154.7560413252, + 5165369.277862051 + ] + ], + "meas_lat": [ + 0.9341977291845097, + 0.9341620422361886, + 0.9341289887201897, + 0.9340925154995868, + 0.9340502484747027, + 0.9340128678846222, + 0.9339702969792177, + 0.9339477745675356, + 0.9339143956069487, + 0.9338673009272427, + 0.9338347231487523, + 0.9338128429395532 + ], + "meas_lon": [ + -0.039569337714437845, + -0.0394604803690885, + -0.039359318234566246, + -0.03926606572328635, + -0.03914232800703642, + -0.039034834123200954, + -0.03891903030012694, + -0.03882974941824365, + -0.03871741853890708, + -0.038602319686930085, + -0.03851846883407553, + -0.03844994217817617 + ], + "meas_ht": [ + 78009.79883316693, + 77831.20354207544, + 77664.69644717604, + 77517.24209363233, + 77316.91530664213, + 77142.80788407566, + 76958.1916129066, + 76803.36867867864, + 76615.55707637589, + 76437.24094479789, + 76303.77256128346, + 76188.47275826824 + ], + "meas_range": [ + 83839.35913329039, + 83740.33676324955, + 83650.64655986936, + 83567.18288250422, + 83464.48024344677, + 83377.45673232341, + 83286.60977747764, + 83221.62612094023, + 83142.13428744262, + 83059.64575911379, + 83000.93009824904, + 82954.89554656077 + ], + "model_lat": [ + 0.9341984959671219, + 0.9341618633579815, + 0.9341277785486444, + 0.9340952007177374, + 0.934053334269257, + 0.9340167308727484, + 0.9339769318673277, + 0.9339479355618667, + 0.9339107317015435, + 0.9338702759470549, + 0.9338408971786679, + 0.933817604920053 + ], + "model_lon": [ + -0.039568822172874184, + -0.039460600665898946, + -0.03936013225744678, + -0.03926425828503005, + -0.03914025008272261, + -0.03903223179007913, + -0.03891455786744328, + -0.03882964093347483, + -0.03871988735385722, + -0.03860031305219093, + -0.03851430205713854, + -0.03844672807852617 + ], + "model_ht": [ + 78007.96659065987, + 77831.63104327311, + 77667.58915264205, + 77510.83050120615, + 77309.5485524352, + 77133.5883195504, + 76942.36899348421, + 76802.98414373519, + 76624.31618154436, + 76430.14134564155, + 76289.05225369317, + 76177.11611285266 + ], + "model_range": [ + 83839.35932285733, + 83740.33677357878, + 83650.64703317253, + 83567.18521487652, + 83464.4833274639, + 83377.46157053398, + 83286.62406500315, + 83221.62612936008, + 83142.13865346192, + 83059.64864007127, + 83000.94251389406, + 82954.90293702218 + ], + "rbeg_lat": 0.9341984959671219, + "rbeg_lon": -0.039568822172874184, + "rbeg_ele": 78007.96659065987, + "rbeg_jd": 2461151.500214981, + "rend_lat": 0.933817604920053, + "rend_lon": -0.03844672807852617, + "rend_ele": 76177.11611285266, + "rend_jd": 2461151.500220079, + "htmin_lat": 0.933817604920053, + "htmin_lon": -0.03844672807852617, + "htmin_ele": 76177.11611285266, + "htmin_jd": 2461151.500220079, + "absolute_magnitudes": [ + 2.4527602455285358, + 2.935326484323726, + 2.667653477222837, + 2.6698211294198684, + 2.9324914534964, + 2.4647566555887437, + 2.837123706123827, + 3.3088190122834766, + 2.3208940419532453, + 2.9330495519000843, + 4.884584879911243, + 3.855789702473728 + ], + "ra_data_los": [ + 4.268555447761289, + 4.2757679637412895, + 4.282381342636115, + 4.287844958142504, + 4.295510917776434, + 4.302003078945247, + 4.3086654772619175, + 4.314676085433959, + 4.321663432280336, + 4.327530324867534, + 4.331910650932565, + 4.3358906783280355 + ], + "dec_data_los": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.07348646708577, + 1.0676655293254065, + 1.0625353098743917, + 1.0568295338648421, + 1.0530700757470863, + 1.0479884168122084, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "x_eci": [ + -0.19949752969882315, + -0.19831370967000816, + -0.19719537224748762, + -0.19653440624741567, + -0.19526604393742353, + -0.19419198311969488, + -0.1931713762708322, + -0.19171870682741307, + -0.19020697856453836, + -0.18946461302037865, + -0.18879990840657096, + -0.18794699339414994 + ], + "y_eci": [ + -0.41951346273949935, + -0.42489567883123996, + -0.42989189519454285, + -0.43469911459331356, + -0.44086253314794654, + -0.44623568023084337, + -0.45209532151521414, + -0.45626258010862597, + -0.4616673137779947, + -0.4676323484404185, + -0.4719169246748595, + -0.4752570683607163 + ], + "z_eci": [ + 0.885555865106421, + 0.8832527014775893, + 0.8810828244894001, + 0.8788691068257918, + 0.8760772790977996, + 0.8735921195735208, + 0.8708011481697114, + 0.8689467161138501, + 0.8664205657152649, + 0.8633789128216748, + 0.8611904613911451, + 0.8595386242905415 + ], + "x_eci_los": [ + -0.19947261818984635, + -0.19828723693595823, + -0.1971673332669236, + -0.19650478472685406, + -0.19523471543822826, + -0.19415896881448527, + -0.19313660866544666, + -0.19168228546062463, + -0.19016877668854015, + -0.18942455378532386, + -0.18875810448810454, + -0.18790350619325405 + ], + "y_eci_los": [ + -0.41952530837615254, + -0.4249080335800251, + -0.42990475582643994, + -0.4347125057399289, + -0.44087640777926657, + -0.4462500458800414, + -0.4521101754434669, + -0.4562778824308138, + -0.4616830511058688, + -0.4676485767305336, + -0.4719336470299235, + -0.4752742636762536 + ], + "z_eci_los": [ + 0.885555865106421, + 0.8832527014775893, + 0.8810828244894001, + 0.8788691068257918, + 0.8760772790977996, + 0.8735921195735208, + 0.8708011481697114, + 0.8689467161138501, + 0.8664205657152649, + 0.8633789128216748, + 0.8611904613911451, + 0.8595386242905415 + ], + "x_stat": -3413601.4235978713, + "y_stat": -1704419.3296659163, + "z_stat": 5094057.194026681, + "stat_eci": [ + -3413601.4235978713, + -1704419.3296659163, + 5094057.194026681 + ], + "stat_eci_los": [ + [ + -3413500.207340225, + -1704622.0299575112, + 5094057.194026681 + ], + [ + -3413495.2262395704, + -1704632.004559309, + 5094057.194026681 + ], + [ + -3413490.2500972687, + -1704641.969159245, + 5094057.194026681 + ], + [ + -3413485.273925883, + -1704651.9337446468, + 5094057.194026681 + ], + [ + -3413480.2977254046, + -1704661.89831553, + 5094057.194026681 + ], + [ + -3413475.316508251, + -1704671.8728591497, + 5094057.194026681 + ], + [ + -3413470.340249567, + -1704681.8374009654, + 5094057.194026681 + ], + [ + -3413465.358960294, + -1704691.8119432305, + 5094057.194026681 + ], + [ + -3413460.37765573, + -1704701.7864431958, + 5094057.194026681 + ], + [ + -3413455.4063112848, + -1704711.7409264494, + 5094057.194026681 + ], + [ + -3413450.4299362227, + -1704721.7054101357, + 5094057.194026681 + ], + [ + -3413445.4535320713, + -1704731.6698792966, + 5094057.194026681 + ] + ], + "plane_N": [ + -0.9633203118488703, + -0.08160610934519683, + -0.2556451049735453 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 6.724691477696396e-05, + 1.5706629728077543e-05, + 0.00010637754358927273, + 0.0002362632170843941, + 0.00027184572472634675, + 0.00034066933402676907, + 0.0005857413387279998, + 1.4225497938347832e-05, + 0.0003240762673397417, + 0.0002633833955488076, + 0.0005469632219021634, + 0.00042211398237902714 + ], + "ang_res_std": 0.0003236383693663661 + } + }, + { + "ObservedPoints.UK00CD": { + "meas1": [ + 3.2350197840765165, + 3.2477072959811264, + 3.2552292551574604, + 3.2758807935362744, + 3.293926737375255, + 3.3099222963095567, + 3.3239878762449746, + 3.3380897824049134, + 3.357389348527242, + 3.371805691892109, + 3.3873157784796977, + 3.4044770059058163, + 3.4212650205456225, + 3.4374557844393463 + ], + "meas2": [ + 1.2609999783504144, + 1.2612395779929737, + 1.2595198256953215, + 1.260472338963183, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814451, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153048, + 1.2547119136331997, + 1.2533115917530044, + 1.253010663890778 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 6.187651883518428e-05, + 0.0399378765188352, + 0.0798018765188349, + 0.11962187651883464, + 0.15939787651883444, + 0.1992378765188352, + 0.2390978765188352, + 0.27892987651883483, + 0.3187098765188352, + 0.3585698765188352, + 0.3983738765188345, + 0.4381978765188348, + 0.4779978765188344, + 0.5178458765188345 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 14, + "JD_data": [ + 2461151.500205559, + 2461151.5002060207, + 2461151.500206482, + 2461151.500206943, + 2461151.500207403, + 2461151.500207864, + 2461151.5002083257, + 2461151.5002087867, + 2461151.500209247, + 2461151.5002097087, + 2461151.5002101692, + 2461151.5002106302, + 2461151.500211091, + 2461151.500211552 + ], + "lat": 0.9302909996683025, + "lon": -0.039105358527994384, + "ele": 80.0, + "station_id": "UK00CD", + "azim_data": [ + 5.966419815139067, + 5.97542519016121, + 5.977778599822299, + 5.993954146606859, + 6.005275633071222, + 6.017243491584776, + 6.02792804365347, + 6.0375395619757395, + 6.051683132695393, + 6.063393040906969, + 6.074541567562709, + 6.0886289147110215, + 6.101951058174785, + 6.116263187282847 + ], + "elev_data": [ + 1.2028290600570986, + 1.2049672270465874, + 1.2077118383916157, + 1.2105467452617509, + 1.214565742676128, + 1.2171622436903107, + 1.2193966508469822, + 1.2223914732590124, + 1.226048237869094, + 1.2281100846973485, + 1.2315217824941145, + 1.2341667869259583, + 1.2373483805420769, + 1.2393083048928824 + ], + "ra_data": [ + 3.2350197840765156, + 3.2477072959811255, + 3.2552292551574595, + 3.275880793536273, + 3.293926737375254, + 3.3099222963095567, + 3.323987876244974, + 3.3380897824049134, + 3.3573893485272417, + 3.371805691892108, + 3.387315778479697, + 3.404477005905816, + 3.421265020545622, + 3.4374557844393454 + ], + "dec_data": [ + 1.2609999783504147, + 1.2612395779929737, + 1.2595198256953215, + 1.2604723389631833, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814453, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153052, + 1.2547119136331992, + 1.2533115917530042, + 1.2530106638907785 + ], + "magnitudes": [ + 4.86, + 4.32, + 5.89, + 4.98, + 2.84, + 2.54, + 2.93, + 2.47, + 2.12, + 2.31, + 2.72, + 2.1, + 2.7, + 2.14 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK00CD_20260421-000017.700500_9470", + "comment": "{\"ff_name\": \"FF_UK00CD_20260421_000013_479_0364800.fits\"}", + "incident_angle": 0.9673542667254906, + "weight": 0.6779664749397492, + "h_residuals": [ + 10.60965356642866, + -24.36076247965052, + 105.75947002249839, + -7.220710828385594, + 52.4106292421939, + 13.980469426264557, + -21.11566161888541, + 16.385999544642043, + 31.159896386556547, + -15.566908035957312, + 42.18031417134474, + 11.207776909509969, + 34.794840093942796, + -31.51281526197431 + ], + "h_res_rms": 38.65757618379624, + "v_residuals": [ + 3.2446947715231613, + -7.4621198683776795, + 32.17069118416699, + -2.20894625977602, + 15.980407194763492, + 4.270405033174591, + -6.460331591938528, + 5.002493265014071, + 9.503554253729916, + -4.7583795692783655, + 12.852019409716682, + 3.4197354351588864, + 10.601440013163899, + -9.632696675175211 + ], + "v_res_rms": 11.775952423901554, + "velocities": [ + 0.0, + 10471.680103624529, + 6537.224031186331, + 16633.282812297697, + 14826.52683173458, + 12873.383577289029, + 11286.270727626033, + 11554.888061040885, + 15564.781179446425, + 11390.115494189105, + 12773.925244016458, + 13582.991536140642, + 13615.88234822533, + 12542.882660933778 + ], + "velocities_prev_point": [ + 10743.85866816658, + 10743.85866816658, + 10743.85866816658, + 10743.85866816658, + 12007.055445941307, + 12492.235701727113, + 12536.279099510239, + 12472.845218100547, + 12649.080739643105, + 12670.109215040093, + 12686.294204401467, + 12730.115433503099, + 12785.779374174901, + 12813.830897857222 + ], + "length": [ + 0.0, + 417.56871581213187, + 678.1686145913418, + 1340.5059361770318, + 1930.2458674361035, + 2443.121469155308, + 2892.992220358482, + 3353.246521605858, + 3972.4135169242427, + 4426.4235205226205, + 4934.8768409354425, + 5475.8058958707115, + 6017.718013330074, + 6517.526801602965 + ], + "state_vect_dist": [ + 37.76837659225623, + 379.80033921987564, + 640.4002379990856, + 1302.7375595847757, + 1892.4774908438474, + 2405.3530925630516, + 2855.223843766226, + 3315.4781450136015, + 3934.6451403319866, + 4388.655143930364, + 4897.108464343187, + 5438.037519278456, + 5979.949636737819, + 6479.758425010708 + ], + "lag": [ + 281.3381680637624, + 106.26937614174375, + -150.0758670031762, + -4.113107714768603, + 69.82284087386734, + 66.06452591993548, + -0.9579939258560444, + -57.2338676012655, + 46.077274171508634, + -16.805993279080212, + -24.51975166286138, + -0.017129900050349534, + 25.779779637870888, + 8.85090948729794 + ], + "lag_line": [ + 12015.663931514431, + -83.75025379184109 + ], + "v_init": 12015.663931514431, + "v_init_stddev": null, + "jacchia_fit": [ + 0.0, + 0.0 + ], + "model_ra": [ + 3.2350317233883077, + 3.2476714150887465, + 3.255438140444483, + 3.2758702514549296, + 3.2941153488378436, + 3.3099924216832672, + 3.323910252516376, + 3.3381929059797426, + 3.3575930020386857, + 3.371733421626219, + 3.3876369233400556, + 3.404593711003503, + 3.4215849629232125, + 3.437217005959056 + ], + "model_dec": [ + 1.2611273499274973, + 1.2609464139140631, + 1.260793448991946, + 1.2603850642422985, + 1.259911335665489, + 1.2594122969591925, + 1.2589067455139373, + 1.2583285427696425, + 1.25746144668329, + 1.2567431069600323, + 1.2558671504145162, + 1.254848700705218, + 1.253736238689485, + 1.252625998752515 + ], + "model_azim": [ + 5.966643957880864, + 5.974911280194201, + 5.9800178789208704, + 5.993802064108692, + 6.006378939289549, + 6.017536643092251, + 6.0274870372718645, + 6.03788102220137, + 6.052329074285987, + 6.0630724131073555, + 6.075406678946376, + 6.088856757848179, + 6.102652853784429, + 6.115634949478984 + ], + "model_elev": [ + 1.202730398539852, + 1.2051959385233475, + 1.2067159430592809, + 1.210615868504016, + 1.2140581450105756, + 1.2170252460038622, + 1.2196057067116586, + 1.2222277443684135, + 1.225732770549091, + 1.2282693521279788, + 1.2310859268705145, + 1.234049575685928, + 1.2369803907988324, + 1.239645469881098 + ], + "model_fit1": [ + 5.966643957880864, + 5.974911280194201, + 5.9800178789208704, + 5.993802064108692, + 6.006378939289549, + 6.017536643092251, + 6.0274870372718645, + 6.03788102220137, + 6.052329074285987, + 6.0630724131073555, + 6.075406678946376, + 6.088856757848179, + 6.102652853784429, + 6.115634949478984 + ], + "model_fit2": [ + 1.202730398539852, + 1.2051959385233475, + 1.2067159430592809, + 1.210615868504016, + 1.2140581450105756, + 1.2170252460038622, + 1.2196057067116586, + 1.2222277443684135, + 1.225732770549091, + 1.2282693521279788, + 1.2310859268705145, + 1.234049575685928, + 1.2369803907988324, + 1.239645469881098 + ], + "meas_eci": [ + [ + -0.3035351281043153, + -0.02844121514996133, + 0.9523956758025997 + ], + [ + -0.3029229405820039, + -0.03226575808186291, + 0.9524686939341127 + ], + [ + -0.3042986687014469, + -0.034729084187786134, + 0.9519433864143417 + ], + [ + -0.3026179106160443, + -0.04088405047247195, + 0.9522346846189463 + ], + [ + -0.30295616797952424, + -0.046510880186298745, + 0.9518688451186214 + ], + [ + -0.30220529917399286, + -0.05135608475551451, + 0.9518584504587548 + ], + [ + -0.30152721261512533, + -0.05561523142857777, + 0.9518341694253778 + ], + [ + -0.30167832324719224, + -0.06005383655938584, + 0.9515165400544865 + ], + [ + -0.30143648571506654, + -0.0660779059164778, + 0.9511938579645285 + ], + [ + -0.3005904430157494, + -0.07044880838966484, + 0.9511479122429234 + ], + [ + -0.30091929891322594, + -0.07546789342433696, + 0.9506588097754449 + ], + [ + -0.3001680129514678, + -0.08077893212098153, + 0.9504598508754388 + ], + [ + -0.3000486040229262, + -0.08617384954180235, + 0.9500236328002793 + ], + [ + -0.2988875713280024, + -0.09110370995523792, + 0.9499296467307672 + ] + ], + "meas_eci_los": [ + [ + -0.30353512752019685, + -0.028441221383888878, + 0.9523956758025997 + ], + [ + -0.3029228461071481, + -0.03226664503462707, + 0.9524686939341127 + ], + [ + -0.30429846603725963, + -0.03473085989960501, + 0.9519433864143417 + ], + [ + -0.3026175532804857, + -0.040886695332201205, + 0.9522346846189463 + ], + [ + -0.30295562663884373, + -0.04651440615989257, + 0.9518688451186214 + ], + [ + -0.30220455226634335, + -0.05136047974617341, + 0.9518584504587548 + ], + [ + -0.301526242054227, + -0.05562049322827857, + 0.9518341694253778 + ], + [ + -0.30167710078673465, + -0.06005997721985251, + 0.9515165400544865 + ], + [ + -0.3014349488853584, + -0.06608491628985995, + 0.9511938579645285 + ], + [ + -0.30058859968688656, + -0.07045667302821534, + 0.9511479122429234 + ], + [ + -0.30091710526470994, + -0.07547663979962246, + 0.9506588097754449 + ], + [ + -0.30016543028477377, + -0.08078852849133539, + 0.9504598508754388 + ], + [ + -0.30004559881096554, + -0.08618431269743403, + 0.9500236328002793 + ], + [ + -0.2988841295616315, + -0.09111500073064646, + 0.9499296467307672 + ] + ], + "model_eci": [ + [ + -3432466.7540115756, + -1731741.8935464248, + 5173716.011629781 + ], + [ + -3432372.547769191, + -1732079.5683393998, + 5173489.145111228 + ], + [ + -3432313.743999133, + -1732290.3063399354, + 5173347.5431581 + ], + [ + -3432164.301102949, + -1732825.91596933, + 5172987.668751478 + ], + [ + -3432031.227056555, + -1733302.8168760038, + 5172667.221548597 + ], + [ + -3431915.4844218343, + -1733717.5583187286, + 5172388.519694615 + ], + [ + -3431813.9466015864, + -1734081.3479430394, + 5172144.033616263 + ], + [ + -3431710.0576202665, + -1734453.532469572, + 5171893.892688835 + ], + [ + -3431570.3102787603, + -1734954.2219146248, + 5171557.403865901 + ], + [ + -3431467.812772388, + -1735321.3520204683, + 5171310.628589792 + ], + [ + -3431353.024745032, + -1735732.5059074648, + 5171034.262150875 + ], + [ + -3431230.901985564, + -1736169.9185074358, + 5170740.239346225 + ], + [ + -3431108.549160772, + -1736608.122839488, + 5170445.669753482 + ], + [ + -3430995.6861842214, + -1737012.276212167, + 5170173.961306149 + ] + ], + "meas_lat": [ + 0.9348979840563066, + 0.9348736638383488, + 0.9348366981997408, + 0.9348063479155164, + 0.9347563423139719, + 0.9347257901580164, + 0.9346991840939688, + 0.9346614322637298, + 0.9346157521163507, + 0.9345905006711562, + 0.934546433761751, + 0.9345127904652669, + 0.9344712488826498, + 0.9344455500907388 + ], + "meas_lon": [ + -0.04164301451607974, + -0.04155291066409006, + -0.041513252115616614, + -0.04136276158768624, + -0.04124541609544114, + -0.041134549051738925, + -0.041037524018862015, + -0.040945674933504726, + -0.040817480073928235, + -0.040718389405427956, + -0.040618299375673295, + -0.04050220635063718, + -0.040391477328569035, + -0.04028058930887304 + ], + "meas_ht": [ + 81396.9675099336, + 81244.77789502541, + 81193.34068859647, + 80935.62631653334, + 80752.07851757792, + 80566.58131972801, + 80403.44378474863, + 80257.65942465847, + 80051.33834901436, + 79883.51877213769, + 79727.19354210321, + 79534.42486416693, + 79356.9731703905, + 79168.05194906335 + ], + "meas_range": [ + 87069.72999388228, + 86836.55913972364, + 86692.05673251014, + 86325.53975271186, + 86002.07075931023, + 85723.10812734772, + 85480.18898973276, + 85233.27293127988, + 84903.47168914345, + 84663.79807745767, + 84397.2537452506, + 84115.98487302613, + 83836.60845417317, + 83581.14830424251 + ], + "model_lat": [ + 0.934899505502217, + 0.9348701702014254, + 0.9348518594457105, + 0.9348053124719798, + 0.9347638562520294, + 0.9347277946223416, + 0.934696156319921, + 0.934663781456173, + 0.9346202190594269, + 0.9345882687471724, + 0.9345524799775028, + 0.9345143970336911, + 0.9344762361830826, + 0.9344410323178205 + ], + "model_lon": [ + -0.04164193466936269, + -0.04155538966397436, + -0.041502471430448676, + -0.04136349692543704, + -0.04124007338001663, + -0.04113312415058746, + -0.04103967585574465, + -0.04094400396225524, + -0.04081430089807224, + -0.040719977257233045, + -0.04061399294714445, + -0.0405010622126221, + -0.04038792316631062, + -0.0402838067317805 + ], + "model_ht": [ + 81393.91662581416, + 81251.79438980817, + 81163.08976953034, + 80937.70349190966, + 80737.05100419743, + 80562.56553501167, + 80409.51917280334, + 80252.95499555573, + 80042.40065984003, + 79887.99406011793, + 79715.1061188881, + 79531.20864436727, + 79347.00196171085, + 79177.11246162215 + ], + "model_range": [ + 87069.7307007453, + 86836.56287739017, + 86692.12721183259, + 86325.54008297449, + 86002.08821370563, + 85723.10937370706, + 85480.19184196735, + 85233.27465311192, + 84903.4779387753, + 84663.79964238926, + 84397.26526402768, + 84115.98568912731, + 83836.61634464488, + 83581.15480030599 + ], + "rbeg_lat": 0.934899505502217, + "rbeg_lon": -0.04164193466936269, + "rbeg_ele": 81393.91662581416, + "rbeg_jd": 2461151.500205559, + "rend_lat": 0.9344410323178205, + "rend_lon": -0.0402838067317805, + "rend_ele": 79177.11246162215, + "rend_jd": 2461151.500211552, + "htmin_lat": 0.9344410323178205, + "htmin_lon": -0.0402838067317805, + "htmin_ele": 79177.11246162215, + "htmin_jd": 2461151.500211552, + "absolute_magnitudes": [ + 5.160663993871118, + 4.6264868745522065, + 6.2001017019905555, + 5.2993034793540605, + 3.1674550176961667, + 2.8745104224398865, + 3.270672559130019, + 2.816954128451293, + 2.47537259597214, + 2.6715112232095644, + 3.0883581282452037, + 2.4756073077614653, + 3.082831290906207, + 2.529458163572881 + ], + "ra_data_los": [ + 3.235019804614262, + 3.247710223963076, + 3.2552350905836143, + 3.275889533472668, + 3.293938375945835, + 3.3099368393903763, + 3.324005326769998, + 3.3381101374401774, + 3.3574126051387827, + 3.371831855939726, + 3.387344844103591, + 3.404508976039951, + 3.4212998922560343, + 3.4374935606518706 + ], + "dec_data_los": [ + 1.2609999783504147, + 1.2612395779929737, + 1.2595198256953215, + 1.2604723389631833, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814453, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153052, + 1.2547119136331992, + 1.2533115917530042, + 1.2530106638907785 + ], + "x_eci": [ + -0.3035351281043153, + -0.3029229405820039, + -0.3042986687014469, + -0.3026179106160443, + -0.30295616797952424, + -0.30220529917399286, + -0.30152721261512533, + -0.30167832324719224, + -0.30143648571506654, + -0.3005904430157494, + -0.30091929891322594, + -0.3001680129514678, + -0.3000486040229262, + -0.2988875713280024 + ], + "y_eci": [ + -0.02844121514996133, + -0.03226575808186291, + -0.034729084187786134, + -0.04088405047247195, + -0.046510880186298745, + -0.05135608475551451, + -0.05561523142857777, + -0.06005383655938584, + -0.0660779059164778, + -0.07044880838966484, + -0.07546789342433696, + -0.08077893212098153, + -0.08617384954180235, + -0.09110370995523792 + ], + "z_eci": [ + 0.9523956758025997, + 0.9524686939341127, + 0.9519433864143417, + 0.9522346846189463, + 0.9518688451186214, + 0.9518584504587548, + 0.9518341694253778, + 0.9515165400544865, + 0.9511938579645285, + 0.9511479122429234, + 0.9506588097754449, + 0.9504598508754388, + 0.9500236328002793, + 0.9499296467307672 + ], + "x_eci_los": [ + -0.30353512752019685, + -0.3029228461071481, + -0.30429846603725963, + -0.3026175532804857, + -0.30295562663884373, + -0.30220455226634335, + -0.301526242054227, + -0.30167710078673465, + -0.3014349488853584, + -0.30058859968688656, + -0.30091710526470994, + -0.30016543028477377, + -0.30004559881096554, + -0.2988841295616315 + ], + "y_eci_los": [ + -0.028441221383888878, + -0.03226664503462707, + -0.03473085989960501, + -0.040886695332201205, + -0.04651440615989257, + -0.05136047974617341, + -0.05562049322827857, + -0.06005997721985251, + -0.06608491628985995, + -0.07045667302821534, + -0.07547663979962246, + -0.08078852849133539, + -0.08618431269743403, + -0.09111500073064646 + ], + "z_eci_los": [ + 0.9523956758025997, + 0.9524686939341127, + 0.9519433864143417, + 0.9522346846189463, + 0.9518688451186214, + 0.9518584504587548, + 0.9518341694253778, + 0.9515165400544865, + 0.9511938579645285, + 0.9511479122429234, + 0.9506588097754449, + 0.9504598508754388, + 0.9500236328002793, + 0.9499296467307672 + ], + "x_stat": -3406048.6136386427, + "y_stat": -1729266.1246223755, + "z_stat": 5090787.796276712, + "stat_eci": [ + -3406048.6136386427, + -1729266.1246223755, + 5090787.796276712 + ], + "stat_eci_los": [ + [ + -3406048.5781234126, + -1729266.1945749386, + 5090787.796276712 + ], + [ + -3406043.5503640417, + -1729276.0974638269, + 5090787.796276712 + ], + [ + -3406038.5225758795, + -1729286.0003380955, + 5090787.796276712 + ], + [ + -3406033.4998326153, + -1729295.8932045705, + 5090787.796276712 + ], + [ + -3406028.4872221127, + -1729305.766042447, + 5090787.796276712 + ], + [ + -3406023.464421443, + -1729315.6588797665, + 5090787.796276712 + ], + [ + -3406018.4365182575, + -1729325.5616956379, + 5090787.796276712 + ], + [ + -3406013.413660088, + -1729335.454503771, + 5090787.796276712 + ], + [ + -3406008.39584702, + -1729345.3373042115, + 5090787.796276712 + ], + [ + -3406003.3678716016, + -1729355.24004859, + 5090787.796276712 + ], + [ + -3405998.350001125, + -1729365.1228198803, + 5090787.796276712 + ], + [ + -3405993.3270280496, + -1729375.0155696734, + 5090787.796276712 + ], + [ + -3405988.309100192, + -1729384.8983118308, + 5090787.796276712 + ], + [ + -3405983.286083734, + -1729394.791004777, + 5090787.796276712 + ] + ], + "plane_N": [ + -0.9508405323585105, + -0.0543339152478599, + -0.30487720098102244 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 0.00012742337477004607, + 0.0002934025890114486, + 0.001275134678871608, + 8.747317337504222e-05, + 0.000637107786152886, + 0.00017052484520883212, + 0.00025833001748669024, + 0.00020100450053262754, + 0.0003836891026201303, + 0.00019227104764286688, + 0.0005224611490777694, + 0.00013929892102162514, + 0.00043386000256940454, + 0.00039426294165447807 + ], + "ang_res_std": 0.0004704327154024583 + } + } + ], + "los_mini_status": true, + "t_ref_station": 2, + "time_diffs_final": [ + 0.0, + -0.0019011817811903332, + -0.00014963188985710543 + ], + "intersection_list": [ + { + "PlaneIntersection.UK002Z_UK00AS": { + "conv_angle": 1.3977829287229966, + "radiant_eci": [ + -0.2119721838741445, + 0.805179044558886, + 0.5538542222163385 + ], + "radiant_eq": [ + 1.8282164010696587, + 0.5869862134998625 + ], + "w1": [ + -0.9772483431518503, + -0.17887979587700067, + -0.1139635662567909 + ], + "w2": [ + -0.16103171664643293, + -0.5877633547814152, + 0.792844893412324 + ], + "rcpa_stat1": [ + -73631.49434216082, + -13477.82963291005, + -8586.668622006024 + ], + "rcpa_stat2": [ + -13242.80545538603, + -48336.04163995439, + 65201.38329524699 + ], + "cpa_eci": [ + -3403324.3337847604, + -1842095.9935247626, + 5097804.319209092 + ], + "weight": 0.0024214538252984794 + } + }, + { + "PlaneIntersection.UK002Z_UK00CD": { + "conv_angle": 1.3408619526894816, + "radiant_eci": [ + -0.2224668519112711, + 0.8032092732613852, + 0.5525969264731806 + ], + "radiant_eq": [ + 1.8409953958825356, + 0.5854769115887319 + ], + "w1": [ + -0.9749128323753357, + -0.18752636752381052, + -0.11991176236393772 + ], + "w2": [ + -0.2150761118164841, + -0.5932692097626505, + 0.7757408786917855 + ], + "rcpa_stat1": [ + -72116.41498877588, + -13871.731802662609, + -8870.132928298488 + ], + "rcpa_stat2": [ + -15374.393477834303, + -42408.9602147798, + 55452.67396327532 + ], + "cpa_eci": [ + -3401809.2544313753, + -1842489.8956945152, + 5097520.854902799 + ], + "weight": 0.002371116245910391 + } + }, + { + "PlaneIntersection.UK00AS_UK00CD": { + "conv_angle": 0.0578872564894713, + "radiant_eci": [ + -0.17933529144848642, + 0.8982410335263312, + 0.4012504192279862 + ], + "radiant_eq": [ + 1.767856898383979, + 0.4128815721461222 + ], + "w1": [ + -0.19672893468820898, + -0.4323680153644721, + 0.8799747868809822 + ], + "w2": [ + -0.25216678471696985, + -0.43620698939268593, + 0.8637912798184956 + ], + "rcpa_stat1": [ + -16494.934626699258, + -36252.32942686249, + 73782.36763061871 + ], + "rcpa_stat2": [ + -19567.098351078548, + -33847.856181592375, + 67026.62662722637 + ], + "cpa_eci": [ + -3430096.3582245708, + -1740671.6590927788, + 5167839.561657299 + ], + "weight": 1.3053179448789485e-05 + } + } + ], + "rbeg_lat": 0.934899505502217, + "rbeg_lon": -0.04164193466936269, + "rbeg_ele": 81393.91662581416, + "rbeg_ele_wgs84": 81445.53494089004, + "rbeg_jd": 2461151.500205559, + "rend_lat": 0.9337921890458436, + "rend_lon": -0.038372180120937446, + "rend_ele": 76055.10224458351, + "rend_ele_wgs84": 76106.4147786973, + "rend_jd": 2461151.5002204534, + "htmin_lat": 0.9337921890458436, + "htmin_lon": -0.038372180120937446, + "htmin_ele": 76055.10224458351, + "htmin_ele_wgs84": 76106.4147786973, + "htmin_jd": 2461151.5002204534, + "state_vect": [ + -3432369.342207372, + -1731768.1585065187, + 5173694.939644102 + ], + "incident_angles": [ + 0.5027492433632643, + 1.2638605479641993, + 0.9659504756451831 + ], + "state_vect_mini": [ + -3432458.2336270753, + -1731772.435690051, + 5173695.492595176 + ], + "radiant_eci_mini": [ + -0.22559573032168215, + 0.8086697491907738, + 0.5432861154165067 + ], + "radiant_eq_mini": [ + 1.842850952886513, + 0.5743463248927236 + ], + "v_init": 12967.718791996132, + "v0z": -4413.997141557832, + "v_avg": 12312.85904311762, + "timing_minimization_successful": true, + "velocity_fit": [ + 12967.718791996132, + -244.37218876758848 + ], + "jacchia_fit": [ + 43.65312744412464, + 2.1924329546040098 + ], + "timing_res": 7.041064582452035e-05, + "timing_stddev": 0.008391105161092927, + "state_vect_avg": [ + -3430549.6763656787, + -1738607.335042225, + 5169100.684310519 + ], + "jd_avg": 2461151.5002130065, + "orbit": { + "Orbit": { + "ra": 1.842850952886513, + "dec": 0.5743463248927236, + "azimuth_apparent": 5.215879750489071, + "elevation_apparent": 0.3473192026194278, + "v_avg": 12312.85904311762, + "v_init": 12967.718791996132, + "v_init_stddev": 10.13578330407507, + "ra_norot": 1.837726586059179, + "dec_norot": 0.5860709340821146, + "azimuth_apparent_norot": 5.227200613194143, + "elevation_apparent_norot": 0.3538606337963674, + "v_avg_norot": 12082.953555046177, + "v_init_norot": 12737.81330392469, + "jd_ref": 2461151.500205559, + "jd_dyn": 2461151.5010063, + "lst_ref": 3.6088563643950953, + "lon_ref": -0.0416338437576483, + "lat_ref": 0.9348968523981452, + "ht_ref": 81381.06170645228, + "ht_ref_wgs84": 81432.6793071907, + "lat_geocentric": 0.9317248461260144, + "zc": 1.2250089673888134, + "zg": 1.6682930856125058, + "v_inf": 12967.718791996132, + "v_g": 6669.855942091, + "ra_g": 1.560157053440776, + "dec_g": 0.21033070922631758, + "L_g": 1.560182664795458, + "B_g": -0.19871170575289931, + "meteor_pos": [ + -129457249.42153406, + -76394902.82501395, + 10345.503483171698 + ], + "v_h": 35477.25261044726, + "v_h_x": 14.598935218756163, + "v_h_y": -32.30740600750524, + "v_h_z": 1.3183551608964306, + "L_h": 1.9952090815491528, + "B_h": -0.037169134413427715, + "la_sun": 0.5331351764388832, + "a": 1.749324945375272, + "e": 0.4367632022782812, + "i": 0.037382200258429005, + "peri": 5.925529697530141, + "node": 3.6728731864642414, + "pi": 3.3152175768147956, + "b": -0.0130840548179203, + "q": 0.9852841804078888, + "Q": 2.513365710342655, + "true_anomaly": 0.35949713851885046, + "eccentric_anomaly": 0.22656761811223144, + "mean_anomaly": 0.12845566895513721, + "last_perihelion": "2026-04-03 17:22:02.404589", + "n": 0.007434915611796666, + "Tj": 4.017379995926152, + "T": 2.313696397548099 + } + }, + "uncertainties": { + "MCUncertainties": { + "ci": 95, + "mc_traj_list": null, + "state_vect_mini": [ + 23.302927645551236, + 37.2070223921984, + 47.8951574781733 + ], + "state_vect_mini_ci": [ + [ + -3432495.617382003, + -3432423.837728241 + ], + [ + -1731807.3913284293, + -1731696.1065867343 + ], + [ + 5173640.759083595, + 5173792.756573728 + ] + ], + "x": 23.302927645551236, + "x_ci": [ + -3432495.617382003, + -3432423.837728241 + ], + "y": 37.2070223921984, + "y_ci": [ + -1731807.3913284293, + -1731696.1065867343 + ], + "z": 47.8951574781733, + "z_ci": [ + 5173640.759083595, + 5173792.756573728 + ], + "vx": 13.858206008974927, + "vx_ci": [ + -2938.2878636842593, + -2861.212929938162 + ], + "vy": 59.54333260289223, + "vy_ci": [ + 10483.746446609672, + 10492.11339262698 + ], + "vz": 57.58213998290366, + "vz_ci": [ + 6945.863075051403, + 7122.386599949237 + ], + "radiant_eci_mini": [ + 0.0016920905173864757, + 0.002356945317379389, + 0.002939081393705436 + ], + "radiant_eci_mini_ci": [ + [ + -0.22617399930035348, + -0.2214846318885415 + ], + [ + 0.8059754297844866, + 0.8128571058966497 + ], + [ + 0.5381818809865024, + 0.5473502583390504 + ] + ], + "rbeg_lon": 2.2088929690296427e-06, + "rbeg_lon_ci": [ + 6.2415409901294865, + 6.241546962406092 + ], + "rbeg_lon_m": 8.38540985943337, + "rbeg_lat": 2.638433710383294e-06, + "rbeg_lat_ci": [ + 0.9348949134986161, + 0.9349036888013773 + ], + "rbeg_lat_m": 16.864870452262917, + "rbeg_ele": 45.558677417463386, + "rbeg_ele_ci": [ + 81325.7017569644, + 81464.2043463111 + ], + "rbeg_ele_wgs84": 45.5588599685256, + "rbeg_ele_wgs84_ci": [ + 81377.31980654877, + 81515.822891308 + ], + "rend_lon": 3.407449543509572e-06, + "rend_lon_ci": [ + 6.244800260516159, + 6.244811360843754 + ], + "rend_lon_m": 12.954725654542509, + "rend_lat": 1.0453520900156793e-06, + "rend_lat_ci": [ + 0.933790446617204, + 0.9337939283069673 + ], + "rend_lat_m": 6.681867643440438, + "rend_ele": 32.15807998869947, + "rend_ele_ci": [ + 76036.61348161136, + 76133.00603886494 + ], + "rend_ele_wgs84": 32.15826496087123, + "rend_ele_wgs84_ci": [ + 76087.92684692098, + 76184.31958180107 + ], + "htmin_lon": 3.407449543509572e-06, + "htmin_lon_ci": [ + 6.244800260516159, + 6.244811360843754 + ], + "htmin_lon_m": 12.954725654542509, + "htmin_lat": 1.0453520900156793e-06, + "htmin_lat_ci": [ + 0.933790446617204, + 0.9337939283069673 + ], + "htmin_lat_m": 6.681867643440438, + "htmin_ele": 32.15807998869947, + "htmin_ele_ci": [ + 76036.61348161136, + 76133.00603886494 + ], + "htmin_ele_wgs84": 32.15826496087123, + "htmin_ele_wgs84_ci": [ + 76087.92684692098, + 76184.31958180107 + ], + "ra": 0.002611503709103583, + "ra_ci": [ + 1.8370325403202352, + 1.8440354813447923 + ], + "dec": 0.003499436816389006, + "dec_ci": [ + 0.5682786814957027, + 0.579194832887812 + ], + "azimuth_apparent": 0.0011488927073448102, + "azimuth_apparent_ci": [ + 5.2149278320957935, + 5.218617334102857 + ], + "elevation_apparent": 0.003985943562293819, + "elevation_apparent_ci": [ + 0.3401376381944182, + 0.35146749322636645 + ], + "v_avg": 25.363986414427266, + "v_avg_ci": [ + 12263.698119250277, + 12331.272064862434 + ], + "v_init": 35.83555406641995, + "v_init_ci": [ + 12906.156161188343, + 13012.85741906931 + ], + "lon_ref": 9.746851701222101e-06, + "lon_ref_ci": [ + 6.241530852167341, + 6.241562416968685 + ], + "lat_ref": 4.037368389248857e-06, + "lat_ref_ci": [ + 0.9348930977264116, + 0.9349048456367035 + ], + "ht_ref": 46.23031409446649, + "ht_ref_ci": [ + 81327.7139370313, + 81465.6113865276 + ], + "lat_geocentric": 4.058469875306498e-06, + "lat_geocentric_ci": [ + 0.9317210791689499, + 0.9317328894806066 + ], + "zc": 0.003984415255032236, + "zc_ci": [ + 1.2208682718192776, + 1.2321888843316788 + ], + "zg": 0.010437731100424239, + "zg_ci": [ + 1.6565960535519642, + 1.6867956381526732 + ], + "v_inf": 35.83555406641995, + "v_inf_ci": [ + 12906.156161188343, + 13012.85741906931 + ], + "v_g": 69.86456353616636, + "v_g_ci": [ + 6549.278440215807, + 6757.316401539255 + ], + "ra_g": 0.005218583891039907, + "ra_g_ci": [ + 1.550331619290657, + 1.5642537989451053 + ], + "dec_g": 0.009175375088677046, + "dec_g_ci": [ + 0.19423320526384985, + 0.22168953806563507 + ], + "L_g": 0.005261542789721588, + "L_g_ci": [ + 1.550262999253746, + 1.5642996777101477 + ], + "B_g": 0.009147790639302643, + "B_g_ci": [ + -0.2147525041002339, + -0.18736660983219178 + ], + "meteor_pos": [ + 0.023402287765977765, + 0.04500344767424416, + 0.04074976099956943 + ], + "meteor_pos_ci": [ + [ + -129457249.45909907, + -129457249.3870311 + ], + [ + -76394902.87402098, + -76394902.73235357 + ], + [ + 10345.446462113892, + 10345.578152204636 + ] + ], + "v_h": 85.48609930680463, + "v_h_ci": [ + 35328.09302105182, + 35577.45936832089 + ], + "v_h_x": 0.03321975068903373, + "v_h_x_ci": [ + 14.53657918185901, + 14.62520287469976 + ], + "v_h_y": 0.08104467976059955, + "v_h_y_ci": [ + -32.40782446857918, + -32.16672196346313 + ], + "v_h_z": 0.04568102257541548, + "v_h_z_ci": [ + 1.2602424833462527, + 1.3973518963978855 + ], + "L_h": 0.00021478363911639652, + "L_h_ci": [ + 1.9947075813557262, + 1.9953602674305384 + ], + "B_h": 0.0013798257879346354, + "B_h_ci": [ + -0.03956396089577628, + -0.035430681123816095 + ], + "la_sun": 0.0, + "la_sun_ci": [ + 0.5331351764388832, + 0.5331351764388832 + ], + "a": 0.020764918399534503, + "a_ci": [ + 1.7136415146239252, + 1.7742356355051931 + ], + "e": 0.00662985086508511, + "e_ci": [ + 0.4252246127134539, + 0.44463172993888717 + ], + "i": 0.0013875907380813083, + "i_ci": [ + 0.03563537240700222, + 0.03979064362004738 + ], + "peri": 0.0036974620554850617, + "peri_ci": [ + 5.918617880304961, + 5.928510238183485 + ], + "node": 6.736619195035191e-05, + "node_ci": [ + 3.6727828832139906, + 3.6729846226879324 + ], + "pi": 0.0036316227110077645, + "pi_ci": [ + 3.308403222800056, + 3.3181100489377644 + ], + "b": 0.0006164034149704704, + "b_ci": [ + -0.01417379179358153, + -0.01237568509476685 + ], + "q": 0.0001887248437724549, + "q_ci": [ + 0.9848735253457906, + 0.9853733247776466 + ], + "Q": 0.04135376621516677, + "Q_ci": [ + 2.4423263887544406, + 2.563117103921014 + ], + "true_anomaly": 0.003631669498448071, + "true_anomaly_ci": [ + 0.3566046072044634, + 0.3663115643921616 + ], + "eccentric_anomaly": 0.00415997564946805, + "eccentric_anomaly_ci": [ + 0.22255528058867302, + 0.23405984987107875 + ], + "mean_anomaly": 0.0038950653377452164, + "mean_anomaly_ci": [ + 0.12441501198245487, + 0.13542723129154152 + ], + "last_perihelion": 0.21162646186227413, + "last_perihelion_ci": [ + 2461133.838177647, + 2461134.4084006962 + ], + "n": 0.00013353213752772863, + "n_ci": [ + 0.00727888466701047, + 0.007668381602780202 + ], + "T": 0.04112839790029051, + "T_ci": [ + 2.243267062584157, + 2.3632930877217384 + ], + "Tj": 0.03300359258971792, + "Tj_ci": [ + 3.978562778709054, + 4.074873090067962 + ], + "ra_norot": 0.0026819047506464275, + "ra_norot_ci": [ + 1.831751910635052, + 1.8389425768418368 + ], + "dec_norot": 0.003535179504345732, + "dec_norot_ci": [ + 0.5799393752075872, + 0.590966744099576 + ], + "v_avg_norot": 25.73922847929732, + "v_avg_norot_ci": [ + 12033.412162374418, + 12102.036345823391 + ], + "v_init_norot": 36.25000387087065, + "v_init_norot_ci": [ + 12675.549576533595, + 12783.621700030268 + ], + "azimuth_apparent_norot": 0.0011626219913197264, + "azimuth_apparent_norot_ci": [ + 5.226249501183861, + 5.229971656717279 + ], + "elevation_apparent_norot": 0.004037622005774978, + "elevation_apparent_norot_ci": [ + 0.3465823432479639, + 0.35805015466321544 + ], + "ht_ref_wgs84": 46.23066689867216, + "ht_ref_wgs84_ci": [ + 81379.33149963012, + 81517.23000879795 + ] + } + }, + "orbit_cov": [ + [ + 4.839509762694922e-05, + 1.2839048270422571e-06, + 0.001529597070645031, + -2.8134845951441427e-05, + 0.0015235437188076953, + -0.0005796707377483675 + ], + [ + 1.2839048270422571e-06, + 3.92532878805147e-08, + 4.283334419962881e-05, + -7.366720739670027e-07, + 4.318110319692552e-05, + -1.5189142360366112e-05 + ], + [ + 0.001529597070645031, + 4.283334419962881e-05, + 0.04932644359606424, + -0.0008848174909047756, + 0.049355279485603636, + -0.018239993412701383 + ], + [ + -2.8134845951441427e-05, + -7.366720739670027e-07, + -0.0008848174909047754, + 1.640345974762046e-05, + -0.0008803958803679891, + 0.00033761695055935486 + ], + [ + 0.0015235437188076953, + 4.318110319692551e-05, + 0.04935527948560364, + -0.0008803958803679891, + 0.049434753433829214, + -0.018149168163731216 + ], + [ + -0.0005796707377483675, + -1.5189142360366112e-05, + -0.018239993412701383, + 0.00033761695055935486, + -0.018149168163731216, + 0.006957026666261073 + ] + ], + "state_vect_cov": [ + [ + 594.8872861480645, + -256.66487170381055, + -967.7624393923977, + 583.6215942851601, + 26.71726004970638, + -985.1200079805594 + ], + [ + -256.66487170381055, + 1522.6343263519718, + 718.6734107124669, + 359.1627911110367, + 46.990250811313, + -641.6507010433324 + ], + [ + -967.7624393923977, + 718.673410712467, + 2514.098708757615, + -931.2754297849413, + -11.982008124620437, + 2312.506952848055 + ], + [ + 583.6215942851601, + 359.1627911110367, + -931.2754297849413, + 923.3221697191502, + 62.170755663341666, + -1621.7192914483069 + ], + [ + 26.71726004970638, + 46.99025081131299, + -11.982008124620435, + 62.170755663341666, + 7.208948184864825, + -91.24287656294936 + ], + [ + -985.1200079805594, + -641.6507010433324, + 2312.5069528480544, + -1621.719291448307, + -91.24287656294938, + 3635.949624000401 + ] + ], + "phase_1_only": false, + "avg_radiant": [ + -0.21707459130810688, + 0.8045078859660982, + 0.5528523159278415 + ], + "radiant_eq": [ + 1.8343430182556113, + 0.5857833676735065 + ], + "best_conv_inter": { + "PlaneIntersection.UK002Z_UK00AS": { + "conv_angle": 1.3977829287229966, + "radiant_eci": [ + -0.2119721838741445, + 0.805179044558886, + 0.5538542222163385 + ], + "radiant_eq": [ + 1.8282164010696587, + 0.5869862134998625 + ], + "w1": [ + -0.9772483431518503, + -0.17887979587700067, + -0.1139635662567909 + ], + "w2": [ + -0.16103171664643293, + -0.5877633547814152, + 0.792844893412324 + ], + "rcpa_stat1": [ + -73631.49434216082, + -13477.82963291005, + -8586.668622006024 + ], + "rcpa_stat2": [ + -13242.80545538603, + -48336.04163995439, + 65201.38329524699 + ], + "cpa_eci": [ + -3403324.3337847604, + -1842095.9935247626, + 5097804.319209092 + ], + "weight": 0.0024214538252984794 + } + }, + "stations_time_dict": { + "OrderedDict": {} + }, + "stations_time_dict_copy": { + "OrderedDict": {} + }, + "v_init_stddev": 10.13578330407507, + "time_diffs": [ + 0.0, + 0.00026701713965955896, + 6.187651883518428e-05 + ], + "longname": "20260421_000017.759_UK", + "pre_mc_longname": "20260421_000017.759_UK", + "save_date": "2026-04-21 09:33:47.459946+00:00" +} \ No newline at end of file diff --git a/archive/samfunctions/searchArchive/requirements.txt b/archive/lambdas/searchArchive/requirements.txt similarity index 100% rename from archive/samfunctions/searchArchive/requirements.txt rename to archive/lambdas/searchArchive/requirements.txt diff --git a/archive/samfunctions/searchArchive/samconfig.toml b/archive/lambdas/searchArchive/samconfig.toml similarity index 100% rename from archive/samfunctions/searchArchive/samconfig.toml rename to archive/lambdas/searchArchive/samconfig.toml diff --git a/archive/samfunctions/searchArchive/searchArchive.py b/archive/lambdas/searchArchive/searchArchive.py similarity index 100% rename from archive/samfunctions/searchArchive/searchArchive.py rename to archive/lambdas/searchArchive/searchArchive.py diff --git a/archive/samfunctions/searchArchive/template.yml b/archive/lambdas/searchArchive/template.yml similarity index 100% rename from archive/samfunctions/searchArchive/template.yml rename to archive/lambdas/searchArchive/template.yml diff --git a/archive/samfunctions/searchArchive/testEvent.json b/archive/lambdas/searchArchive/testEvent.json similarity index 100% rename from archive/samfunctions/searchArchive/testEvent.json rename to archive/lambdas/searchArchive/testEvent.json diff --git a/archive/samfunctions/updateDnsRecords/lambda_function.py b/archive/lambdas/updateDnsRecords/lambda_function.py similarity index 100% rename from archive/samfunctions/updateDnsRecords/lambda_function.py rename to archive/lambdas/updateDnsRecords/lambda_function.py diff --git a/archive/samfunctions/updateDnsRecords/samconfig.toml b/archive/lambdas/updateDnsRecords/samconfig.toml similarity index 100% rename from archive/samfunctions/updateDnsRecords/samconfig.toml rename to archive/lambdas/updateDnsRecords/samconfig.toml diff --git a/archive/samfunctions/updateDnsRecords/template.yaml b/archive/lambdas/updateDnsRecords/template.yaml similarity index 100% rename from archive/samfunctions/updateDnsRecords/template.yaml rename to archive/lambdas/updateDnsRecords/template.yaml diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index 7676e445e..00d569938 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -16,8 +16,8 @@ alias stats='if [ "$DATADIR" == "" ] ; then echo select env first; else tail $DA alias matchstatus='if [ "$SRC" == "" ] ; then echo select env first; else grep "Running" $SRC/logs/matchJob.log && grep TRAJ $SRC/logs/matchJob.log | grep SOLVING && echo -n "Completed " && grep Observations: $SRC/logs/matchJob.log | wc -l && grep "nightlyJob" $SRC/logs/nightlyJob.log ; fi ' alias spacecalc='ls -1 | egrep -v "ukmon-shared" | while read i ; do \du -s $i ; done | sort -n' -alias startcalc='~/prod/utils/stopstart-calcengine.sh start' -alias stopcalc='~/prod/utils/stopstart-calcengine.sh stop' +alias startcalc='$SRC/utils/stopstart-calcengine.sh start' +alias stopcalc='$SRC/utils/stopstart-calcengine.sh stop' function dev { source ~/dev/config.ini >/dev/null @@ -37,9 +37,9 @@ function calcserver { ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) isrunning=$(echo $sts | cut -d " " -f 1) if [ $isrunning -ne 16 ] ; then - /home/ec2-user/prod/utils/stopstart-calcengine.sh start + $HOME/prod/utils/stopstart-calcengine.sh start echo "starting server on ${ipaddr}... waiting 10s..." sleep 10 fi - ssh -i ~/.ssh/markskey.pem ec2-user@$ipaddr + ssh -i ~/.ssh/markskey.pem $SERVERUSERID@$ipaddr } diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index b772d578b..79e18c870 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -22,8 +22,7 @@ export PATH=$PATH:/usr/local/geos/bin:/usr/local/proj4/bin if shopt -q login_shell ; then echo "" - echo "Type 'dev' to activate the dev environment" - echo "Type 'prd' to activate the dev environment" + echo "THIS IS THE PROD SERVER!!" echo "" echo " Some handy aliases that work in either environment are" echo " logs => go to the logs folder" @@ -34,8 +33,9 @@ if shopt -q login_shell ; then echo " tnj => tail the nightly job log" echo " spacecalc => display space usage in the current folder" echo "" - fi - export NVM_DIR="$HOME/.nvm" + prd +fi +export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion export PATH=$PATH:$(dirname $(which node)) diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index cd413ef7e..584a4d90b 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -1,4 +1,21 @@ -# how to move accounts to a new server +# Replacing the UKMON helper server +The ukmon helper server provides two services. +* camera authentication and key management +* batch processing + +# how to move batch processing +The best approach is to reinstall all the code using the ansible deployment scripts. +Both dev and prod envs should be created, though arguably the dev env should be on a separate server. +Additionally, WMPL will need to be reinstalled. +After doing this, the additional UKMON python requirements can be added. + +The contents of ~/prod/data should be replicated to the new server. +The contents of ~/keymgmt should be replicated to the new server. + +Finally, double check that all required SSM variables exist in the account holding the server. These +are used to build the config file, but originally were only in Mark McIntyre's account. + +## how to move accounts to a new server The basic process is to extract the user accounts from the current server along with group and password info, then import it back in on the new server. Its important to avoid accidentally overwriting system or otherwise-existing accounts on the new server. @@ -8,7 +25,7 @@ new accounts starting at 1000 and working both upwards and downwards! Steps in brief. NB must all be done as root, of course. -## On the old server +### On the old server ``` bash mkdir /root/move/ export UGIDLIMIT=500 @@ -24,7 +41,7 @@ scp /root/move/* newserver:/tmp ``` -## On the new server +### On the new server In summary: backup the existing files, remove any accounts from the .mig files that are already present in the target, then append the filtered data. @@ -33,7 +50,7 @@ When comparing groups, remember new ids will get added to the sftp group. ``` bash mkdir -p /root/move/bkp -mv /tmp/*.mig /tmp/arsftp.tar.gz /root/move +mv /tmp/*.mig /tmp/varsftp.tar.gz /root/move cp /etc/passwd /etc/group /etc/shadow /etc/gshadow /root/move/bkp cd / diff --git a/archive/static_content/js/searchdialog.js b/archive/static_content/js/searchdialog.js index 5b9e0ea52..d5716b036 100644 --- a/archive/static_content/js/searchdialog.js +++ b/archive/static_content/js/searchdialog.js @@ -70,6 +70,17 @@ form.addEventListener("submit", function (event) { op = op + "l:" + strstat + "_"; } //jQuery.support.cors = true; + if (d1 == "" ){ + console.log("missing start date"); + const stdt = new Date(Date.now()); + stdt.setHours(stdt.getHours() - 12); + d1 = stdt.toISOString(); + } + if (d2 == "" ){ + console.log("missing end date"); + const enddt = new Date(Date.now()); + d2 = enddt.toISOString(); + } payload = {"d1": d1, "d2": d2, "opts": op }; console.log(payload); document.getElementById("searchresults").innerHTML = "Searching...."; @@ -96,6 +107,9 @@ function myFunc(myObj) { txt = "" txt += ""; txt += ""; + if (myObj.length == 0){ + alert("No data available, check dates"); + } for (x in myObj) { console.log(myObj[x]); var dtarr = myObj[x].split(','); diff --git a/archive/static_content/search/index.html b/archive/static_content/search/index.html index 5fa067bec..391ede95f 100644 --- a/archive/static_content/search/index.html +++ b/archive/static_content/search/index.html @@ -78,8 +78,9 @@

Search the Archive.



This page allows you to search the archive for specific events in the matched and single-station detections.
- Time should be entered as local time, so in the UK summer you should enter the time in BST. + Time should be entered as local time, so in the UK summer you should enter the time in BST.

+

NOTE - default date range is the last 24 hours. This might be a HUGE amount of data during busy showers.

To search the livestream, go over to this page.

diff --git a/archive/terraform/mjmm/dev_ssm_parameters.tf b/archive/terraform/mjmm/dev_ssm_parameters.tf index e622d5475..a9216604f 100644 --- a/archive/terraform/mjmm/dev_ssm_parameters.tf +++ b/archive/terraform/mjmm/dev_ssm_parameters.tf @@ -32,7 +32,7 @@ resource "aws_ssm_parameter" "dev_livebucket" { resource "aws_ssm_parameter" "dev_siteurl" { name = "dev_siteurl" type = "String" - value = "http://mjmm-ukmonarchive.co.uk.s3-website.eu-west-2.amazonaws.com" + value = "https://www.ukmeteors.co.uk/dummy/" tags = { "billingtag" = "ukmon" } @@ -50,7 +50,16 @@ resource "aws_ssm_parameter" "dev_envname" { resource "aws_ssm_parameter" "dev_calcinstance" { name = "dev_calcinstance" type = "String" - value = "i-08cd1d5f6e1056f6b" #"i-0da38ed8aea1a1d85" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "dev_calcuser" { + name = "dev_calcuser" + type = "String" + value = "ubuntu" tags = { "billingtag" = "ukmon" } diff --git a/archive/terraform/mjmm/iam.tf b/archive/terraform/mjmm/iam.tf index 766e2d668..1da045ac5 100644 --- a/archive/terraform/mjmm/iam.tf +++ b/archive/terraform/mjmm/iam.tf @@ -198,7 +198,7 @@ resource "aws_iam_role_policy" "stsAssumeLambda" { Statement = { Action = "sts:AssumeRole" Effect = "Allow" - Resource = "arn:aws:iam::822069317839:role/service-role/S3FullAccess" + Resource = "arn:aws:iam::183798037734:role/s3AccessForRadio" } Version = "2012-10-17" } diff --git a/archive/terraform/mjmm/s3.tf b/archive/terraform/mjmm/s3.tf index 740ba4cc6..5a05aef5a 100644 --- a/archive/terraform/mjmm/s3.tf +++ b/archive/terraform/mjmm/s3.tf @@ -1,5 +1,6 @@ # Copyright (C) 2018-2023 Mark McIntyre ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmonarchive-co-uk" { bucket = "mjmm-ukmonarchive.co.uk" tags = { @@ -52,8 +53,9 @@ data "aws_iam_policy_document" "websiteacesspolicy" { resources = ["${aws_s3_bucket.mjmm-ukmonarchive-co-uk.arn}/*"] } } - +*/ ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmon-shared" { bucket = "mjmm-ukmon-shared" timeouts {} @@ -113,8 +115,9 @@ resource "aws_s3_bucket_versioning" "shared_versioning" { status = "Suspended" } } - +*/ ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmon-live" { bucket = "mjmm-ukmon-live" tags = { @@ -175,4 +178,4 @@ resource "aws_s3_bucket_versioning" "live_versioning" { status = "Suspended" } } - +*/ diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 683a1367c..73dff36bc 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -50,7 +50,18 @@ resource "aws_ssm_parameter" "prod_envname" { resource "aws_ssm_parameter" "prod_calcinstance" { name = "prod_calcinstance" type = "String" - value = "i-04cd701c3cfc980f5" # "i-08cd1d5f6e1056f6b" + value = "i-0ab47af23705beb31" + #value = "i-04cd701c3cfc980f5" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "prod_calcuser" { + name = "prod_calcuser" + type = "String" + value = "ubuntu" + #value = "ec2-user" # "ubuntu" tags = { "billingtag" = "ukmon" } diff --git a/archive/terraform/mjmm/ec2.tf b/archive/terraform/mjmm/ukmonhelper_ec2.tf similarity index 100% rename from archive/terraform/mjmm/ec2.tf rename to archive/terraform/mjmm/ukmonhelper_ec2.tf diff --git a/archive/terraform/mjmm/variables.tf b/archive/terraform/mjmm/variables.tf index f1d68fb92..bad991dce 100644 --- a/archive/terraform/mjmm/variables.tf +++ b/archive/terraform/mjmm/variables.tf @@ -11,8 +11,8 @@ variable "webbucket" {default = "ukmda-website"} variable "sharedbucket" {default = "ukmda-shared"} variable "livebucket" {default = "ukmda-live"} -variable "dev_sharedbucket" { default = "mjmm-ukmon-shared" } -variable "dev_webbucket" { default = "mjmm-ukmonarchive.co.uk" } -variable "dev_livebucket" { default = "mjmm-ukmon-live" } +variable "dev_webbucket" { default = "ukmda-website" } +variable "dev_sharedbucket" { default = "ukmda-shared" } +variable "dev_livebucket" { default = "ukmda-live" } variable "vpc_id" { default = "vpc-a19015c8" } diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/calcserver.tf similarity index 96% rename from archive/terraform/ukmda/ec2.tf rename to archive/terraform/ukmda/calcserver.tf index 29c5fd71c..fe1248458 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/calcserver.tf @@ -17,7 +17,7 @@ resource "aws_instance" "calc_server" { "Name" = "calcengine" "billingtag" = "ukmda" } - volume_size = 100 + volume_size = 120 } primary_network_interface { network_interface_id = aws_network_interface.calcserver_if.id @@ -47,7 +47,7 @@ resource "aws_network_interface" "calcserver_if" { resource "aws_instance" "ubuntu_calc_server" { ami = "ami-0bdf149a42243bde8" - instance_type = "c6g.4xlarge" + instance_type = "c8g.2xlarge" iam_instance_profile = aws_iam_instance_profile.calcserverrole.name key_name = aws_key_pair.marks_key.key_name tags = { @@ -58,10 +58,10 @@ resource "aws_instance" "ubuntu_calc_server" { } root_block_device { tags = { - "Name" = "calcengine2" + "Name" = "calcengine_ub" "billingtag" = "ukmda" } - volume_size = 100 + volume_size = 120 } primary_network_interface { network_interface_id = aws_network_interface.ubuntu_calcserver_if.id diff --git a/archive/terraform/ukmda/ecs_vpc.tf b/archive/terraform/ukmda/ecs_vpc.tf index 6c7243599..3f88c92a9 100644 --- a/archive/terraform/ukmda/ecs_vpc.tf +++ b/archive/terraform/ukmda/ecs_vpc.tf @@ -5,10 +5,6 @@ #data used by the code in several places data "aws_region" "current" {} -# some variables for the cluster and task defns -variable "ecsloggroup" { default = "/ecs/trajcont" } -variable "containername" { default = "trajcont" } - # create a VPC for the cluster resource "aws_vpc" "ecs_vpc" { cidr_block = "172.128.0.0/16" diff --git a/archive/terraform/ukmda/ecscluster-test.tf b/archive/terraform/ukmda/ecscluster-test.tf new file mode 100644 index 000000000..44c532659 --- /dev/null +++ b/archive/terraform/ukmda/ecscluster-test.tf @@ -0,0 +1,73 @@ +# terraform to create ECS cluster +# Copyright (C) 2018-2023 Mark McIntyre + +variable "ecsloggrouptest" { default = "/ecs/trajcontest" } +variable "containernametest" { default = "trajconttest" } + +# create a cluster +resource "aws_ecs_cluster" "trajsolvertest" { + name = "trajsolvertest" + tags = { + "billingtag" = "ukmda" + } +} + +# declare the capacity provider type, in this case FARGATE +resource "aws_ecs_cluster_capacity_providers" "trajsolvertest_cap" { + cluster_name = aws_ecs_cluster.trajsolvertest.name + capacity_providers = ["FARGATE", "FARGATE_SPOT"] +} + +data "template_file" "tasktest_json_template" { + template = file("files/trajsolver/trajsolver_container.json") + vars = { + acctid = data.aws_caller_identity.current.account_id + regionid = "eu-west-2" + repoid = "calcengine/trajsolvertest" + contname = var.containernametest + loggrp = var.ecsloggrouptest + } +} + +# define the task +# the definition of the container it runs are in the webapp.json file +resource "aws_ecs_task_definition" "trajsolvertest_task" { + family = "trajsolvertest" + container_definitions = data.template_file.tasktest_json_template.rendered + cpu = 4096 + memory = 8192 + network_mode = "awsvpc" + tags = { + billingtag = "ukmda" + } + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.ecstaskrole.arn + task_role_arn = aws_iam_role.ecstaskrole.arn + runtime_platform { + operating_system_family = "LINUX" + } +} + +# create a local file containing the clustername and a few other details +# +resource "null_resource" "createECSdetailstest" { + triggers = { + clusname = join(",", tolist([aws_ecs_cluster.trajsolvertest.name, + aws_subnet.ecs_subnet.id, + aws_security_group.ecssecgrp.id, + aws_iam_role.ecstaskrole.arn, var.ecsloggrouptest, + var.containernametest])) + } + provisioner "local-exec" { + command = "echo $env:CLUSNAME $env:SECGRP $env:SUBNET $env:IAMROLE $env:LOGGRP $env:CONTNAME > ../../ukmon_pylib/traj/clusdetailstest-ukmda.txt" + interpreter = ["pwsh.exe", "-command"] + environment = { + CLUSNAME = "${aws_ecs_cluster.trajsolvertest.name}" + SECGRP = "${aws_security_group.ecssecgrp.id}" + SUBNET = "${aws_subnet.ecs_subnet.id}" + IAMROLE = "${aws_iam_role.ecstaskrole.arn}" + LOGGRP = "${var.ecsloggrouptest}" + CONTNAME = "${var.containernametest}" + } + } +} diff --git a/archive/terraform/ukmda/ecscluster.tf b/archive/terraform/ukmda/ecscluster.tf index 974d4c596..2dbb16566 100644 --- a/archive/terraform/ukmda/ecscluster.tf +++ b/archive/terraform/ukmda/ecscluster.tf @@ -1,6 +1,10 @@ # terraform to create ECS cluster # Copyright (C) 2018-2023 Mark McIntyre +# some variables for the cluster and task defns +variable "ecsloggroup" { default = "/ecs/trajcont" } +variable "containername" { default = "trajcont" } + # create a cluster resource "aws_ecs_cluster" "trajsolver" { name = "trajsolver" diff --git a/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py b/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py index 654ad06a9..40c3a4214 100644 --- a/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py +++ b/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py @@ -61,28 +61,19 @@ def storeInDDb(evtdets, camdets): return response['ResponseMetadata']['HTTPStatusCode'] -def updateLiveTable(event, dtval): - record = event['Records'][0] - fname = record['s3']['object']['key'] - _, barefname = os.path.split(fname) - if 'P.jpg' not in barefname: - print(f'{barefname} not a jpg') - return +def updateLiveTable(evtdets, camdets, fname): + statname = camdets['camid'] + dtval = evtdets['dtval'] expdate = int((dtval + datetime.timedelta(days=90)).timestamp()) tstamp = str(int(dtval.timestamp()*1000)) yr = dtval.strftime('%Y') mth = dtval.strftime('%m') ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table('live') - if barefname[0] == 'M': - statname = barefname[17:].replace('P.jpg','').replace('_',' ') - else: - statname = barefname[3:9] - - print(f'inserting {barefname} with timestamp {dtval}') + print(f'inserting {fname} with timestamp {dtval}') response = table.put_item( Item={ - 'image_name': barefname, + 'image_name': fname, 'timestamp': tstamp, 'image_timestamp': tstamp, 'station_name': statname, @@ -94,19 +85,15 @@ def updateLiveTable(event, dtval): return response['ResponseMetadata']['HTTPStatusCode'] -def processXml(event): - record = event['Records'][0] - fname = record['s3']['object']['key'] +def processXml(remote_xmlname): buck = 'ukmda-live' s3 = boto3.resource('s3') - if '.xml' not in fname: - fname = fname.replace('P.jpg','.xml') - _, barefname = os.path.split(fname) + _, barefname = os.path.split(remote_xmlname) tmpdir = mkdtemp() xmlname = os.path.join(tmpdir, barefname) try: - s3.meta.client.download_file(buck, fname, xmlname) + s3.meta.client.download_file(buck, remote_xmlname, xmlname) except Exception: print('xml file not available') return None, None @@ -157,7 +144,12 @@ def processXml(event): def lambda_handler(event, context): - evtdets, camdets = processXml(event) - if evtdets is not None: - storeInDDb(evtdets, camdets) - updateLiveTable(event, evtdets['dtval']) + record = event['Records'][0] + fname = record['s3']['object']['key'] + _, barefname = os.path.split(fname) + if ('M' in barefname and 'P.jpg' in barefname) or ('FF' in barefname and '.jpg' in barefname): + xmlname = fname.replace('P.jpg','.xml').replace('.jpg', '.xml') + evtdets, camdets = processXml(xmlname) + if evtdets is not None: + storeInDDb(evtdets, camdets) + updateLiveTable(evtdets, camdets, barefname) diff --git a/archive/terraform/ukmda/ukmdalive_bucket.tf b/archive/terraform/ukmda/ukmdalive_bucket.tf index 7e1c405bb..45ea7d50c 100644 --- a/archive/terraform/ukmda/ukmdalive_bucket.tf +++ b/archive/terraform/ukmda/ukmdalive_bucket.tf @@ -9,58 +9,6 @@ resource "aws_s3_bucket" "ukmdalive" { } provider = aws.eu-west-1-prov } -/* -resource "aws_s3_bucket_policy" "ukmdalivebp" { - bucket = aws_s3_bucket.ukmdalive.id - provider = aws.eu-west-1-prov - policy = jsonencode( - { - Id = "ukmda-live-bp" - Statement = [ - { - "Sid": "DataSyncCreateS3LocationAndTaskAccess", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::${var.eeaccountid}:role/DataSyncBetweenAccounts" - }, - "Action": [ - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:AbortMultipartUpload", - "s3:DeleteObject", - "s3:GetObject", - "s3:ListMultipartUploadParts", - "s3:PutObject", - "s3:GetObjectTagging", - "s3:PutObjectTagging" - ], - "Resource": [ - "${aws_s3_bucket.ukmdalive.arn}", - "${aws_s3_bucket.ukmdalive.arn}/*" - ] - }, - { - "Sid": "replicatelive", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::${var.eeaccountid}:role/service-role/replicatelive" - }, - "Action": [ - "s3:Put*", - "s3:Get*" - ], - "Resource": [ - "${aws_s3_bucket.ukmdalive.arn}", - "${aws_s3_bucket.ukmdalive.arn}/*" - ] - } - ] - Version = "2008-10-17" - } - ) -} -*/ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalivelcp" { bucket = aws_s3_bucket.ukmdalive.id diff --git a/archive/ukmon_pylib/analysis/analyseGmnData.py b/archive/ukmon_pylib/analysis/analyseGmnData.py index 77920b99c..749216f3c 100644 --- a/archive/ukmon_pylib/analysis/analyseGmnData.py +++ b/archive/ukmon_pylib/analysis/analyseGmnData.py @@ -2,11 +2,40 @@ import pandas as pd import os +import sys +import datetime from converters.gmnTxtToPandas import dirpath -def findDuplicates(yr, mth=None): +def analyseAMonth(fulldf, yr, mth): + if mth != 0: + beg = datetime.datetime(yr, mth, 1) + if mth < 12: + end = datetime.datetime(yr, mth+1, 1) + else: + end = datetime.datetime(yr+1, 1, 1) + mthdf = fulldf[fulldf.utc_beg >= beg] + mthdf = mthdf[mthdf.utc_beg < end] + else: + mthdf = fulldf + tot_num = len(mthdf) + uktraj = mthdf[mthdf.stats.str.contains('UK')] + num_uk_traj = len(uktraj) + allstats = ','.join(uktraj.stats) + stats = list(set(allstats.split(','))) + ukstats = [x for x in stats if 'UK' in x] + otherstats = [x for x in stats if 'UK' not in x] + distinctuk = [x for x in ukstats if '_' not in x] + distinctother = [x for x in otherstats if '_' not in x] + ctrys = list(set([x[:2] for x in otherstats])) + otherctrydf = uktraj[uktraj.stats.str.contains('|'.join(ctrys))] + if mth == 0: + uktraj.to_csv('uk-trajectories.csv', index=False) + return len(distinctuk), len(distinctother), num_uk_traj, tot_num, len(otherctrydf) + + +def findDuplicatesById(yr, mth=None): if mth: datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') else: @@ -18,3 +47,59 @@ def findDuplicates(yr, mth=None): print(duperows) print(len(df)) return duperows + + +def atleastOneStation(stats,statsnext): + if stats is None or statsnext is None: + return False + stats = stats.split(',') + statsnext = statsnext.split(',') + return any(i in statsnext for i in stats) + + +def findDuplicatesByStatAndJD(yr, mth=None, df=None): + if not df: + dirpath = '.' + if mth: + datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') + else: + datafile = os.path.join(dirpath, 'parquet', f'gmn_{yr:04d}.parquet.snap') + df = pd.read_parquet(datafile) + df['statsnext'] = df.stats.shift(-1) + df['jd_next'] = df.jd_beg.shift(-1) + df['Lat1_next'] = df.Lat1.shift(-1) + df['Lon1_next'] = df.Lon1.shift(-1) + nearjd = df.query('abs(jd_beg - jd_next) < (0.5/86400)') + nearlat = nearjd.query('abs(Lat1 - Lat1_next) < 1 and abs(Lon1 - Lon1_next) < 1') + samestats = nearlat.query('stats == statsnext') + nearlat['overlapstats'] = nearlat.apply(lambda row: atleastOneStation(row.stats, row.statsnext), axis=1) + commonstats = nearlat[nearlat.overlapstats] + print(f'there are {len(df)} trajectories in the period') + print(f'there are {len(nearjd)} ({round(100*len(nearjd)/len(df),2)}%) events within 1s') + print(f'there are {len(commonstats)} ({round(100*len(commonstats)/len(df),2)}%) with at least one common station') + print(f'there are {len(samestats)} ({round(100*len(samestats)/len(df),2)}%) with the same stations') + return nearjd, commonstats, samestats + + +if __name__ == '__main__': + yr = int(sys.argv[1]) + fulldf = None + data = [] + for mth in range(1,13): + print(f'processing month {mth}') + df = pd.read_parquet(f'parquet/monthly/gmn_{yr}{mth:02d}.parquet.snap') + cams, othercams, traj, totaltraj, othertraj = analyseAMonth(df, yr, mth) + data.append([mth, cams, othercams, traj, totaltraj, othertraj]) + fulldf = df if fulldf is None else pd.concat([fulldf, df]) + + result = pd.DataFrame(data, columns=['Mth','Camcount','OtherCams','TrajCount','TotalTraj','NonUKTraj']) + tmpres = result.set_index('Mth') + plot = tmpres.plot(title='UK Statistics') + fig = plot.get_figure() + fig.savefig(f'uk-analysis-{yr}.jpg') + + cams, othercams, traj, totaltraj, otherctry, = analyseAMonth(fulldf, yr, 0) + data.append([0, cams, othercams, traj, totaltraj, otherctry]) + + result = pd.DataFrame(data, columns=['Mth','Camcount','OtherCams','TrajCount','TotalTraj','NonUKTraj']) + result.to_csv('2025-stats.csv', index=False) diff --git a/archive/ukmon_pylib/analysis/gatherDetectionData.py b/archive/ukmon_pylib/analysis/gatherDetectionData.py index b9031167b..d7e45b9b8 100644 --- a/archive/ukmon_pylib/analysis/gatherDetectionData.py +++ b/archive/ukmon_pylib/analysis/gatherDetectionData.py @@ -12,7 +12,7 @@ def getUncalibratedImageList(dtstr=None): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') flines = open(logfile, 'r').readlines() uncal = [f for f in flines if 'not recalibrated' in f] @@ -37,7 +37,7 @@ def getRawData(idlist, outpth): def updateSingleDB(yr, consdt): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) sngls = pd.read_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv')) consdf = pd.read_csv(os.path.join(datadir, 'single', 'used', f'consumed_{consdt}.txt'), names=['Filename']) conslist = list(consdf.Filename) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index 29636148f..02551d78a 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -467,7 +467,7 @@ def magDistributionVis(dta, shwrname, outdir, binwidth=0.2): def showerAnalysis(shwr, dtstr): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) # set up paths, files etc # check if month was passed in diff --git a/archive/ukmon_pylib/analysis/stationAnalysis.py b/archive/ukmon_pylib/analysis/stationAnalysis.py index 519bf4790..fb7a2e9d5 100644 --- a/archive/ukmon_pylib/analysis/stationAnalysis.py +++ b/archive/ukmon_pylib/analysis/stationAnalysis.py @@ -140,7 +140,7 @@ def reportOneSite(yr, mth, loc, sngl, mful, idlist, outdir): when = f'{mth:02d}-{yr}' idxfile = os.path.join(outdir,'index.html') - templatedir=os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templatedir=os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) shutil.copyfile(os.path.join(templatedir, 'header.html'), idxfile) outf = open(idxfile, 'a+') @@ -291,7 +291,7 @@ def pushToWebsite(fuloutdir, outdir, websitebucket): matchcols = ['_Y_ut','_M_ut','_mag','_mjd','mjd','_stream','orbname','stations'] snglcols = ['ID','Shwr','Dtstamp','Ver', 'M', 'Y'] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camlist = loadLocationDetails() sngl = pd.read_parquet(os.path.join(datadir, 'single', f'singles-{yr}.parquet.snap'), columns=snglcols) diff --git a/archive/ukmon_pylib/analysis/summaryAnalysis.py b/archive/ukmon_pylib/analysis/summaryAnalysis.py index 1aa6117a4..5df930768 100644 --- a/archive/ukmon_pylib/analysis/summaryAnalysis.py +++ b/archive/ukmon_pylib/analysis/summaryAnalysis.py @@ -13,7 +13,7 @@ def showerSummaryByPeriod(dtstr): yr = int(dtstr[:4]) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) cols = ['_stream','_M_ut'] filt = None matchfile = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) @@ -34,7 +34,7 @@ def showerSummaryByPeriod(dtstr): def createSummWebpage(dtstr, outdir=None): yr = dtstr[:4] mth = None - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if len(dtstr) > 4: mth = dtstr[4:6] if outdir is None: @@ -43,7 +43,7 @@ def createSummWebpage(dtstr, outdir=None): if outdir is None: outdir = os.path.join(datadir, 'reports', yr, 'showers') os.makedirs(outdir, exist_ok=True) - templatedir=os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templatedir=os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) idxfile = os.path.join(outdir, 'index.html') shutil.copyfile(os.path.join(templatedir, 'header.html'), idxfile) with open(idxfile, 'a+') as outf: @@ -73,7 +73,7 @@ def createSummWebpage(dtstr, outdir=None): def createSummJS(dtstr, shwrdata, maxlines=None): yr = dtstr[:4] mth = None - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if len(dtstr) > 4: mth = dtstr[4:6] outdir = os.path.join(datadir, 'reports', yr, 'showers', mth) diff --git a/archive/ukmon_pylib/converters/gmnTxtToPandas.py b/archive/ukmon_pylib/converters/gmnTxtToPandas.py index ad98886f9..4e759b6b9 100644 --- a/archive/ukmon_pylib/converters/gmnTxtToPandas.py +++ b/archive/ukmon_pylib/converters/gmnTxtToPandas.py @@ -17,7 +17,7 @@ 'Lat1','Lat1sd','Lon1','Lon1sd','H1','H1sd','Lat2','Lat2sd','Lon2','Lon2sd','H2','H2sd', 'Dur','Amag','PkHt','F1','mass','Qc','MedianFitErr','BegIn','EndIn','NumStat','stats'] -datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') +datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) dirpath = os.path.join(datadir, 'gmndata') @@ -37,7 +37,7 @@ def loadOneFile(fname): df = df.fillna(value=np.nan) df['jd_beg'] = df.jd_beg.astype(float) df['utc_beg'] = pd.to_datetime(df.utc_beg) - df['ian_uo'] = df.iau_no.astype(int) + df['iau_no'] = df.iau_no.astype(int) for c in range(5,82): try: df[colhdrs[c]] = df[colhdrs[c]].astype(float) @@ -138,4 +138,10 @@ def getStats(): if __name__ == '__main__': - doYear(sys.argv[1]) + if sys.argv[1] == 'doYear': + doYear(sys.argv[2]) + if sys.argv[1] == 'convert': + df = loadOneFile(sys.argv[2]) + fn, _ = os.path.splitext(sys.argv[2]) + df.to_parquet(f'{fn}.parquet.snap', index=False) + diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index 96d24dab9..911abcb3e 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -9,6 +9,13 @@ import paramiko from scp import SCPClient from time import sleep +import pandas as pd +import datetime + +import pymysql.cursors + +from wmpl.Utils.TrajConversions import datetime2JD +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase def findInputDataByMonth(yyyymm, archbucket, outdir): @@ -97,6 +104,95 @@ def deleteFromCalcServerByMonth(outfname): return +def removeDeletedTraj(csvfile): + """ + Remove deleted trajectories from the consolidated match CSV and Parquet files + and from the search index. + """ + + csvdata = open(csvfile, 'r').readlines() + if 'search' in csvfile: + ts_end = float(csvdata[-1].split(',')[0]) + dt_end = datetime.datetime.fromtimestamp(ts_end, tz=datetime.timezone.utc) + jdt_end = datetime2JD(dt_end) + else: + jdt_end = float(csvdata[-1].split(',')[3]) + 2400000.5 + + jdt_beg =jdt_end - 21 + + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + i=0 + for traj in deltrajs: + fldr = os.path.basename(os.path.dirname(traj[0])) + match = [tr for tr in csvdata if fldr in tr] + if len(match) > 0: + for thismtch in match: + print(f'removing {fldr}') + idx = csvdata.index(thismtch) + _ = csvdata.pop(idx) + i += 1 + print(f'removed {i} trajectories') + + open(csvfile, 'w').writelines(csvdata) + + if 'search' not in csvfile: + df = pd.read_csv(csvfile, skipinitialspace=True) + df = df.drop_duplicates(subset=['_mjd','_sol','_ID1','_ra_o','_dc_o','_amag','_ra_t','_dc_t']) + df.to_csv(csvfile, index=False) + + return + + +def getSqlLoginDetails(): + # retrieve password and host from SSM. This allows me to manage them from Terraform + ssm = boto3.client('ssm', region_name='eu-west-1') + res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) + password = res['Parameter']['Value'] + res = ssm.get_parameter(Name='prod_dbhost') + host = res['Parameter']['Value'] + # should really do these too but they won't change often if at all + user = 'batch' + db = 'ukmon' + return host, user, password, db + + +def removeDelTrajFromDb(): + dt_end = datetime.datetime.now(tz=datetime.timezone.utc) + jdt_end = datetime2JD(dt_end) + jdt_beg =jdt_end - 7 + + # get list of deleted trajectories + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + # get connection to the SQL database + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + + count = 0 + for traj in deltrajs: + fldr = os.path.basename(os.path.dirname(traj[0])) + sqlstr = f"delete from matches where orbname like '{fldr}%'" + with connection.cursor() as cursor: + cursor.execute(sqlstr) + result = cursor.fetchall() + count += len(result) + connection.commit() + connection.close() + print(f'cleaned up {count} trajectories') + return + + if __name__ == '__main__': arg_parser = argparse.ArgumentParser(description="Find and clear down historical raw data.") diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py index f1b477283..2dadf8cb6 100644 --- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py +++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py @@ -25,8 +25,8 @@ csvdir = 'f:/videos/meteorcam/usermgmt/csvkeys' lnxdir='/home/ec2-user/keymgmt/csvkeys' else: - csvdir='/home/ec2-user/keymgmt/csvkeys' - lnxdir='/home/ec2-user/keymgmt/csvkeys' + csvdir=os.path.expanduser('~/keymgmt/csvkeys') + csvdir=os.path.expanduser('~/keymgmt/csvkeys') def checkIfOnGMN(camid): diff --git a/archive/ukmon_pylib/maintenance/manageTraj.py b/archive/ukmon_pylib/maintenance/manageTraj.py index fae24d735..0a684babd 100644 --- a/archive/ukmon_pylib/maintenance/manageTraj.py +++ b/archive/ukmon_pylib/maintenance/manageTraj.py @@ -8,7 +8,7 @@ # delete an orbit from the database def deleteDuplicate(trajname, jd=None): - datadir = os.getenv('DATADIR', default='/home/pi/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) yr=trajname[:4] if int(yr) > 2021: fname = os.path.join(datadir, 'matched','matches-full-{}.parquet.snap'.format(yr)) diff --git a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py index 28d7b9c02..59b37d3b8 100644 --- a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py +++ b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py @@ -107,7 +107,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): # ax.add_feature(cfeature.LAKES, edgecolor='black') # ax.add_feature(cfeature.RIVERS) ax.gridlines() - pylib=os.getenv('PYLIB', default='/home/ec2-user/prod/ukmon_pylib') + pylib=os.getenv('PYLIB', default=os.path.expanduser('~/prod/ukmon_pylib')) os.environ['CARTOPY_USER_BACKGROUNDS'] = os.path.join(pylib, 'share','maps') ax.background_img(name='BM', resolution='high', extent= [minn, maxn, mina, maxa]) @@ -143,7 +143,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): plt.plot(yi, xi, 'bx', markersize=10, linewidth=1) plt.tight_layout() - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if datadir is None: datadir='.' plt.savefig(os.path.join(datadir, 'stations.png'), dpi=200) @@ -158,7 +158,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): plotlabels = False else: plotlabels = True - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) srcfile = os.path.join(datadir, 'admin', 'cameraLocs.json') intersect = False if len(sys.argv) > 2: diff --git a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py index feb2a5079..24107d46b 100644 --- a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py +++ b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py @@ -14,7 +14,7 @@ def findOtherBadEvents(): evts = [] - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'dailyreports', 'latest.txt') with open(lastf, 'r') as inf: lis = inf.readlines() @@ -33,7 +33,7 @@ def findOtherBadEvents(): def findFailedEvents(prof='ukmonshared'): #session = boto3.Session(profile_name=prof) logcli = boto3.client('logs', region_name='eu-west-2') - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') if os.path.isfile(lastf): with open(lastf, 'r') as inf: @@ -102,7 +102,7 @@ def rerunFails(fails, prof='ukmonshared'): session=boto3.Session(profile_name=prof) lambd = session.client('lambda', region_name='eu-west-2') - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') with open(lastf, 'w') as outf: rundt = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 09c961dce..e85ff3d66 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -71,7 +71,7 @@ def findRowCamTimings(stationid, uploaddate, ddb=None): def getDayCamTimings(uploaddate, ddb=None, outfile=None, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if not ddb: ddb = boto3.resource('dynamodb', region_name='eu-west-2') #, endpoint_url="http://thelinux:8000") table = ddb.Table('uploadtimes') @@ -136,9 +136,10 @@ def deleteRowCamTimings(stationid, dtstamp, ddb=None): def backPopulate(stationid): s3bucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - fldrs = glob.glob1(f'/home/ec2-user/ukmon-shared/matches/RMSCorrelate/{stationid}/', '*') + basepath = os.path.expanduser('~/prod/ukmon-shared/matches/RMSCorrelate') + fldrs = glob.glob1(os.path.join(basepath, stationid), '*') for fldr in fldrs: - s3objects = glob.glob1(f'/home/ec2-user/ukmon-shared/matches/RMSCorrelate/{stationid}/{fldr}/', 'FTPd*') + s3objects = glob.glob1(os.path.join(basepath, stationid. fldr), 'FTPd*') if len(s3objects) > 0: s3obj = s3objects[0] fullobj = f'matches/RMSCorrelate/{stationid}/{fldr}/{s3obj}' @@ -147,7 +148,7 @@ def backPopulate(stationid): if __name__ == '__main__': - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) ddb = boto3.resource('dynamodb', region_name='eu-west-2') if os.path.isfile('/sys/devices/virtual/dmi/id/board_asset_tag'): # crude check for EC2 diff --git a/archive/ukmon_pylib/metrics/costMetrics.py b/archive/ukmon_pylib/metrics/costMetrics.py index 36f83dd2b..9b0236fe6 100644 --- a/archive/ukmon_pylib/metrics/costMetrics.py +++ b/archive/ukmon_pylib/metrics/costMetrics.py @@ -17,7 +17,7 @@ def monthlyCostByService(dtstr, acctid): mth = int(dtstr[4:6]) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) df = pd.read_csv(os.path.join(datadir, 'costs', f'costs-{acctid}-90-days.csv')) if acctid != '183798037734': df = df[(df.Tag.str.contains('ukmda')) | (df.Tag.str.contains('ukmon'))] diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index 8114c4e3a..e1dcd4f96 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -5,6 +5,9 @@ from datetime import datetime import sys +import os +import glob +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase def getDailyObsCounts(logf): @@ -34,27 +37,42 @@ def getDailyObsCounts(logf): def getMatchStats(logf): - with open(logf) as inf: - loglines = inf.readlines() + loglines = open(logf).readlines() + + events = [line.strip() for line in loglines if 'Analysing' in line and 'observations' in line] + if len(events) > 0: + # new-style log + addlines = [x for x in events if ' 0 observations' not in x] + addoffs = -5 + oldstyle = False + else: + addlines = [line.strip() for line in loglines if 'Added' in line and 'observations' in line] + addoffs = 1 + oldstyle = True - addlines = [line.strip() for line in loglines if 'Added' in line and 'observations' in line] - nocallines = [line.strip() for line in loglines if 'Skipping' in line and 'recalibrated' in line] - misdflines = [line.strip() for line in loglines if 'Skipping' in line and 'missing data' in line] - uncal = len(nocallines) - missdf = len(misdflines) + uncal = len([line.strip() for line in loglines if 'Skipping' in line and 'recalibrated' in line]) + missdf = len([line.strip() for line in loglines if 'Skipping' in line and 'missing data' in line]) + added=0 for li in addlines: spls=li.split(' ') - added = added + int(spls[1]) + added += int(spls[addoffs]) beglowr = len([line.strip() for line in loglines if 'Begin height lower than the end height' in line]) badalti = len([line.strip() for line in loglines if 'Meteor heights outside allowed range' in line]) badvelo = len([line.strip() for line in loglines if 'Velocity difference too high' in line]) badangl = len([line.strip() for line in loglines if 'Max convergence angle too small' in line]) - trajs = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] - spls = trajs[0].split(' ') - trajs = int(spls[1]) + cands = 0 + if oldstyle: + cands = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] + spls = cands[0].split(' ') + cands = int(spls[1]) + else: + trajlines = [line.strip() for line in loglines if 'Saved' in line and ' 0 candidates' not in line] + for li in trajlines: + spls=li.split(' ') + cands += int(spls[-2]) nonphys = beglowr + badalti + badvelo + badangl tot = added + uncal + missdf @@ -73,9 +91,42 @@ def getMatchStats(logf): d2=datetime.strptime(etim,'%H:%M:%S') cstime = str(d2 - d1) - return tot, added, uncal, missdf, nonphys, trajs, runtime, cstime + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + rundate=os.getenv('rundate') + if not rundate: + rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() + dailydbdir = os.path.join(datadir, 'latest','dailydbs') + trajdb = TrajectoryDatabase(dailydbdir, f'{rundate}_trajectories.db') + trajs = len(trajdb.getTrajBasics('.',[0,9999999])) + + return tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime + + +def updateStats(obscount, candcount, trajcount, runtime): + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + rundate=os.getenv('rundate') + if not rundate: + rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() + statsdir = os.path.join(datadir, 'dailyreports') + reports = glob.glob(os.path.join(statsdir, f'{rundate}*.txt')) + + # only update stats if the daily report exists already + if len(reports) > 0: + dailyrep = os.path.split(reports[-1])[1] + statsdata = open(os.path.join(statsdir,'stats.txt'), 'r').readlines() + + # remove any existing entry for today + currentstats = [li for li in statsdata if rundate in li] + if len(currentstats) > 0: + statsdata.pop(statsdata.index(currentstats[0])) + # add the new entry and save + statsdata.append(f'{dailyrep} {obscount} {candcount} {trajcount} {runtime}\n') + open(os.path.join(statsdir,'stats.txt'), 'w').writelines(statsdata) + return if __name__ == '__main__': - tot, added, uncal, missdf, nonphys, trajs, runtime, cstime = getMatchStats(sys.argv[1]) - print(tot, added, uncal, missdf, nonphys, trajs, runtime, cstime) + tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(sys.argv[1]) + if len(sys.argv) < 3: + updateStats(added, cands, trajs, runtime) + print(tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime) diff --git a/archive/ukmon_pylib/metrics/timingMetrics.py b/archive/ukmon_pylib/metrics/timingMetrics.py index 0f4606da5..d88273d09 100644 --- a/archive/ukmon_pylib/metrics/timingMetrics.py +++ b/archive/ukmon_pylib/metrics/timingMetrics.py @@ -47,7 +47,7 @@ def graphOfData(logf, dtstr): plt.grid(axis='x') plt.gca().invert_yaxis() logname, _ = os.path.splitext(os.path.basename(logf)) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) os.makedirs(os.path.join(datadir, 'batchcharts'), exist_ok=True) plt.savefig(os.path.join(datadir, 'batchcharts',f'./{dtstr}-{logname}.jpg'), dpi=100) plt.close() @@ -130,7 +130,7 @@ def getLogStats(nightlogf, matchlogf, thisdy): if __name__ == '__main__': dtstr = sys.argv[1] - logdir = os.path.join(os.getenv('SRC', default='/home/ec2-user/prod'), 'logs') + logdir = os.path.join(os.getenv('SRC',default=os.path.expanduser('~/prod')), 'logs') nowdt = datetime.datetime.now().strftime('%Y%m%d') if nowdt == dtstr: nightf = os.path.join(logdir, 'nightlyJob.log') diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index 74ce7ac86..ab3fd50d9 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -13,7 +13,7 @@ def getCamLocDirFov(camid, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camdb = json.load(open(os.path.join(datadir, 'admin', 'cameraLocs.json'))) if camid not in camdb.keys(): return False @@ -22,7 +22,7 @@ def getCamLocDirFov(camid, datadir=None): def updateCamLocDirFovDB(datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camdb = {} ppfiles = glob.glob(os.path.join(datadir, 'consolidated','platepars', '*.json')) for ppf in ppfiles: @@ -80,7 +80,7 @@ def createCDCsv(targetloc): cd4csv = cd4csv[['site','stationid','oldcode','direction','camtype','dummycode','active']] cd4csv = cd4csv.rename(columns={'stationid':'camid', 'oldcode':'lid', 'direction':'sid'}) cd4csv.sort_values(by=['active','camid'], inplace=True) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) outfname = os.path.join(datadir, targetloc, 'camera-details.csv') cd4csv.to_csv(outfname, index=False) with open(os.path.join(datadir, 'activecamcount.txt'), 'w') as outf: diff --git a/archive/ukmon_pylib/reports/cameraStatusReport.py b/archive/ukmon_pylib/reports/cameraStatusReport.py index 0872feff7..256ad0765 100644 --- a/archive/ukmon_pylib/reports/cameraStatusReport.py +++ b/archive/ukmon_pylib/reports/cameraStatusReport.py @@ -25,7 +25,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): caminfo = caminfo.drop(columns=['direction','oldcode','active','camtype','eMail', 'humanName']) caminfo.rename(columns={'site':'Site'}, inplace=True) if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) tmplist = pd.read_csv(os.path.join(datadir,'reports','camuploadtimes.csv'), index_col=False) fldrlist = pd.merge(tmplist, caminfo, on=['stationid'], how='inner') @@ -54,7 +54,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): def createStatusReportJSfile(stati, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) os.makedirs(os.path.join(datadir, 'reports'), exist_ok=True) repfile = os.path.join(datadir, 'reports','camrep.js') with open(repfile, 'w') as outf: diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 05aa32aa5..9426f5d6d 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -15,7 +15,7 @@ def createBarChart(datadir=None, yr=None): if yr is None: yr=datetime.datetime.now().year if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) fname = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): print('{} missing', fname) diff --git a/archive/ukmon_pylib/reports/createExchangeFiles.py b/archive/ukmon_pylib/reports/createExchangeFiles.py index 820650ab5..e11a5588a 100644 --- a/archive/ukmon_pylib/reports/createExchangeFiles.py +++ b/archive/ukmon_pylib/reports/createExchangeFiles.py @@ -164,7 +164,7 @@ def createCameraFile(datadir): def createAll(targdate=None, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/datas') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if targdate is None: targdate = datetime.datetime.now() createCameraFile(datadir) diff --git a/archive/ukmon_pylib/reports/createSearchableFormat.py b/archive/ukmon_pylib/reports/createSearchableFormat.py index e2df79c2b..4424cc4bf 100644 --- a/archive/ukmon_pylib/reports/createSearchableFormat.py +++ b/archive/ukmon_pylib/reports/createSearchableFormat.py @@ -110,7 +110,7 @@ def convertMatchToSrchable(datadir, year, newonly=True): print('usage: python createSearchableFormat.py year mode outdir') exit(1) else: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) year = sys.argv[1] mode = sys.argv[2] diff --git a/archive/ukmon_pylib/reports/createSummaryTable.py b/archive/ukmon_pylib/reports/createSummaryTable.py index cff8e40ce..a0d861bb1 100644 --- a/archive/ukmon_pylib/reports/createSummaryTable.py +++ b/archive/ukmon_pylib/reports/createSummaryTable.py @@ -18,7 +18,7 @@ def createSummaryTable(curryr=None, datadir=None): if curryr is None: curryr = str(datetime.datetime.now().year) if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) fname = os.path.join(datadir, 'summarytable.js') with open(fname, 'w') as f: f.write('$(function() {\n') diff --git a/archive/ukmon_pylib/reports/dailyReport.py b/archive/ukmon_pylib/reports/dailyReport.py index 8c7852070..a511c4055 100644 --- a/archive/ukmon_pylib/reports/dailyReport.py +++ b/archive/ukmon_pylib/reports/dailyReport.py @@ -115,7 +115,7 @@ def LookForMatchesRMS(doff, dayfile, statsfile): # now send the post to groups.io targeturl='https://archive.ukmeteors.co.uk' - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) recs = open(os.path.join(datadir, 'admin','dailyReportRecips.txt'), 'r').readlines() mailFrom = 'markmcintyre99@googlemail.com' mailRecip = recs[0].strip() diff --git a/archive/ukmon_pylib/reports/extractors.py b/archive/ukmon_pylib/reports/extractors.py index d7fc4bb80..65670b38d 100644 --- a/archive/ukmon_pylib/reports/extractors.py +++ b/archive/ukmon_pylib/reports/extractors.py @@ -19,7 +19,7 @@ def createSplitMatchFile(yr, mth=None, shwr=None, matches=None): shwr (string): optional shower code. If provided, the data will be filtered """ - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if matches is None: infname = os.path.join(datadir, 'matched',f'matches-full-{yr}.parquet.snap') if not os.path.isfile(infname): @@ -58,7 +58,7 @@ def createUFOSingleMonthlyExtract(yr, mth=None, shwr=None, dta=None): """ # print('ufo singles file') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if dta is None: fname = os.path.join(datadir, 'consolidated','M_{}-unified.csv'.format(yr)) if not os.path.isfile(fname): @@ -94,7 +94,7 @@ def createRMSSingleMonthlyExtract(yr, mth=None, shwr=None, dta=None, withshower= """ #print(f'rms singles file, withshower {withshower}') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if dta is None: fname = os.path.join(datadir, 'single','singles-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): @@ -147,7 +147,7 @@ def extractAllShowersData(ymd): showerlist = sl.getMajorShowers(True, True).strip().split(' ') print(f'processing data for {ymd}') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) infname = os.path.join(datadir, 'matched',f'matches-full-{yr}.parquet.snap') if not os.path.isfile(infname): print(f'unable to open {infname}') diff --git a/archive/ukmon_pylib/reports/findBestMp4s.py b/archive/ukmon_pylib/reports/findBestMp4s.py index a07611ddd..22f533700 100644 --- a/archive/ukmon_pylib/reports/findBestMp4s.py +++ b/archive/ukmon_pylib/reports/findBestMp4s.py @@ -20,7 +20,7 @@ def getBestNMatches(reqdate=None, numtoget=10): tod = reqdate + datetime.timedelta(days=1) yr = reqdate.year - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') # select only the columns we need @@ -81,7 +81,7 @@ def getUrlFromFilename(fname): def getBestNMp4s(yr, mth, numtoget): - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') # select only the columns we need diff --git a/archive/ukmon_pylib/reports/findFireballs.py b/archive/ukmon_pylib/reports/findFireballs.py index 60b7af0c0..d30949fa6 100644 --- a/archive/ukmon_pylib/reports/findFireballs.py +++ b/archive/ukmon_pylib/reports/findFireballs.py @@ -19,7 +19,7 @@ # Manually mark a trajectoriy as a "fireball" # def markAsFireball(trajname, tof=True): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) yr=trajname[:4] if int(yr) > 2021: fname = os.path.join(datadir, 'matched','matches-full-{}.parquet.snap'.format(yr)) @@ -142,7 +142,7 @@ def findFBPre2020(df, mag=-4): def findFireballs(dtval, shwr, minmag=-3.99, matchdataset = None): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) orbdir = 'matches' # check if month was passed in diff --git a/archive/ukmon_pylib/reports/getLivestreamData.py b/archive/ukmon_pylib/reports/getLivestreamData.py index 3db0dfe24..18c15b1d3 100644 --- a/archive/ukmon_pylib/reports/getLivestreamData.py +++ b/archive/ukmon_pylib/reports/getLivestreamData.py @@ -72,7 +72,7 @@ def getLatestLiveFiles(daysback=None, df=None): if __name__ == '__main__': - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) df = getLatestLiveFiles(-30) prvfile = os.path.join(datadir, 'ukmonlive', 'livefeed.csv') if os.path.isfile(prvfile): diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index 2bdb89c52..bbb7f0fba 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -58,8 +58,8 @@ def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): def createCoveragePage(): apikey = os.getenv('APIKEY') apikey = decodeApiKey(apikey) - templdir = os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) with open(os.path.join(templdir, 'coverage-maps.html'), 'r') as inf: lis = inf.readlines() with open(os.path.join(datadir, 'latest','coverage-maps.html'), 'w') as outf: diff --git a/archive/ukmon_pylib/reports/reportActiveShowers.py b/archive/ukmon_pylib/reports/reportActiveShowers.py index 7786eafa6..7a58d91fc 100644 --- a/archive/ukmon_pylib/reports/reportActiveShowers.py +++ b/archive/ukmon_pylib/reports/reportActiveShowers.py @@ -15,7 +15,7 @@ def createShowerIndexPage(dtstr, shwr, shwrname, outdir, datadir): - templdir = os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) idxfile = os.path.join(outdir, 'index.html') shutil.copyfile(os.path.join(templdir,'header.html'), idxfile) @@ -175,7 +175,7 @@ def reportActiveShowers(ymd, thisshower=None, thismth=None, includeMinor=False): else: shwrlist = [thisshower] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) # pltdir=os.path.join(datadir, 'showerplots') dtstr = ymd[:4] if thismth is not None: diff --git a/archive/ukmon_pylib/reports/reportBadCameras.py b/archive/ukmon_pylib/reports/reportBadCameras.py index bdf4f6ef2..cc0938ee8 100644 --- a/archive/ukmon_pylib/reports/reportBadCameras.py +++ b/archive/ukmon_pylib/reports/reportBadCameras.py @@ -19,7 +19,7 @@ print('usage: python reportBadCameras.py daysback') exit(0) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if datadir is None: print('define DATADIR first') exit(1) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index f7eeed01a..2d8738584 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -8,14 +8,16 @@ import datetime import numpy import csv -import json import shutil import tempfile import boto3 +import glob from traj.pickleAnalyser import getVMagCodeAndStations from reports.CameraDetails import findSite, loadLocationDetails +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase, ObservationsDatabase + def processLocalFolder(trajdir, basedir): # load camera details @@ -39,30 +41,89 @@ def processLocalFolder(trajdir, basedir): return outstr -def getTrajPaths(trajdict): - trajpaths=[] - fullnames=[] - for traj in trajdict: - fullnames.append(trajdict[traj]['traj_file_path']) - pth, _ = os.path.split(trajdict[traj]['traj_file_path']) - trajpaths.append(pth) - return trajpaths, fullnames - - -def getListOfNewMatches(dir_path, tfile, prevtfile): - with open(os.path.join(dir_path, tfile), 'r') as inf: - trajs = json.load(inf) - with open(os.path.join(dir_path, prevtfile), 'r') as inf: - ptrajs = json.load(inf) - newtrajs = {k:v for k,v in trajs['trajectories'].items() if k not in ptrajs['trajectories']} - #print(len(newtrajs)) - _, newdirs = getTrajPaths(newtrajs) +def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): + os.makedirs(db_path, exist_ok=True) + db_name = f'{rundate}_trajectories.db' if rundate else 'trajectories.db' + dailydb = TrajectoryDatabase(db_path=db_path, db_name=db_name, purge_records=True) + flist = glob.glob(os.path.join(dir_path, 'trajectories_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if dailydb.mergeTrajDatabase(fl): + os.remove(fl) + else: + print('error') + + if os.getenv('TESTMODE').lower() == 'true': + trajdir = 'matches/distrib/test' + else: + trajdir = 'matches/RMSCorrelate' + + cur = dailydb.dbhandle.execute('select traj_file_path from trajectories where status=1') + newtrajs = cur.fetchall() + + if len(newtrajs) > 0: + # now get a list of logically-deleted trajs from the current master DB + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + + # get the date range for the new trajectories + cur = dailydb.dbhandle.execute('select min(jdt_ref), max(jdt_ref) from trajectories where status=1') + vals = cur.fetchall() + jdt_beg = float(vals[0][0]) + jdt_end = float(vals[0][1]) + + # retrieve a list of logically-deleted trajectories from within that date range + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + # iterate over the delete list and update the daily db and new traj list accordingly + print('cleaning up deleted trajectories in dailydb') + for testtr in deltrajs: + if testtr in newtrajs: + sqlstr = f'update trajectories set status=0 where traj_file_path="{testtr[0]}"' + dailydb.dbhandle.execute(sqlstr) + dailydb.dbhandle.commit() + cur = dailydb.dbhandle.execute(f'select traj_id, status from trajectories where traj_file_path="{testtr[0]}"') + msg = cur.fetchone() + print(f'updated {msg}') + newtrajs.pop(newtrajs.index(testtr)) + + dailydb.closeTrajDatabase() + + newdirs = [] + for traj in newtrajs: + newdirs.append(os.path.join(trajdir, traj[0])) + return newdirs -def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): - prevdbname = 'prev_' + dbname - newdirs = getListOfNewMatches(dir_path, dbname, prevdbname) +def updatePairedDB(dir_path, db_path='/tmp', rundate=None): + os.makedirs(db_path, exist_ok=True) + db_name = f'{rundate}_observations.db' if rundate else 'observations.db' + obsdb = ObservationsDatabase(db_path=db_path, db_name=db_name, purge_records=True) + flist = glob.glob(os.path.join(dir_path, 'observations_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if obsdb.mergeObsDatabase(fl): + os.remove(fl) + else: + print('error') + + cur = obsdb.dbhandle.execute('select count(*) from paired_obs where status=1') + obscount = cur.fetchall() + + return obscount + + +def findNewMatches(dir_path, out_path, offset, repdtstr): + daily_path = os.path.join(os.path.split(dir_path)[0], 'dailydbs') + newdirs = getListOfNewMatches(dir_path, daily_path, rundate=repdtstr) # load camera details caminfo = loadLocationDetails() caminfo = caminfo[caminfo.active==1] @@ -72,15 +133,15 @@ def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): else: repdt = datetime.datetime.now() - datetime.timedelta(int(offset)) - os.makedirs(os.path.join(out_path, 'dailyreports'), exist_ok=True) + os.makedirs(out_path, exist_ok=True) # create filename. Allow for three reruns in a day - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_1.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_1.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_2.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_2.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_3.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_3.txt')) s3 = boto3.client('s3') srcbucket=os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] @@ -90,10 +151,11 @@ def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): trajdir = trajdir[trajdir.find('matches'):] trajpath, picklename = os.path.split(trajdir) localpick = os.path.join(tmpdir, picklename) - s3.download_file(srcbucket, trajdir, localpick) try: + s3.download_file(srcbucket, trajdir, localpick) bestvmag, shwr, stationids = getVMagCodeAndStations(localpick) except: + print(f'unable to find {trajdir}') bestvmag, shwr, stationids = 0,'',[''] stations=[] for statid in stationids: @@ -127,20 +189,26 @@ def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): outf.write('\n') # finally, create a "latest.txt" as well - latestlist = os.path.join(out_path, 'dailyreports', 'latest.txt') + latestlist = os.path.join(out_path, 'latest.txt') shutil.copy(matchlist, latestlist) return if __name__ == '__main__': repdtstr = None - dbname = None if len(sys.argv) > 4: repdtstr = sys.argv[4] - if len(sys.argv) > 5: - dbname = sys.argv[5] - else: - dbname = 'processed_trajectories.json.bigserver' + + cand_db_dir = sys.argv[1] + daily_db_dir = sys.argv[2] + offset = sys.argv[3] - # arguments dblocation, datadir, days ago, rundate eg 20220524, full path to database - findNewMatches(sys.argv[1], sys.argv[2], sys.argv[3], repdtstr, dbname) + flist = glob.glob(os.path.join(cand_db_dir, 'trajectories_*.db')) + if len(flist) == 0: + print('no container databases to process, aborting') + else: + # arguments dblocation, datadir, days ago, rundate eg 20220524 + findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) + # update the daily database of paired observations + daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') + updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) diff --git a/archive/ukmon_pylib/requirements.txt b/archive/ukmon_pylib/requirements.txt index aeaba31b9..8b9f56c62 100644 --- a/archive/ukmon_pylib/requirements.txt +++ b/archive/ukmon_pylib/requirements.txt @@ -48,4 +48,5 @@ dynamodb_json pytest pytest-cov requests -paho-mqtt \ No newline at end of file +paho-mqtt +scp \ No newline at end of file diff --git a/archive/ukmon_pylib/traj/clusdetails-ee.txt b/archive/ukmon_pylib/traj/clusdetails-ee.txt deleted file mode 100644 index 1e17ed3f7..000000000 --- a/archive/ukmon_pylib/traj/clusdetails-ee.txt +++ /dev/null @@ -1,6 +0,0 @@ -trajsolver -sg-0d37a3b8ee1a3a1c6 -subnet-0c224d5642fb71023 -arn:aws:iam::822069317839:role/ecsTaskExecutionRole -/ecs/trajcont -trajcont diff --git a/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt new file mode 100644 index 000000000..d3be6424f --- /dev/null +++ b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt @@ -0,0 +1,6 @@ +trajsolvertest +sg-06fb7bbee3b4f5e94 +subnet-019eef41d5eaf419b +arn:aws:iam::183798037734:role/ecsTaskExecutionRole +/ecs/trajcontest +trajconttest diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index b213f4b45..c1fee16df 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -3,123 +3,90 @@ import os import sys import glob -import shutil import datetime -from wmpl.Trajectory.CorrelateRMS import TrajectoryReduced, DatabaseJSON -#from wmpl.Trajectory.CorrelateRMS import MeteorObsRMS, PlateparDummy, MeteorPointRMS # noqa: F401 - - -# support class so i can add any paired obs objects -class dummyMeteorObsRMS(object): - def __init__(self, station_code, id): - self.station_code = station_code - self.id = id - - -# merge a distributed engine DB back into the master DB -def mergeDatabases(srcdir, srcdb, masterpth, masterfile, mastdb = None): - newdb = os.path.join(srcdir, srcdb) - mergedb = DatabaseJSON(newdb) - if mastdb is None: - mastdb = DatabaseJSON(masterfile) - mastdb.db_file_path = masterfile - # merge successful trajectories - for traj in mergedb.trajectories: - traj_obj = TrajectoryReduced(traj, json_dict = mergedb.trajectories[traj].__dict__) - traj_file_path = traj_obj.traj_file_path - traj_file_path = masterpth + '/' + traj_file_path[traj_file_path.find('trajectories'):] - traj_obj.traj_file_path = traj_file_path - mastdb.addTrajectory(traj_file_path, traj_obj=traj_obj) - # merge failed trajectories - for traj in mergedb.failed_trajectories: - traj_obj = TrajectoryReduced(traj, json_dict = mergedb.failed_trajectories[traj].__dict__) - traj_file_path = traj_obj.traj_file_path - traj_file_path = masterpth + '/' + traj_file_path[traj_file_path.find('trajectories'):] - traj_obj.traj_file_path = traj_file_path - mastdb.addTrajectory(traj_file_path, traj_obj=traj_obj, failed=True) - # merge paired obs data - for p in mergedb.paired_obs: - ids = mergedb.paired_obs[p] - for id in ids: - met_obs = dummyMeteorObsRMS(p, id) - mastdb.addPairedObservation(met_obs) - # save the master DB again - mastdb.save() - return mastdb - - -# utility to patch the database to have the right trajectory folder -def patchTrajDB(dbfile, targpath, oldstr='/home/ec2-user/data/RMSCorrelate'): - - with open(dbfile, 'r') as inf: - with open(os.path.join('/tmp/processed_trajectories.json'), 'w') as outf: - for lin in inf: - outf.write(lin.replace(oldstr, targpath)) - shutil.copyfile('/tmp/processed_trajectories.json', dbfile) - return - - -# utility to count how many of each type of observation was in a database -def countDataInDb(path_to_db): - mergedb = DatabaseJSON(path_to_db) - trajs = len(mergedb.trajectories) - failed = len(mergedb.failed_trajectories) - pairs = len(mergedb.paired_obs) - print(path_to_db, trajs, failed, pairs) - return - - -# utility to dump the detection dates (JDs) to a file -def dumpJDs(path_to_db, fname): - mergedb = DatabaseJSON(path_to_db) - with open(fname,'w') as outf: - outf.write('traj\n') - for t in mergedb.trajectories: - outf.write(f'{t}\n') - outf.write('failed\n') - for t in mergedb.failed_trajectories: - outf.write(f'{t}\n') - - -# utility to compare two databases and dump the JDs of any new events -def dumpNewEntries(db1, db2, fname): - olddb = DatabaseJSON(db1) - newdb = DatabaseJSON(db2) - newtraj = set(newdb.trajectories) - set(olddb.trajectories) - newfail = set(newdb.failed_trajectories) - set(olddb.failed_trajectories) - print(len(newtraj), len(newfail)) - with open(fname,'w') as outf: - outf.write('traj\n') - for t in newtraj: - outf.write(f'{t}\n') - outf.write('failed\n') - for t in newfail: - outf.write(f'{t}\n') +from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase +from wmpl.Trajectory.CorrelateRMS import RMSDataHandle +from wmpl.Utils.TrajConversions import jd2Date + + +def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=False, matchstart=3): + """ + merge container databases into the master database, looking for and cleaning deleted trajectories + + arguments: + srcdir location of container databases + dbdir location of master databases + basedir location of raw data and trajectories + + keyword args: + ignore_missing default false: if true, create master DBs if not present + purge_records default false: if true, purge records from master DBs + matchstart default 3: range of days to scan for duplicate and deleted trajectories + + """ + + targdb = os.path.join(dbdir, 'observations.db') + if os.path.isfile(targdb) or ignore_missing: + obsdb = ObservationsDatabase(dbdir, purge_records=purge_records) + flist = glob.glob(os.path.join(srcdir, 'observations_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if obsdb.mergeObsDatabase(fl): + os.remove(fl) + obsdb.closeObsDatabase() + + targdb = os.path.join(dbdir, 'trajectories.db') + if os.path.isfile(targdb) or ignore_missing: + trajdb = TrajectoryDatabase(dbdir, purge_records=purge_records) + flist = glob.glob(os.path.join(srcdir, 'trajectories_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if trajdb.mergeTrajDatabase(fl): + os.remove(fl) + + # get the latest date in the database for use below + cur = trajdb.dbhandle.execute('select max(jdt_ref) from trajectories where status=1') + vals = cur.fetchall() + jdt_end = float(vals[0][0]) + trajdb.closeTrajDatabase() + + # find and remove duplicates using WMPL's built-in routine to clean up the trajectory DB + # NB: can only be run on the calcserver where trajectory folders are present + + # select range midday on the latest date to matchstart days earlier + dt_end = jd2Date(int(jdt_end) + 1, dt_obj=True, tzinfo=datetime.timezone.utc) + dt_beg = jd2Date(int(jdt_end) + 1 - matchstart, dt_obj=True, tzinfo=datetime.timezone.utc) + event_time_range = [dt_beg, dt_end] + + dh = RMSDataHandle(basedir, dt_range=event_time_range, db_dir=dbdir, output_dir=basedir,mcmode=1, archivemonths=0) + dh.updateTrajectoryDatabase(dt_range=event_time_range) + + # should never need to run this part as the candidates are all made in one place + targdb = os.path.join(dbdir, 'candidates.db') + if os.path.isfile(targdb): + canddb = CandidateDatabase(dbdir) + flist = glob.glob(os.path.join(srcdir, 'candidates_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if canddb.mergeCandDatabase(fl): + os.remove(fl) + canddb.closeCandDatabase() + + print('done') if __name__ == '__main__': if len(sys.argv) < 3: - print('usage: consolidateDistTraj folder_containing_srcdbs full_path_to_targb') + print('usage: consolidateDistTraj folder_containing_srcdbs targ_dbdir') exit(0) srcdir = sys.argv[1] - masterdb = sys.argv[2] - if len(sys.argv) > 3: - rundt = sys.argv[3] - else: - rundt = datetime.datetime.now().strftime('%Y%m%d') - # real path to the trajectories as per the master database - masterpth = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' - - flist = glob.glob1(srcdir, f'{rundt}*.json') - flist.sort() - mastdb = None - for fl in flist: - #countDataInDb(os.path.join(srcdir, fl)) - tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') - print(f'{tstamp} processing {fl}') - sys.stdout.flush() - mastdb = mergeDatabases(srcdir, fl, masterpth, masterdb, mastdb) - - print('patching path in mastdb') - patchTrajDB(masterdb, masterpth, oldstr='/home/ec2-user/prod/data/distrib') + dbdir = sys.argv[2] + basedir = os.path.dirname(os.path.normpath(dbdir)) + mergeDatabases(srcdir, dbdir, basedir) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 97555ca96..d3b6c99c7 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -6,21 +6,8 @@ import os import sys import datetime -import json - -# read the task template to determine the paths to write any data to -# -def getTrajsolverPaths(): - templdir,_ = os.path.split(__file__) - with open(os.path.join(templdir, 'taskrunner.json'), 'r') as inf: - taskdets = json.load(inf) - taskenv = taskdets['overrides']['containerOverrides'][0]['environment'] - srcpath = [x for x in taskenv if x['name']=='SRCPATH'][0]['value'] # path from which trajsolver will consume candidates - outpath = [x for x in taskenv if x['name']=='OUTPATH'][0]['value'] # path to which trajsolver will publish trajectories - webpath = [x for x in taskenv if x['name']=='WEBPATH'][0]['value'] # web loc to which trajsolver will publish trajectories - - return srcpath, outpath, webpath +from traj.distributeCandidates import getTrajsolverPaths # make sure the local trajectories folder is synced with the master copy @@ -36,7 +23,7 @@ def refreshTrajectories(outf, matchstart, matchend, trajpath): return -# make sure the master copy is updated with any new locally updated trajectories +# make sure the shared bucket is updated with any new locally updated trajectories # def pushUpdatedTrajectoriesShared(outf, matchstart, matchend, targpath): for d in range(matchend, matchstart+1): @@ -46,15 +33,17 @@ def pushUpdatedTrajectoriesShared(outf, matchstart, matchend, targpath): ymd=thisdt.strftime('%Y%m%d') trajloc=f'trajectories/{yr}/{ym}/{ymd}' outf.write(f'if [ -d {trajloc} ] ; then \n') - outf.write(f'aws s3 sync {trajloc} {targpath}/matches/RMSCorrelate/{trajloc} --exclude "*" --include "*.pickle" --include "*report.txt" --quiet\n') - outf.write(f'aws s3 sync {trajloc}/plots {targpath}/matches/RMSCorrelate/{trajloc}/plots --quiet\n') + outf.write(f'aws s3 sync {trajloc} {targpath}/{trajloc} --exclude "*" --include "*.pickle" --include "*report.txt" --quiet\n') + outf.write(f'if [ -d {trajloc}/plots ] ; then \n') + outf.write(f'aws s3 sync {trajloc}/plots {targpath}/{trajloc}/plots --quiet\n') + outf.write('fi\n') outf.write('fi\n') - outf.write(f'aws s3 sync trajectories/{yr}/plots {targpath}/matches/RMSCorrelate/trajectories/{yr}/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {targpath}/matches/RMSCorrelate/trajectories/{yr}/{ym}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/plots {targpath}/trajectories/{yr}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {targpath}/trajectories/{yr}/{ym}/plots --quiet\n') return -# make sure the master copy is updated with any new locally updated trajectories +# make sure the website is updated with any new locally updated trajectories # def pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath): for d in range(matchend, matchstart+1): @@ -63,16 +52,18 @@ def pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath): ym=thisdt.strftime('%Y%m') ymd=thisdt.strftime('%Y%m%d') trajloc=f'trajectories/{yr}/{ym}/{ymd}' - targloc=f'reports/{yr}/orbits/{ym}/{ymd}' + targloc=f'{yr}/orbits/{ym}/{ymd}' outf.write(f'if [ -d {trajloc} ] ; then \n') outf.write(f'aws s3 sync {trajloc} {webpath}/{targloc} --quiet\n') outf.write('fi\n') outf.write(f'aws s3 sync {trajloc}/plots {webpath}/{targloc}/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/plots {webpath}/reports/{yr}/orbits/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {webpath}/reports/{yr}/orbits/{ym}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/plots {webpath}/{yr}/orbits/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {webpath}/{yr}/orbits/{ym}/plots --quiet\n') return +# Create the density plots showing meteor showers +# def createDensityPlots(outf, calcdir, enddt, includeyear=True): yr = enddt.year ym = enddt.strftime('%Y%m') @@ -98,18 +89,24 @@ def createDensityPlots(outf, calcdir, enddt, includeyear=True): return -def SyncRawData(outf, matchstart, matchend, shbucket, calcdir): +# Sync the raw camera data from shared storage to the local disk +# +def SyncRawData(outf, matchstart, matchend, shbucket): # camera data - no need to replicate it for an historical date - outf.write(f'targdirs=$(aws s3 ls {shbucket}/matches/RMSCorrelate/ | egrep -v "traj|daily|test|plot|proce"|grep PRE | awk \'{{print $2}}\')\n') + outf.write(f'targdirs=$(aws s3 ls {shbucket}/ | egrep -v "traj|daily|test|plot|proce|dbs"|grep PRE | awk \'{{print $2}}\')\n') outf.write('for td in $targdirs ; do\n') for d in range(matchend+1, matchstart+2): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) trgdy=thisdt.strftime('%Y%m%d') - outf.write(f' aws s3 sync {shbucket}/matches/RMSCorrelate/$td {calcdir}/$td --exclude "*" --include "${{td:0:6}}_{trgdy}*" --quiet\n') + outf.write(f' aws s3 sync {shbucket}/$td ./$td --exclude "*" --include "${{td:0:6}}_{trgdy}*" --quiet\n') outf.write('done\n') return +# +# Get a list of images that are used by the solutions +# + def gatherUsedImageList(outf, matchstart, matchend, shbucket): for d in range(matchend, matchstart+1): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) @@ -117,67 +114,85 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): mth = thisdt.month dy = thisdt.day trajloc = f'trajectories/{yr}/{yr}{mth:02d}/{yr}{mth:02d}{dy:02d}' - out_dir = '/home/ec2-user/data/distrib' + out_dir = '~/data/distrib' outf.write(f'python -c "from traj.pickleAnalyser import getAllImages;getAllImages(\'{trajloc}\', \'{out_dir}\');"\n') - outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt"\n') + outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt --quiet"\n') outf.write(f'rm {out_dir}/consumed_*.txt\n') return -def createExecConsolSh(matchstart, matchend, execconsolsh): - shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! - _, outpath, _ = getTrajsolverPaths() +# +# Create the bash script that consolidates the generated data and makes sure the website and shared area are updated + +def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): + + istest = True if istest.lower()=='true' else False + print(f'istest is {istest}') + + srcpath, shbucket, webbucket = getTrajsolverPaths(istest=istest) + csuser = os.getenv('SERVERUSERID', default='ec2-user') + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) + includeyear = False + if enddt.day == 30: + includeyear = True with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - # outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') + outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execConsol start\n') + outf.write(f'aws s3 sync {srcpath}/ ~/data/distrib/canddbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --quiet\n') - outf.write('python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json\n') + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/canddbs/ {calcdir}/dbs/\n') + + outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') - refreshTrajectories(outf, matchstart, matchend, outpath) + refreshTrajectories(outf, matchstart, matchend, shbucket) outf.write('logger -s -t execConsol creating density plots\n') - createDensityPlots(outf, calcdir, enddt) + createDensityPlots(outf, calcdir, enddt, includeyear) outf.write('logger -s -t execConsol pushing data back to S3\n') pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket) pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) outf.write('logger -s -t execConsol getting the image list\n') gatherUsedImageList(outf, matchstart, matchend, shbucket) - #outf.write('unset AWS_PROFILE\n') + outf.write('logger -s -t execConsol done\n') return +# +# Create a bash script to replot the density charts if needed + + +def createExecReplotSh(matchstart, matchend, execconsolsh, istest=''): + + istest = True if istest.lower()=='true' else False -def createExecReplotSh(matchstart, matchend, execconsolsh): shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! - _, outpath, _ = getTrajsolverPaths() + csuser = os.getenv('SERVERUSERID', default='ec2-user') + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + _, outpath, _ = getTrajsolverPaths(istest=istest) + enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - #outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execReplot start\n') refreshTrajectories(outf, matchstart, matchend, outpath) createDensityPlots(outf, calcdir, enddt, False) gatherUsedImageList(outf, matchstart, matchend, shbucket) - #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execReplot done\n') return -def createDistribMatchingSh(matchstart, matchend, execmatchingsh): - shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') +def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): + csuser = os.getenv('SERVERUSERID', default='ec2-user') startdt = datetime.datetime.now() + datetime.timedelta(days=-matchstart) enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) @@ -186,66 +201,63 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh): enddtstr = enddt.strftime('%Y%m%d-080000') rundatestr = enddt.strftime('%Y%m%d') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + + _, outpath, webpath = getTrajsolverPaths(istest=istest) - srcpath, outpath, _ = getTrajsolverPaths() + srcpath = os.getenv('UKMONSHAREDBUCKET') + '/matches/RMSCorrelate' with open(execmatchingsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - #outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') outf.write(f'cd {calcdir}\n') outf.write('df -h . \n') # fetch anything thats new from S3 + outf.write('logger -s -t execdistrib syncing latest trajectories from shared S3\n') + # in test, we seeded the test area with prod trajectories in April 2026 refreshTrajectories(outf, matchstart, matchend, outpath) outf.write('logger -s -t execdistrib syncing the raw data from shared S3\n') - outf.write(f'aws s3 cp {srcpath}/processed_trajectories.json {calcdir}/processed_trajectories.json --quiet\n') - outf.write(f'ls -ltr {calcdir}/*.json\n') - - SyncRawData(outf, matchstart, matchend, shbucket, calcdir) + SyncRawData(outf, matchstart, matchend, srcpath) outf.write('logger -s -t execdistrib starting correlator to update existing matches and create candidates\n') - outf.write(f'mkdir -p {calcdir}/candidates\n') - outf.write(f'rm {calcdir}/candidates/*.pickle >/dev/null 2>&1\n') - outf.write(f'time python -m wmpl.Trajectory.CorrelateRMS {calcdir} -i 1 -l -r \"({startdtstr},{enddtstr})\"\n') + outf.write('mkdir -p ./candidates/processed\n') + outf.write('rm ./candidates/*.pickle >/dev/null 2>&1\n') + outf.write(f'time python -m wmpl.Trajectory.CorrelateRMS . --dbdir ./dbs --logdir ./logs --mcmode 4 -l -r \"({startdtstr},{enddtstr})\"\n') # backup the raw candidates in case i need to reprocess some by hand - outf.write(f'mkdir -p {calcdir}/candidates/bkp\n') - outf.write(f'tar cvfz {calcdir}/candidates/bkp/{rundatestr}.tgz {calcdir}/candidates/*.pickle\n') - outf.write(f'find {calcdir}/candidates/bkp/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') + outf.write(f'tar czf ./candidates/processed/{rundatestr}.tgz ./candidates/*.pickle\n') + outf.write('find ./candidates/processed/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') + outf.write('find ./logs/ -mtime +28 -exec rm -f ' + '{} \\;\n') outf.write('logger -s -t execdistrib backing up the database to trajdb\n') - outf.write(f'cp {calcdir}/processed_trajectories.json {calcdir}/trajdb/processed_trajectories.json.{rundatestr}\n') - - outf.write('logger -s -t execdistrib Syncing the database back to shared S3\n') - outf.write(f'if [ -s {calcdir}/processed_trajectories.json ] ; then\n') - outf.write(f'aws s3 cp {calcdir}/processed_trajectories.json {srcpath}/processed_trajectories.json --quiet\n') - outf.write('else echo "bad database file" ; fi \n') + outf.write(f'tar cvzf ./trajdb/databases_{rundatestr}.tgz dbs/observations.db dbs/trajectories.db dbs/candidates.db\n') + outf.write('find ./trajdb/ -name "*" -mtime +14 -exec rm -f ' + '{} \\;\n') outf.write('logger -s -t execdistrib distributing candidates and launching containers\n') - outf.write(f'time python -m traj.distributeCandidates {rundatestr} {calcdir}/candidates {srcpath}\n') + outf.write(f'time python -m traj.distributeCandidates {rundatestr} ./candidates {istest}\n') # do this again to fetch todays results outf.write('logger -s -t execdistrib refetch latest trajectories\n') refreshTrajectories(outf, matchstart, matchend, outpath) - - outf.write('logger -s -t execdistrib and sync the back to S3 as well\n') - pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket) - pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) + + outf.write('logger -s -t execdistrib and sync back to S3 as well\n') + pushUpdatedTrajectoriesShared(outf, matchstart, matchend, outpath) + pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath) - #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execdistrib done\n') if __name__ == '__main__': if len(sys.argv) < 4: - print('Usage: createDistribMatchingSh day1 day2 outfile') + print('Usage: createDistribMatchingSh day1 day2 outfile optional_istest') exit(1) matchstart = int(sys.argv[1]) matchend = int(sys.argv[2]) outfname = sys.argv[3] + if len(sys.argv) > 4: + istest = True if sys.argv[4].lower()=='true' else False - createDistribMatchingSh(matchstart, matchend, outfname) + createDistribMatchingSh(matchstart, matchend, outfname, istest) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 3030c9b08..e81d2f1e2 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -3,6 +3,8 @@ # Used when running the correlator in 'distributed' mode. # Pick up new groups of candidate trajectories and distribute them, # then wait for completion and gather the logs. + +# NB this runs on the calculation engine server # import json @@ -16,13 +18,30 @@ import pickle -def getClusterDetails(templdir): - sts = boto3.client('sts') - accid = sts.get_caller_identity()['Account'] - if accid == '183798037734': - clusdetails = os.path.join(templdir, 'clusdetails-mda.txt') +# read the task template to determine the paths to write any data to +# +def getTrajsolverPaths(istest=False): + templdir,_ = os.path.split(__file__) + if istest: + taskjson = 'taskrunner_test.json' + else: + taskjson = 'taskrunner.json' + with open(os.path.join(templdir, taskjson), 'r') as inf: + taskdets = json.load(inf) + taskenv = taskdets['overrides']['containerOverrides'][0]['environment'] + srcpath = [x for x in taskenv if x['name']=='SRCPATH'][0]['value'] # path from which trajsolver will consume candidates + outpath = [x for x in taskenv if x['name']=='OUTPATH'][0]['value'] # path to which trajsolver will publish trajectories + webpath = [x for x in taskenv if x['name']=='WEBPATH'][0]['value'] # web loc to which trajsolver will publish trajectories + + return srcpath, outpath, webpath + + +def getClusterDetails(istest=False): + templdir,_ = os.path.split(__file__) + if istest: + clusdetails = os.path.join(templdir, 'clusdetailstest-ukmda.txt') else: - clusdetails = os.path.join(templdir, 'clusdetails-mm.txt') + clusdetails = os.path.join(templdir, 'clusdetails-mda.txt') with open(clusdetails) as inf: lis = inf.readlines() clusname = lis[0].strip() @@ -34,12 +53,16 @@ def getClusterDetails(templdir): return [clusname, secgrp, subnet, exrolearn, loggrp, contname] -def createTaskTemplate(rundate, buckname, clusdets, spandays=3): +def createTaskTemplate(rundate, buckname, clusdets, spandays=3, istest=False): d1 = (rundate + datetime.timedelta(days = -(spandays-1))).strftime('%Y%m%d') d2 = (rundate + datetime.timedelta(days = 1)).strftime('%Y%m%d') templdir,_ = os.path.split(__file__) - templatefile = os.path.join(templdir, 'taskrunner.json') + if istest: + taskjson = 'taskrunner_test.json' + else: + taskjson = 'taskrunner.json' + templatefile = os.path.join(templdir, taskjson) clusname = clusdets[0] secgrp = clusdets[1] subnet = clusdets[2] @@ -65,16 +88,26 @@ def getDebugStatus(): return False -def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): +def distributeCandidates(rundate, srcdir, maxcount=20, istest=False): + + clusdets = getClusterDetails(istest=istest) + # the target for distribution is the source for the trajsolver + targdir, _, _ = getTrajsolverPaths(istest=istest) clusname = clusdets[0] + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] + ecsclient = boto3.client('ecs', region_name='eu-west-2') status = ecsclient.describe_clusters(clusters=[clusname]) if len(status['clusters']) == 0: print('cluster not running!') return False + print(clusdets, targdir) + print(f'Reading from {srcdir}') # obtain a list of picklefiles and sort by name flist = glob.glob1(srcdir, '*.pickle') @@ -85,12 +118,10 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): # work out how many buckets i need numcands = len(flist) + print(f'processing {numcands} candidates') numbucks = int(math.ceil(numcands/maxcount)) # create buckets - targdir = targdir[5:] - outbucket=targdir[:targdir.find('/')] - targdir = targdir[targdir.find('/')+1:] buckroot = os.path.join(targdir, rundate.strftime('%Y%m%d')) taskarns = [None] * numbucks @@ -101,14 +132,14 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): s3 = boto3.client('s3') for i in range(0, numbucks): - bucknames[i] = buckroot + f'_{i:02d}' + bucknames[i] = buckroot + f'_{i:03d}' filelist = flist[i::numbucks] for fli in filelist: src = os.path.join(srcdir, fli) dst = os.path.join(bucknames[i], fli) s3.upload_file(src, outbucket, dst) - taskjson = createTaskTemplate(rundate, bucknames[i], clusdets) + taskjson = createTaskTemplate(rundate, bucknames[i], clusdets, istest=istest) response = ecsclient.run_task(**taskjson) while len(response['tasks']) == 0: @@ -117,7 +148,7 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): taskarn = response['tasks'][0]['taskArn'] taskarns[i] = taskarn jsontempls[i] = taskjson - print(taskarn[51:]) + print(taskarn.split('/')[-1]) if isDbg is True and i == 3: break @@ -143,23 +174,31 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): return True -def monitorProgress(rundate): +def monitorProgress(rundatestr, istest=''): + + istest = False if (istest=='' or istest.lower() == 'false') else True client = boto3.client('ecs', region_name='eu-west-2') s3 = boto3.client('s3') - archbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - templdir,_ = os.path.split(__file__) - clusdets = getClusterDetails(templdir) + clusdets = getClusterDetails(istest=istest) + targdir, _, _ = getTrajsolverPaths(istest=istest) + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] # load the buckets, tasks and cluster name from the dump file - rundate = datetime.datetime.strptime(rundate, '%Y%m%d') - picklefile = os.path.join(datadir, 'distrib', rundate.strftime('%Y%m%d') + '.pickle') - rempickle = f"matches/distrib/{rundate.strftime('%Y%m%d')}.pickle" + rundate = datetime.datetime.strptime(rundatestr, '%Y%m%d') + if istest: + picklefile = os.path.join(datadir, 'distrib', 'test', rundatestr + '.pickle') + else: + picklefile = os.path.join(datadir, 'distrib', rundatestr + '.pickle') + rempickle = f"{targdir}/{rundatestr}.pickle" + os.makedirs(os.path.join(datadir, 'distrib', 'test'), exist_ok=True) try: - s3.download_file(archbucket, rempickle, picklefile) + s3.download_file(outbucket, rempickle, picklefile) except: - print('no containers to monitor') + print(f'no containers to monitor in {outbucket}/{rempickle}') return dumpdata = pickle.load(open(picklefile,'rb')) bucknames = dumpdata[0] @@ -190,59 +229,49 @@ def monitorProgress(rundate): thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) + taskid = tsk["arn"].split('/')[-1] try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {tsk["arn"][51:]} completed already') + print(f'{thisbuck}: task {taskid} completed already') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + for tsk in sts['tasks']: - print(f'checking {tsk["taskArn"][51:]}') + taskid = tsk["taskArn"].split('/')[-1] idx = taskarns.index(tsk['taskArn']) + _, thisbuck = os.path.split(bucknames[idx]) + print(f'{thisbuck}: checking {taskid} ') #print(tsk['taskArn'], tsk['lastStatus']) if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': # retry the task - thisjson = createTaskTemplate(rundate, bucknames[idx]) + thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] - print('task restarted') + print(f'{thisbuck}: task restarted') else: thisarn=taskarns.pop(idx) thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print('task completed') + print(f'{thisbuck}: task completed') # collect the logs from CloudWatch - realfname=None - logdir = os.path.join(datadir, '..', 'logs', 'distrib') - os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + if len(taskarns) > 99: sts = client.describe_tasks(cluster=clusname, tasks=taskarns[99:199]) for tsk in sts['failures']: @@ -252,87 +281,92 @@ def monitorProgress(rundate): thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) + taskid = tsk["arn"].split('/')[-1] try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {tsk["arn"][51:]} completed already') + print(f'{thisbuck}: task {taskid} completed already') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + for tsk in sts['tasks']: - print(f'checking {tsk["taskArn"][51:]}') + taskid = tsk["taskArn"].split('/')[-1] idx = taskarns.index(tsk['taskArn']) + _, thisbuck = os.path.split(bucknames[idx]) + print(f'{thisbuck} : checking {taskid} ') if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': # retry the task - thisjson = createTaskTemplate(rundate, bucknames[idx]) + thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] - print('task restarted') + print(f'{thisbuck}: task restarted') else: thisarn=taskarns.pop(idx) thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print('task completed') - - # collect the logs from CloudWatch - realfname=None - logdir = os.path.join(datadir, '..', 'logs', 'distrib') - os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + print(f'{thisbuck}: task completed') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + if taskcount > 0: # wait 60s before checking again - print('sleeping for 60s') - time.sleep(60.0) + print('sleeping for 30s') + time.sleep(30.0) return -def getMissedLogs(picklefile): +def getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir): + # collect the logs from CloudWatch + realfname=None + logdir = os.path.join(datadir, '..', 'logs', 'distrib') + os.makedirs(logdir, exist_ok=True) + tmpfname = os.path.join(logdir, f'{thisarn.split("/")[-1]}.log') + with open(tmpfname, 'w') as outf: + for events in getLogDetails(loggrp, thisarn.split("/")[-1], contname): + for evt in events: + evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) + msg = evt['message'] + outf.write(f'{evtdt} {msg}\n') + if msg[:10] == 'processing': + realfname = msg[11:].strip() + if realfname: + locname = os.path.join(logdir, f'{realfname}.log') + os.rename(tmpfname, locname) + remlog = f'{targdir}/logs/{realfname}.log' + s3.upload_file(Filename=locname, Bucket=outbucket, Key=remlog) + return + + +def getMissedLogs(picklefile, istest=False): + + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) s3 = boto3.client('s3') - archbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] + _, targdir, _ = getTrajsolverPaths(istest=istest) + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] + + clusdets = getClusterDetails(istest=istest) + contname = clusdets[5] + dumpdata = pickle.load(open(picklefile,'rb')) taskarns = dumpdata[1] - loggrp = '/ecs/trajcont' - contname = 'trajcont' - logdir = '/home/ec2-user/prod/logs/distrib' + loggrp = '/ecs/trajcont' for thisarn in taskarns: - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): @@ -341,39 +375,43 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): """ client = boto3.client('logs', region_name=region_name) + logstreamname = f'ecs/{contname}/{thisarn}' # first request - response = client.get_log_events( - logGroupName=loggrp, - logStreamName=f'ecs/{contname}/{thisarn}', - startFromHead=True) - yield response['events'] - - # second and later - while True: - prev_token = response['nextForwardToken'] + #print(f'looking for {logstreamname} in {loggrp}') + try: response = client.get_log_events( logGroupName=loggrp, - logStreamName=f'ecs/{contname}/{thisarn}', - nextToken=prev_token) - # same token then break - if response['nextForwardToken'] == prev_token: - break + logStreamName=logstreamname, + startFromHead=True) yield response['events'] + # second and later + while True: + prev_token = response['nextForwardToken'] + response = client.get_log_events( + logGroupName=loggrp, + logStreamName=logstreamname, + nextToken=prev_token) + # same token then break + if response['nextForwardToken'] == prev_token: + break + yield response['events'] + except Exception: + dtval = int(datetime.datetime.now().timestamp() * 1000) + msg = f'log for {thisarn} not available' + yield [{'timestamp': dtval, 'message': msg}] + if __name__ == '__main__': - if len(sys.argv) < 4: - rundt = datetime.datetime(2022,4,21) - srcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates' # hardcoded on calcserver - buck = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - targdir = f'{buck}/matches/distrib' + if len(sys.argv) < 3: + print('usage: python distributeCandidates yyyymmdd ./candidates istest\n') + print(' optional istest if True then test mode used') else: rundt = datetime.datetime.strptime(sys.argv[1], '%Y%m%d') srcdir = sys.argv[2] - targdir = sys.argv[3] - - templdir,_ = os.path.split(__file__) - clusdets = getClusterDetails(templdir) - print(clusdets) - distributeCandidates(rundt, srcdir, targdir, clusdets) + if len(sys.argv)>3: + istest = True if sys.argv[3].lower()=='true' else False + else: + istest = False + distributeCandidates(rundt, srcdir, istest=istest) diff --git a/archive/ukmon_pylib/traj/jsonDbMaintenance.py b/archive/ukmon_pylib/traj/jsonDbMaintenance.py deleted file mode 100644 index e5aa984c3..000000000 --- a/archive/ukmon_pylib/traj/jsonDbMaintenance.py +++ /dev/null @@ -1,72 +0,0 @@ -# small script to archive off old data from the JSON database -import datetime -import os -import sys -from dateutil.relativedelta import relativedelta - -from wmpl.Utils.TrajConversions import datetime2JD, jd2Date -from wmpl.Trajectory.CorrelateRMS import DatabaseJSON - -# Name of json file with the list of processed directories -JSON_DB_NAME = "processed_trajectories.json" - - -def archiveOldRecords(db, db_dir, older_than=3): - """ - Archive off old records to keep the database size down - - Keyword Arguments: - older_than: [int] number of months to keep, default 3 - """ - class DummyMetObs(): - def __init__(self, station, obs_id): - self.station_code = station - self.id = obs_id - - archdate = datetime.datetime.now(datetime.timezone.utc) - relativedelta(months=older_than) - archdate_jd = datetime2JD(archdate) - - arch_db_path = os.path.join(db_dir, 'archive', f'{archdate.strftime("%Y%m")}_{JSON_DB_NAME}') - os.makedirs(os.path.join(db_dir, 'archive'), exist_ok=True) - archdb = DatabaseJSON(arch_db_path) - - for traj in [t for t in db.trajectories if t < archdate_jd]: - if traj < archdate_jd: - archdb.addTrajectory(None, db.trajectories[traj], False) - db.removeTrajectory(db.trajectories[traj]) - - for traj in [t for t in db.failed_trajectories if t < archdate_jd]: - if traj < archdate_jd: - archdb.addTrajectory(None, db.failed_trajectories[traj], True) - db.removeTrajectory(db.failed_trajectories[traj]) - - for station in db.processed_dirs: - arch_processed = [dirname for dirname in db.processed_dirs[station] if - datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for dirname in arch_processed: - archdb.addProcessedDir(station, dirname) - db.processed_dirs[station].remove(dirname) - - for station in db.paired_obs: - arch_processed = [obs_id for obs_id in db.paired_obs[station] if - datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for obs_id in arch_processed: - archdb.addPairedObservation(DummyMetObs(station, obs_id)) - db.paired_obs[station].remove(obs_id) - - archdb.save() - db.save() - return - - -if __name__ == '__main__': - db_dir = sys.argv[1] - database_path = os.path.join(db_dir, JSON_DB_NAME) - db = DatabaseJSON(database_path) - soonest = jd2Date(min(min(list(db.trajectories.keys())), min(list(db.failed_trajectories.keys()))), dt_obj=True) - - nowdt = datetime.datetime.now() - mthsback = int((nowdt - soonest).days/30) - for i in range(mthsback, 2, -1): - print(f'archiving {i} months back') - archiveOldRecords(db, db_dir, i) diff --git a/archive/ukmon_pylib/traj/manualBulkReruns.py b/archive/ukmon_pylib/traj/manualBulkReruns.py index d94510575..497310da0 100644 --- a/archive/ukmon_pylib/traj/manualBulkReruns.py +++ b/archive/ukmon_pylib/traj/manualBulkReruns.py @@ -406,7 +406,7 @@ def prepDataForRange(startdt, enddt): # working out what has been used, then collecting the FTPs and platepars, # and finally removing the already-used meteors from the FTPs # - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) sdt = datetime.datetime.strptime(startdt, '%Y%m') edt = datetime.datetime.strptime(enddt, '%Y%m') smth = sdt.month @@ -430,7 +430,7 @@ def prepDataForRange(startdt, enddt): def moveInterestingCands(): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) pickledir = os.path.join(datadir, 'historic', 'candidates') goodcandr = os.path.join(datadir, 'historic', 'goodcands') os.makedirs(goodcandr, exist_ok=True) diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index ddfe6fbc6..5afb69442 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -122,6 +122,8 @@ def getShowerDets(shwr): def createUFOOrbitFile(traj, outdir, amag, mass, shower_obj): + outdir = os.path.expanduser(outdir) + #print('Creating UFO Orbit style output file') orb = traj.orbit if shower_obj is None: @@ -378,6 +380,7 @@ def computeAbsoluteMagnitudes(traj, meteor_list): def draw3Dmap(traj, outdir): + outdir = os.path.expanduser(outdir) lats = [] lons = [] alts = [] @@ -493,6 +496,8 @@ def calcAdditionalValues(traj): def createAdditionalOutput(traj, outdir): + outdir = os.path.expanduser(outdir) + # calculate the values amag, vmag, mass, id, cod, shwrname, orb, shower_obj, lg, bg, vg, _ = calcAdditionalValues(traj) @@ -582,16 +587,18 @@ def getListOfImages(picklename): class TrajQualityParams(object): def __init__(self): - self. min_traj_points = 6 - self. min_qc = 5.0 - self. max_e = 1.5 - self. max_radiant_err = 2.0 - self. max_vg_err = 10.0 - self. max_begin_ht = 160 - self. min_end_ht = 20 - - -def getAllImages(dir_path, out_path): + self.min_traj_points = 6 + self.min_qc = 5.0 + self.max_e = 1.5 + self.max_radiant_err = 2.0 + self.max_vg_err = 10.0 + self.max_vg = 120.0 + self.max_begin_ht = 160 + self.min_end_ht = 20 + + +def getAllImages(dir_path, outdir): + outdir = os.path.expanduser(outdir) traj_quality_params = TrajQualityParams() trajs = loadTrajectoryPickles(dir_path, traj_quality_params, verbose=True) imglist = [] @@ -604,6 +611,6 @@ def getAllImages(dir_path, out_path): except Exception: pass outfname = os.path.split(dir_path)[1] - with open(os.path.join(out_path, f'consumed_{outfname}.txt'), 'w') as outf: + with open(os.path.join(outdir, f'consumed_{outfname}.txt'), 'w') as outf: for img in imglist: outf.write(f'{img}\n') diff --git a/archive/ukmon_pylib/traj/taskrunner.json b/archive/ukmon_pylib/traj/taskrunner.json index 630b61384..be99acf10 100644 --- a/archive/ukmon_pylib/traj/taskrunner.json +++ b/archive/ukmon_pylib/traj/taskrunner.json @@ -29,6 +29,10 @@ { "name": "WEBPATH", "value": "s3://ukmda-website/reports" + }, + { + "name": "MAX_STNS", + "value": "9999" } ] } diff --git a/archive/ukmon_pylib/traj/taskrunner_test.json b/archive/ukmon_pylib/traj/taskrunner_test.json new file mode 100644 index 000000000..d834b6e2f --- /dev/null +++ b/archive/ukmon_pylib/traj/taskrunner_test.json @@ -0,0 +1,48 @@ +{ + "cluster": "trajsolvertest", + "count": 1, + "enableECSManagedTags": true, + "enableExecuteCommand": true, + "group": "trajsolvergrp", + "launchType": "FARGATE", + "networkConfiguration": { + "awsvpcConfiguration": { + "subnets":[], + "securityGroups": [], + "assignPublicIp": "ENABLED" + } + }, + "overrides": { + "containerOverrides": [ + { + "name": "trajconttest", + "command": [], + "environment": [ + { + "name": "SRCPATH", + "value": "s3://ukmda-shared/test/matches/distrib" + }, + { + "name": "OUTPATH", + "value": "s3://ukmda-shared/test/matches/RMSCorrelate" + }, + { + "name": "WEBPATH", + "value": "s3://ukmda-website/dummy" + }, + { + "name": "MAX_STNS", + "value": "9999" + } + ] + } + ], + "executionRoleArn": "", + "taskRoleArn": "" + }, + "platformVersion": "LATEST", + "propagateTags": "TASK_DEFINITION", + "referenceId": "", + "startedBy": "cli", + "taskDefinition": "trajsolvertest" +} diff --git a/archive/usermgmt/addAdminUser.sh b/archive/usermgmt/addAdminUser.sh deleted file mode 100644 index eb02ec10c..000000000 --- a/archive/usermgmt/addAdminUser.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# simple script to create a new user-maintenance account -# and save the access key locally. Can only be used by an account with IAM Admin permissions. -$username = $args[0] -aws iam create-user --user-name $username --profile ukmda_admin -aws iam add-user-to-group --user-name $username --user-group Administrators --profile ukmda_admin diff --git a/archive/usermgmt/addFBApiKey.sh b/archive/usermgmt/addFBApiKey.sh deleted file mode 100644 index eb957f893..000000000 --- a/archive/usermgmt/addFBApiKey.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# copyright Mark McIntyre, 2024- - -# script to add an api key for a user - -$username = $1 -aws apigateway create-api-key --name "$username" --region eu-west-1 --description "key for $username" --enabled --profile ukmda_admin \ No newline at end of file diff --git a/archive/usermgmt/addSftpUser.sh b/archive/usermgmt/addSftpUser.sh deleted file mode 100644 index 82fdcbd26..000000000 --- a/archive/usermgmt/addSftpUser.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# bash script to create new sftp user -# -userid=$1 -shortid=$2 -updatemode=$3 -oldloc=$4 - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -cd $here - -if [ -f ../config.ini ] ; then - source ~/dev/config.ini - keydir=/home/ec2-user/dev/keymgmt -else - source ~/prod/config.ini - keydir=/home/ec2-user/keymgmt - -fi -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -cd $keydir - -logger -s -t addSftpUser "adding user $userid $shortid $oldloc" - -# wait for the file uploads to complete. For some reason paramiko does this asynchronously -counter=0 -# all-lowercase versions of the names -userid="${userid,,}" -shortid_l="${shortid,,}" - -ls $keydir/$shortid_l.key > /dev/null 2>&1 -while [[ $? -ne 0 && $counter -ne 5 ]] ; do - sleep 2 - logger -s -t addSftpUser "waiting for raw key for $shortid" - ls $keydir/keys/$shortid_l.key > /dev/null 2>&1 - counter=$((counter + 1)) -done -if [ ! -f $keydir/keys/$shortid_l.key ] ; then - logger -s -t addSftpUser "missing keyfile for $shortid" - exit 1 -fi - -# add a unix user and set their homedir to /var/sftp/userid -grep -w $userid /etc/passwd -if [ $? -eq 1 ] ; then - dt=$(date +%Y-%m-%d) - logger -s -t addSftpUser "Creating unix user $userid" - sudo useradd --system --shell /usr/sbin/nologin --groups sftp --home /var/sftp/$userid --comment '${dt}' $userid - sudo mkdir /var/sftp/$userid - sudo chown root:sftp /var/sftp/$userid - sudo chmod 751 /var/sftp/$userid - # create the .ssh folder, platepar folder and empty client copy of the ini file - sudo mkdir /var/sftp/$userid/.ssh - sudo mkdir /var/sftp/$userid/platepar - sudo touch /var/sftp/$userid/ukmon.ini.client - # make these three writeable by the client - sudo chown $userid:$userid /var/sftp/$userid/platepar /var/sftp/$userid/.ssh /var/sftp/$userid/ukmon.ini.client -else - logger -s -t addSftpUser "Unix user $userid already exists" -fi -# copy the public key to the right place -logger -s -t addSftpUser "Applying the public key $userid" -dos2unix $keydir/sshkeys/$userid.pub -sudo cp $keydir/sshkeys/$userid.pub /var/sftp/$userid/.ssh/authorized_keys -sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys -sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys - -# add the key to logupload's authorized_keys file -logger -s -t addSftpUser "Adding the key to loguploads authorized_keys" -awk 1 $keydir/sshkeys/*.pub > /tmp/logul.pub -sudo cp /tmp/logul.pub /var/sftp/logupload/.ssh/authorized_keys -sudo chown logupload:logupload /var/sftp/logupload/.ssh/authorized_keys -rm /tmp/logul.pub - -# copy the ini and aws key files -logger -s -t addSftpUser "Copying the ini file and aws keyfile" -sudo cp $keydir/inifs/$userid.ini /var/sftp/$userid/ukmon.ini -sudo cp $keydir/keys/$shortid_l.key /var/sftp/$userid/live.key -sudo cp $keydir/csvkeys/${shortid}.csv /var/sftp/$userid/$userid.csv -sudo dos2unix /var/sftp/$userid/live.key -sudo dos2unix /var/sftp/$userid/ukmon.ini -sudo dos2unix /var/sftp/$userid/$userid.csv -sudo chown $userid:$userid /var/sftp/$userid/$userid.csv /var/sftp/$userid/ukmon.ini /var/sftp/$userid/live.key - -# if we are moving a station, update the old ini file -# so that the next run points to the new loc -if [ ! -z $oldloc ] ; then - logger -s -t addSftpUser "Moving $oldloc to $userid" - sudo cp /var/sftp/$oldloc/ukmon.ini /var/sftp/$oldloc/ukmon.ini.bkp - sudo cp $keydir/inifs/$userid.ini /var/sftp/$oldloc/ukmon.ini - # and copy the ssh authorized keys file - sudo cp $oldloc/.ssh/authorized_keys /var/sftp/$userid/.ssh/ - sudo chown -R $userid:$userid /var/sftp/$userid/.ssh/authorized_keys - sudo chmod 644 /var/sftp/$userid/.ssh/authorized_keys -fi - -logger -s -t addSftpUser "Finished" - -# run these as root to create the structure -# groupadd sftp -# mkdir -p /var/sftp -# chown root:root /var/sftp -# chmod 751 /var/sftp - -# Add this to /etc/ssh/sshd_config -# Match group sftp -# ChrootDirectory /var/sftp/%u -# AllowTCPForwarding no -# X11Forwarding no -# ForceCommand internal-sftp -# -# and then reload sshd -# service sshd reload diff --git a/archive/usermgmt/camTableMaintenance.py b/archive/usermgmt/camTableMaintenance.py deleted file mode 100644 index 30e27b54b..000000000 --- a/archive/usermgmt/camTableMaintenance.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Create and access dynamodb tables containing camera upload timings etc -# -# Copyright (C) 2018-2023 Mark McIntyre - -import boto3 -import os - -from stationMaint2 import addRow, loadLocationDetails - - -def addCreatedDate(): - bucket = 'ukmda-shared' - s3 = boto3.client('s3') - camdets = loadLocationDetails() - for _, cam in camdets.iterrows(): - res = s3.list_objects_v2(Bucket=bucket,Prefix=f'matches/RMSCorrelate/{cam["stationid"]}/', Delimiter='/') - if 'CommonPrefixes' in res: - earliest_fldr = os.path.split(res['CommonPrefixes'][0]['Prefix'][:-1])[1] - created = earliest_fldr.split('_')[1] - else: - created = '' - newdata = {'stationid': cam['stationid'], 'site': cam['site'], 'humanName':cam['humanName'], 'eMail': cam['eMail'], - 'direction': cam['direction'], 'camtype': cam['camtype'], 'active': cam['active'], - 'created': created, 'oldcode': cam['stationid']} - print(newdata) - addRow(newdata) - return - - -# remove a row from the table keyed on stationid adn datestamp in yyyymmdd_hhmmss format -def deleteRow(stationid, ddb=None): - if not ddb: - ddb = boto3.resource('dynamodb', region_name='eu-west-2') - table = ddb.Table('camdetails') - try: - table.delete_item(Key={'stationid': stationid}) - except Exception: - pass - return - - -def dumpCamTable(outdir, statdets=None, ddb=None, exportmindets=False): - if statdets is None: - statdets = loadLocationDetails(ddb=ddb) - statdets = statdets[statdets.active==1] - if exportmindets: - statdets = statdets[['stationid', 'eMail']] - statdets.to_csv(os.path.join(outdir,'camtable.csv'), index=False) diff --git a/archive/usermgmt/testkeys.py b/archive/usermgmt/testkeys.py deleted file mode 100644 index 398d9beed..000000000 --- a/archive/usermgmt/testkeys.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2018-2023 Mark McIntyre -# test that the keys provide read and write access - -import boto3 -import os -import sys - - -def readKeyFile(filename): - if not os.path.isfile(filename): - print('credentials file missing, cannot continue') - exit(1) - with open(filename, 'r') as fin: - lis = fin.readlines() - vals = {} - for li in lis: - if li[0]=='#': - continue - if '=' in li: - valstr = li.split(' ')[1] - data = valstr.split('=') - val = data[1].strip() - if val[0]=='"': - val = val[1:len(val)-1] - vals[data[0]] = val - if 'S3FOLDER' not in vals and 'CAMLOC' in vals: - vals['S3FOLDER'] = f'archive/{vals["CAMLOC"]}' - if 'S3FOLDER' in vals and vals['S3FOLDER'][-1] == '/': - vals['S3FOLDER'] = vals['S3FOLDER'][:-1] - if 'ARCHBUCKET' not in vals: - vals['ARCHBUCKET'] = 'ukmda-shared' - if 'LIVEBUCKET' not in vals: - vals['LIVEBUCKET'] = 'ukmda-live' - if 'WEBBUCKET' not in vals: - vals['WEBBUCKET'] = 'ukmda-website' - if 'ARCHREGION' not in vals: - vals['ARCHREGION'] = 'eu-west-2' - if 'LIVEREGION' not in vals: - vals['LIVEREGION'] = 'eu-west-1' - if 'MATCHDIR' not in vals: - vals['MATCHDIR'] = 'matches/RMSCorrelate' - #print(vals) - return vals - - -def uploadTest(keyfile): - myloc = os.path.split(os.path.abspath(__file__))[0] - filename = os.path.join(myloc, 'keys', keyfile) - keys = readKeyFile(filename) - target = keys['ARCHBUCKET'] - archreg = keys['ARCHREGION'] - livereg = keys['ARCHREGION'] - awskey = keys['AWS_ACCESS_KEY_ID'] - awssec = keys['AWS_SECRET_ACCESS_KEY'] - - with open('test.txt', 'w') as f: - f.write('test') - try: - conn = boto3.Session(aws_access_key_id=awskey, aws_secret_access_key=awssec, region_name=archreg) - s3 = conn.resource('s3') - target='ukmda-shared' - s3.meta.client.upload_file('test.txt', target, 'archive/test.txt') - key = {'Objects': []} - key['Objects'] = [{'Key': 'archive/test.txt'}] - s3.meta.client.delete_objects(Bucket=target, Delete=key) - print(f'{keyfile} archive test successful') - except Exception: - print(f'{keyfile} archive test FAILED') - - try: - conn = boto3.Session(aws_access_key_id=awskey, aws_secret_access_key=awssec, region_name=livereg) - s3 = conn.resource('s3') - target = 'ukmda-live' - s3.meta.client.upload_file('test.txt', target, 'test.txt') - key = {'Objects': []} - key['Objects'] = [{'Key': 'test.txt'}] - s3.meta.client.delete_objects(Bucket=target, Delete=key) - print(f'{keyfile} live test successful') - except Exception: - print(f'{keyfile} live test FAILED') - - os.remove('test.txt') - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print("usage: python testKeys.py keyname\n") - exit(1) - else: - k = sys.argv[1] - uploadTest(k) diff --git a/archive/usermgmt/updateAwsKey.sh b/archive/usermgmt/updateAwsKey.sh deleted file mode 100644 index ff508560d..000000000 --- a/archive/usermgmt/updateAwsKey.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# bash script to update the keyfile to remove plaintext AWS security details -# - -targ=$1 -force=$2 - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -if [ -f ../config.ini ] ; then - source ~/dev/config.ini - keydir=/home/ec2-user/dev/keymgmt -else - source ~/prod/config.ini - keydir=/home/ec2-user/keymgmt - -fi -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -export AWS_PROFILE=ukmonshared -python -m maintenance.getUserAndKeyInfo update $targ $force -export AWS_PROFILE= \ No newline at end of file diff --git a/archive/usermgmt/updateKeysForSplit.py b/archive/usermgmt/updateKeysForSplit.py deleted file mode 100644 index f2025a0df..000000000 --- a/archive/usermgmt/updateKeysForSplit.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import boto3 -import json -import sys -import shutil - - -def createKeyFile(livekey, archkey, location): - archbucket = os.getenv('SRCBUCKET', default='ukmda-shared') - livebucket = os.getenv('LIVEBUCKET', default='ukmda-live') - webbucket = os.getenv('WEBSITEBUCKET', default='ukmda-website') - outf = 'keys/' + location.lower() + '.key' - if archkey is None: - return outf - with open(outf, 'w') as ouf: - ouf.write(f"export AWS_ACCESS_KEY_ID={archkey['AccessKey']['AccessKeyId']}\n") - ouf.write(f"export AWS_SECRET_ACCESS_KEY={archkey['AccessKey']['SecretAccessKey']}\n") - ouf.write(f"export LIVE_ACCESS_KEY_ID={livekey['AccessKey']['AccessKeyId']}\n") - ouf.write(f"export LIVE_SECRET_ACCESS_KEY={livekey['AccessKey']['SecretAccessKey']}\n") - ouf.write('export AWS_DEFAULT_REGION=eu-west-1\n') - ouf.write(f'export CAMLOC="{location}"\n') - ouf.write(f'export S3FOLDER="archive/{location}/"\n') - ouf.write(f'export ARCHBUCKET={archbucket}\n') - ouf.write(f'export LIVEBUCKET={livebucket}\n') - ouf.write(f'export WEBBUCKET={webbucket}\n') - ouf.write('export ARCHREGION=eu-west-2\n') - ouf.write('export LIVEREGION=eu-west-1\n') - ouf.write('export MATCHDIR=matches/RMSCorrelate\n') - os.chmod(outf, 0o666) - return outf - - -def readOldKeyFile(camid): - keyf= f'/var/sftp/{camid}/live.key' - if not os.path.isfile(keyf): - print(f'invalid camid {camid}') - exit() - flines = open(keyf, 'r').readlines() - camloc = None - for li in flines: - if 'AWS_ACCESS_KEY_ID' in li: - oldkey = li.split('=')[1].strip() - elif 'AWS_SECRET_ACCESS_KEY' in li: - oldsec= li.split('=')[1].strip() - elif 'CAMLOC' in li: - camloc= li.split('=')[1].strip().strip('"') - elif 'LIVE_ACCESS_KEY_ID' in li: - print('already processed') - exit() - if camloc is None: - print('malformed key file - no camloc') - exit() - return {'AccessKey': {'AccessKeyId': oldkey, 'SecretAccessKey': oldsec}}, camloc - - -def createNewUser(location): - archuserdets = 'users/' + location + '_arch.txt' - archkeyf = 'rawkeys/arch/' + location + '.key' - archprof = os.getenv('ARCH_PROFILE', default='ukmda_admin') - archconn = boto3.Session(profile_name=archprof) - iamc = archconn.client('iam') - sts = archconn.client('sts') - acct = sts.get_caller_identity()['Account'] - policyarn = 'arn:aws:iam::' + acct + ':policy/UkmonLive' - policyarn2 = 'arn:aws:iam::' + acct + ':policy/UKMDA-shared' - try: - _ = iamc.get_user(UserName=location) - print('location exists, not adding it') - archkey = None - except Exception: - print('new location') - usr = iamc.create_user(UserName=location) - with open(archuserdets, 'w') as outf: - outf.write(str(usr)) - archkey = iamc.create_access_key(UserName=location) - with open(archkeyf, 'w') as outf: - outf.write(json.dumps(archkey, indent=4, sort_keys=True, default=str)) - with open(os.path.join('csvkeys', location + '_arch.csv'),'w') as outf: - outf.write('Access key ID,Secret access key\n') - outf.write('{},{}\n'.format(archkey['AccessKey']['AccessKeyId'], archkey['AccessKey']['SecretAccessKey'])) - _ = iamc.attach_user_policy(UserName=location, PolicyArn=policyarn) - _ = iamc.attach_user_policy(UserName=location, PolicyArn=policyarn2) - os.chmod(archuserdets, 0o666) - os.chmod(archkeyf, 0o666) - return archkey - - -def copyKeyFileToTarget(newkeyfile, camid): - targkeyf = f'/var/sftp/{camid}/live.key' - bkpkeyf = newkeyfile + '.bkp' - if not os.path.isfile(bkpkeyf): - shutil.copyfile(targkeyf, bkpkeyf) - shutil.copyfile(newkeyfile, targkeyf) - return - - -if __name__ == '__main__': - os.chdir('/home/ec2-user/keymgmt') - os.makedirs('users/', exist_ok=True) - os.makedirs('csvkeys/', exist_ok=True) - os.makedirs('rawkeys/arch/', exist_ok=True) - os.makedirs('keys/', exist_ok=True) - - camid = sys.argv[1] - print(f'processing {camid}... ', end='') - sys.stdout.flush() - oldkey, location = readOldKeyFile(camid) - archkey = createNewUser(location) - newkeyfile = createKeyFile(oldkey, archkey, location) - copyKeyFileToTarget(newkeyfile, camid) diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh new file mode 100644 index 000000000..bd0836b9a --- /dev/null +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre +# +# this script reads a list of logically-deleted trajectories from the sqlite database +# and moves the corresponding pickle and report data to a backup area +# These trajectories are generally duplicated events + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $here/../config.ini >/dev/null 2>&1 +conda activate $HOME/miniconda3/envs/${WMPL_ENV} + +cd ${DATADIR}/distrib + +startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') +jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") + +echo "checking main storage, website and csvfiles" +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + export moved=0 + trajdir=$(dirname $traj) + #echo $trajdir + srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir + trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir + aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet + export moved=1 + done + + yr=${trajdir:13:4} + trajpth=$(basename $trajdir) + webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth + aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv $webloc/$fname $newloc/$fname --quiet + export moved=1 + done + + trajpth1=${trajdir:34:19} + csvname=$(echo $trajpth1 | sed 's/_/-/g') + csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} + aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do + aws s3 mv $csvloc/$fname $newloc/$fname --quiet + export moved=1 + done + csvloc=$DATADIR/orbits/${yr}/fullcsv/processed + ls -1 $csvloc/ | grep $csvname | while read fname; do + aws s3 mv $csvloc/$fname $newloc/$fname --quiet + export moved=1 + done + [ $moved == 1 ] && echo "moved $trajpth" +done + +echo "checking consolidated matches" +lasttraj=$(sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | tail -1) +trajdir=$(dirname $lasttraj) +yr=${trajdir:13:4} +trajpth=$(basename $trajdir) +matchcsv=$DATADIR/matched/matches-full-${yr}.csv +grep $trajpth $matchcsv +if [ $? == 0 ] ; then + python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$matchcsv')" + python -m converters.toParquet $matchcsv + aws s3 sync $DATADIR/matched/ $UKMONSHAREDBUCKET/matches/matched/ --include "*" --exclude "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + aws s3 sync $DATADIR/matched/ $UKMONSHAREDBUCKET/matches/matchedpq/ --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + aws s3 sync $DATADIR/matched/ $WEBSITEBUCKET/browse/parquet/ --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + + srchcsv=$DATADIR/searchidx/${yr}-allevents.csv + python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$srchcsv')" + aws s3 sync $DATADIR/searchidx/ $WEBSITEBUCKET/search/indexes/ --exclude "*" --include "*allevents.csv" --quiet +fi + +export AWS_PROFILE=ukmonshared # needed for mariadb connection details +echo "checking Mariadb Database" +python -c "from maintenance.dataMaintenance import removeDelTrajFromDb;removeDelTrajFromDb()" +unset AWS_PROFILE diff --git a/archive/utils/getCostMetrics.sh b/archive/utils/getCostMetrics.sh index 699bb2577..99e7fc8a7 100644 --- a/archive/utils/getCostMetrics.sh +++ b/archive/utils/getCostMetrics.sh @@ -26,4 +26,4 @@ python $PYLIB/metrics/costMetrics.py . eu-west-1 $thismth #export AWS_PROFILE=realukms #python $PYLIB/metrics/costMetrics.py . eu-west-1 $thismth -export AWS_PROFILE= \ No newline at end of file +export AWS_PROFILE= diff --git a/archive/utils/loadMatchCsvMDB.sh b/archive/utils/loadMatchCsvMDB.sh index 94363039c..b2fd845f7 100644 --- a/archive/utils/loadMatchCsvMDB.sh +++ b/archive/utils/loadMatchCsvMDB.sh @@ -1,7 +1,6 @@ #!/bin/bash # Copyright (C) 2018-2023 Mark McIntyre # -# script to clear out old log files and other old data here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 diff --git a/archive/utils/loadSingleCsvMDB.sh b/archive/utils/loadSingleCsvMDB.sh index a3b9c8018..09fdaaaab 100644 --- a/archive/utils/loadSingleCsvMDB.sh +++ b/archive/utils/loadSingleCsvMDB.sh @@ -1,7 +1,6 @@ #!/bin/bash # Copyright (C) 2018-2023 Mark McIntyre # -# script to clear out old log files and other old data here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index a5687db67..a0aa9d567 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -8,6 +8,8 @@ fi RUNTIME_ENV=$1 envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') +export AWS_PROFILE=default + # read from AWS SSM Parameterset SRC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_srcdir --query Parameters[0].Value | tr -d '"') SITEURL=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_siteurl --query Parameters[0].Value | tr -d '"') @@ -20,6 +22,8 @@ SERVERINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_ BKPINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_backupinstance --query Parameters[0].Value | tr -d '"') SERVERSSHKEY=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_sshkey --query Parameters[0].Value | tr -d '"') NJLOGGRP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_batchloggroup --query Parameters[0].Value | tr -d '"') +SERVERUSERID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcuser --query Parameters[0].Value | tr -d '"') +unset AWS_PROFILE # hardcoded PYLIB=$SRC/ukmon_pylib @@ -34,6 +38,15 @@ WMPL_ENV=wmpl APIKEY=$(cat ~/.ssh/gmapsapikey) KMLTEMPLATE=*70km.kml +# variables that can be used in scripts and python files to ensure test data doesnt overwrite live data +if [ "$RUNTIME_ENV" == "DEV" ] ; then + TESTMODE="true" + TESTSUFF="/test" +else + TESTMODE="false" + TESTSUFF="" +fi + # create the config file now=$(date +%Y-%m-%d-%H:%M:%S) CFGFILE=~/${envname}/config.ini @@ -50,6 +63,7 @@ echo "TEMPLATES=${TEMPLATES}" >> ${CFGFILE} echo "RCODEDIR=${RCODEDIR}" >> ${CFGFILE} echo "DATADIR=${DATADIR}" >> ${CFGFILE} echo "SERVERINSTANCEID=${SERVERINSTANCEID}" >> ${CFGFILE} +echo "SERVERUSERID=${SERVERUSERID}" >> ${CFGFILE} echo "BKPINSTANCEID=${BKPINSTANCEID}" >> ${CFGFILE} echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> ${CFGFILE} echo "RMS_ENV=${RMS_ENV}" >> ${CFGFILE} @@ -62,6 +76,8 @@ echo "SERVERSSHKEY=${SERVERSSHKEY}" >> ${CFGFILE} echo "APIKEY=${APIKEY}" >> ${CFGFILE} echo "KMLTEMPLATE=${KMLTEMPLATE}" >> ${CFGFILE} echo "NJLOGGRP=${NJLOGGRP}" >> ${CFGFILE} +echo "TESTMODE=${TESTMODE}" >> ${CFGFILE} +echo "TESTSUFF=${TESTSUFF}" >> ${CFGFILE} echo "" >> ${CFGFILE} echo "export RUNTIME_ENV SRC SITEURL" >> ${CFGFILE} echo "export WEBSITEBUCKET UKMONSHAREDBUCKET" >> ${CFGFILE} @@ -69,8 +85,10 @@ echo "export PYLIB TEMPLATES RCODEDIR DATADIR BKPINSTANCEID AWS_DEFAULT_REGION" echo "export RMS_ENV RMS_LOC WMPL_ENV WMPL_LOC" >> ${CFGFILE} echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFILE} echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} -echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID NJLOGGRP" >> ${CFGFILE} +echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID SERVERUSERID NJLOGGRP" >> ${CFGFILE} echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib" >> ${CFGFILE} +echo "export TESTMODE TESTSUFF" >> ${CFGFILE} + # enable miniconda echo "source ~/.condaon" >> ${CFGFILE} # function to log to cloudwatch diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh index 3235281a2..e2334bfee 100644 --- a/archive/utils/statsToMqtt.sh +++ b/archive/utils/statsToMqtt.sh @@ -5,7 +5,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then source /home/bitnami/venvs/openhabstuff/bin/activate python $here/statsToMqtt.py diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') - if [ $diskpct -gt 9 ] ; then + if [ $diskpct -gt 90 ] ; then /home/bitnami/src/maintenance/mysql_checks.sh msg="$(hostname) diskspace was ${diskpct}%" subj="Diskspace alert for $(hostname)" @@ -13,7 +13,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi elif [ "$(hostname)" == "ukmcalcserver" ] ; then - source /home/ec2-user/venvs/wmpl/bin/activate + source $HOME/venvs/wmpl/bin/activate python $here/statsToMqtt.py diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') if [ $diskpct -gt 80 ] ; then diff --git a/archive/utils/updateDb.sh b/archive/utils/updateDb.sh index 2b8df8c27..501497f0b 100644 --- a/archive/utils/updateDb.sh +++ b/archive/utils/updateDb.sh @@ -11,10 +11,11 @@ else rundt=$1 fi +cd ~/prod/data/brightness mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; -load data local infile '/home/ec2-user/prod/data/brightness/CaptureNight_${rundt}.csv' +load data local infile './CaptureNight_${rundt}.csv' into table brightness fields terminated by ','; select count(*) from ukmon.brightness; EOD diff --git a/archive/website/createFireballPage.sh b/archive/website/createFireballPage.sh index 7c63ba4bb..396aeb65d 100644 --- a/archive/website/createFireballPage.sh +++ b/archive/website/createFireballPage.sh @@ -88,7 +88,7 @@ echo "\$(document).ready(function() {" >> reportindex.js # need single quotes here to allow the hash to be printed echo '$("#fbtableid").DataTable({' >> reportindex.js echo "columnDefs : [" >> reportindex.js -echo "{ Type : \"numeric\", targets : [2]}" >> reportindex.js +echo "{ Type : \"numeric\", targets : [1]}" >> reportindex.js echo " ]," >> reportindex.js echo "order : [[ 1, \"asc\"],[2,\"desc\"]]," >> reportindex.js echo "paging: false" >> reportindex.js diff --git a/archive/website/createOrbitIndex.sh b/archive/website/createOrbitIndex.sh index 2d92cec21..3b05e1e8f 100644 --- a/archive/website/createOrbitIndex.sh +++ b/archive/website/createOrbitIndex.sh @@ -90,9 +90,9 @@ else j=0 echo "" >> $idxfile fi - if [ $domth -eq 1 ] ; then - echo "$i" >> $DATADIR/orbits/$yr/$ym.txt - fi + #if [ $domth -eq 1 ] ; then + # echo "$i" >> $DATADIR/orbits/$yr/$ym.txt + #fi done fi echo "
DateTimeSourceShowerMagCameraLink
" >> $idxfile diff --git a/archive/website/templates/searchdialog.js b/archive/website/templates/searchdialog.js deleted file mode 100644 index e85d171cf..000000000 --- a/archive/website/templates/searchdialog.js +++ /dev/null @@ -1,122 +0,0 @@ -// initialize input widgets first -// Copyright (C) 2018-2023 Mark McIntyre - -$('#selectInterval .time').timepicker({ - 'showDuration': true, - 'timeFormat': 'g:ia' -}); - -$('#selectInterval .date').datepicker({ - 'format': 'd/m/yyyy', - 'autoclose': true, - 'endDate': '0' - -}); - -// initialize datepair -var res = document.getElementById("selectInterval"); -var dateSelect = new Datepair(res, { - 'defaultDateDelta': 0, // days - 'defaultTimeDelta': 900000 // milliseconds = 15 minutes -}); - -$('#selectInterval').on('rangeSelected', function(){ - var startdate = $('#selectInterval .date:first').datepicker('getDate'); - var starttime = $('#selectInterval .time:first').timepicker('getTime', startdate); - if(starttime != null){ - var dateval = starttime.getTime(); - var timediff = dateSelect.getTimeDiff(); - var endval = dateval + timediff; - endtime = new Date(); - endtime.setTime(endval); - $('#datestart').text(starttime.toISOString()); - $('#dateend').text(endtime.toISOString()); - if(timediff > 0){ - $('#statusfield').text('Valid range selected'); - }else{ - $('#statusfield').text('Date range must be > 0'); - } - } -}); - -var apiurl = 'https://api.ukmeteors.co.uk/detections'; -var form = document.querySelector("form"); -form.addEventListener("submit", function (event) { - //console.log("Saving value", form.elements.value.value); - var d1 = document.getElementById("datestart").innerHTML; - var d2 = document.getElementById("dateend").innerHTML; - var matchOnly = document.getElementById("matchesOnly").checked; - console.log(matchOnly) - var op = ""; - if (matchOnly == true ) { - op = op + "t:_"; - } - var magSelect = document.getElementById("magselect").value; - if (magSelect != 1 ) { - if(magSelect == 2) {op = op + "m:0_"} - if(magSelect == 3) {op = op + "m:-4_"} - } - var shwrSelect = document.getElementById("shwrselect").value; - if (shwrSelect != 1 ) { - var e = document.getElementById("shwrselect"); - var strshwr = e.options[e.selectedIndex].text; - op = op + "s:" + strshwr + "_"; - } - var statSelect = document.getElementById("statselect").value; - if (statSelect != 1 ) { - var e = document.getElementById("statselect"); - var strstat = e.options[e.selectedIndex].text; - op = op + "l:" + strstat + "_"; - } - //jQuery.support.cors = true; - payload = {"d1": d1, "d2": d2, "opts": op }; - console.log(payload); - document.getElementById("searchresults").innerHTML = "Searching...."; - $.ajax({ - url: apiurl, - type: "GET", - data: payload, - dataType: 'jsonp', - error: function (xhr, status, ex ) { - if (status === 'error' ) { - alert("Too much data, try a narrower range"); - console.log(xhr.status); - } - }, - complete: function (xhr, status) { - } - }); - event.preventDefault();} - ); - -function myFunc(myObj) { - console.log(myObj); - var x, txt = ""; - txt = "" - txt += ""; - txt += ""; - for (x in myObj) { - console.log(myObj[x]); - var dtarr = myObj[x].split(','); - var cams = dtarr[4].replaceAll(";", "; "); - if (dtarr[0] > 0 ){ - txt +='"; - } - } - txt += "
DateTimeSourceShowerMagCameraLink
'; - dt = new Date(dtarr[0]*1000); - txt += dt.toISOString(); - txt += ""; - txt += dtarr[1]; - txt += ""; - txt += dtarr[2]; - txt += ""; - txt += dtarr[3]; - txt += ""; - txt += cams; - txt += ""; - errimg = "onerror=\"this.onerror=null;this.src='/img/missing.png';\""; - txt += "\"Image"; - txt += "
"; - document.getElementById("searchresults").innerHTML = txt; -} diff --git a/bumpver.toml b/bumpver.toml index 9cab7605b..f8ef5f0f4 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2024.04.2" +current_version = "2026.04.1" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true @@ -13,6 +13,6 @@ push = true "bumpver.toml" = [ 'current_version = "{version}"', ] -".github/workflows/build_usermgmt.yml" = [ - 'tag_name: {version}' +"archive/README.md" = [ + "version: {version}" ] diff --git a/install_or_update.sh b/install_or_update.sh new file mode 100755 index 000000000..8efa03f02 --- /dev/null +++ b/install_or_update.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre + +if [[ "$1" != "PROD" && "$1" != "DEV" ]] ; then + echo "must provide runtime env of PROD or DEV" + exit -1 +fi +RUNTIME_ENV=$1 +envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +mkdir -p ~/${envname} + +cd $here/archive + +for loc in analysis ukmon_pylib website cronjobs utils share static_content +do + rsync -avz $loc/ ~/${envname}/$loc + chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 +done +DATADIR=~/$envname/data +mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} +mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} +mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} +mkdir -p ~/$envname/logs + diff --git a/tests/test_apis.py b/tests/test_apis.py index 4394b5d16..3d98c763d 100644 --- a/tests/test_apis.py +++ b/tests/test_apis.py @@ -56,7 +56,7 @@ def test_getMatchPickle(): for chunk in res.iter_content(chunk_size=4096): data = data + chunk jsd = json.loads(data) - assert len(jsd) == 62 + assert len(jsd) == 64 assert len(jsd['observations']) == 5 else: assert False diff --git a/utils/reimageCalcServer.ps1 b/utils/reimageCalcServer.ps1 new file mode 100644 index 000000000..9f776af31 --- /dev/null +++ b/utils/reimageCalcServer.ps1 @@ -0,0 +1,9 @@ +# script to re-image the UKMON calc server +# copyright Mark McIntyre, 2026- + +$dtstr=(get-date -uformat "%Y%m%d") + +$instdetails=(aws ec2 describe-instances --filters Name="tag:Name",Values="calcengine_ub" --profile ukmonshared --region eu-west-2) +$instanceid= (Write-Output $instdetails| convertfrom-json)[0].reservations.instances.instanceid + +aws ec2 create-image --instance-id $instanceid --name "calcengine_${dtstr}" --description "Latest Calc Engine image" --profile ukmonshared --region eu-west-2 \ No newline at end of file