From a76261bb00e383fe14e1711bc6a8f1ed722bd66a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Oct 2025 21:07:57 +0100 Subject: [PATCH 01/20] remove old v1 match data api --- .../files/matchApiJson/ukmdaMatchApi.json | 149 ------------------ .../files/matchDataApi/matchDataApiHandler.py | 113 ------------- archive/terraform/ukmda/matchApiLambda.tf | 91 ----------- 3 files changed, 353 deletions(-) delete mode 100644 archive/terraform/ukmda/files/matchApiJson/ukmdaMatchApi.json delete mode 100644 archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py delete mode 100644 archive/terraform/ukmda/matchApiLambda.tf diff --git a/archive/terraform/ukmda/files/matchApiJson/ukmdaMatchApi.json b/archive/terraform/ukmda/files/matchApiJson/ukmdaMatchApi.json deleted file mode 100644 index 4c7473d64..000000000 --- a/archive/terraform/ukmda/files/matchApiJson/ukmdaMatchApi.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "openapi" : "3.0.1", - "info" : { - "title" : "ukmonMatchApi", - "description" : "return info about daily matches", - "version" : "2021-02-15T19:23:10Z" - }, - "servers" : [ { - "url" : "https://oaa3lqdkvf.execute-api.eu-west-1.amazonaws.com/{basePath}", - "variables" : { - "basePath" : { - "default" : "/prod" - } - } - } ], - "paths" : { - "/" : { - "get" : { - "parameters" : [ { - "name" : "reqval", - "in" : "query", - "required" : true, - "schema" : { - "type" : "string" - } - }, { - "name" : "reqtyp", - "in" : "query", - "required" : true, - "schema" : { - "type" : "string" - } - }, { - "name" : "points", - "in" : "query", - "required" : false, - "schema" : { - "type" : "string" - } - }, { - "name" : "statid", - "in" : "query", - "required" : false, - "schema" : { - "type" : "string" - } - } - ], - "responses" : { - "200" : { - "description" : "200 response", - "headers" : { - "Access-Control-Allow-Origin" : { - "schema" : { - "type" : "string" - } - } - }, - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/Empty" - } - } - } - } - }, - "x-amazon-apigateway-request-validator" : "Validate body, query string parameters, and headers", - "x-amazon-apigateway-integration" : { - "httpMethod" : "POST", - "uri" : "arn:aws:apigateway:eu-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:eu-west-2:${thisacct}:function:matchDataApiHandler/invocations", - "responses" : { - "default" : { - "statusCode" : "200", - "responseParameters" : { - "method.response.header.Access-Control-Allow-Origin" : "'*'" - } - } - }, - "passthroughBehavior" : "when_no_match", - "contentHandling" : "CONVERT_TO_TEXT", - "type" : "aws_proxy" - } - }, - "options" : { - "responses" : { - "200" : { - "description" : "200 response", - "headers" : { - "Access-Control-Allow-Origin" : { - "schema" : { - "type" : "string" - } - }, - "Access-Control-Allow-Methods" : { - "schema" : { - "type" : "string" - } - }, - "Access-Control-Allow-Headers" : { - "schema" : { - "type" : "string" - } - } - }, - "content" : { - "application/json" : { - "schema" : { - "$ref" : "#/components/schemas/Empty" - } - } - } - } - }, - "x-amazon-apigateway-integration" : { - "responses" : { - "default" : { - "statusCode" : "200", - "responseParameters" : { - "method.response.header.Access-Control-Allow-Methods" : "'GET,OPTIONS'", - "method.response.header.Access-Control-Allow-Headers" : "'Content-Type,X-Amz-Date,Authorization,X-Api-Key,X-Amz-Security-Token'", - "method.response.header.Access-Control-Allow-Origin" : "'*'" - } - } - }, - "requestTemplates" : { - "application/json" : "{\"statusCode\": 200}" - }, - "passthroughBehavior" : "when_no_match", - "type" : "mock" - } - } - } - }, - "components" : { - "schemas" : { - "Empty" : { - "title" : "Empty Schema", - "type" : "object" - } - } - }, - "x-amazon-apigateway-request-validators" : { - "Validate body, query string parameters, and headers" : { - "validateRequestParameters" : true, - "validateRequestBody" : true - } - } -} \ No newline at end of file diff --git a/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py b/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py deleted file mode 100644 index 57ffa44ce..000000000 --- a/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (C) 2018-2023 Mark McIntyre -import datetime -import os -import boto3 -import json - - -def lambda_handler(event, context): - target = os.getenv('SRCHBUCKET', default='ukmda-shared') - webbuck = os.getenv('WEBBUCKET', default='ukmda-website') - #print('received event', json.dumps(event)) - qs = event['queryStringParameters'] - if qs is None: - return { - 'statusCode': 200, - 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1]' - } - reqtyp = qs['reqtyp'] - reqval = qs['reqval'] - points = False - if 'points' in qs: - points = True - - if reqtyp not in ['matches','detail','station','summary']: - res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}' - else: - print(f'reqtype {reqtyp} reqval {reqval} points {points}') - if reqtyp == 'summary': - d1 = datetime.datetime.strptime(reqval, '%Y%m%d') - idxfile = 'matches/matched/matches-full-{:04d}.csv'.format(d1.year) - res = '{"no matches"}' - fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ - '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ - 'dtstamp,orbname,iau,name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' - expr = f"SELECT {fieldlist} from s3object s where s._localtime like '_{reqval}%'" - fhi = {"FileHeaderInfo": "Use"} - elif reqtyp == 'matches': - d1 = datetime.datetime.strptime(reqval, '%Y%m%d') - idxfile = 'matches/matched/matches-full-{:04d}.csv'.format(d1.year) - res = '{"no matches"}' - expr = "SELECT s.orbname from s3object s where s._localtime like '_{}%'".format(reqval) - fhi = {"FileHeaderInfo": "Use"} - elif reqtyp == 'detail': - idxfile = 'matches/matched/matches-full-{}.csv'.format(reqval[:4]) - res = '{"event not found"}' - expr = "SELECT * from s3object s where s.orbname='{}'".format(reqval) - fhi = {"FileHeaderInfo": "Use"} - elif reqtyp == 'station': - statid = qs['statid'] - print(f'statid {statid}') - d1 = datetime.datetime.strptime(reqval, '%Y%m%d') - idxfile = 'matches/matched/matches-full-{:04d}.csv'.format(d1.year) - res = '{"no matches"}' - expr = "SELECT s.orbname from s3object s where s._localtime like '_{}%' and s.stations like '%{}%'".format(reqval, statid) - fhi = {"FileHeaderInfo": "Use"} - - s3 = boto3.client('s3') - resp = s3.select_object_content(Bucket=target, Key=idxfile, ExpressionType='SQL', - Expression=expr, InputSerialization={'CSV': fhi, 'CompressionType': 'NONE'}, OutputSerialization={'JSON': {}}, ) - res='' - for event in resp['Payload']: - if 'Records' in event: - res = res + event['Records']['Payload'].decode('utf-8') - if reqtyp == 'summary': - res = '[' + res.replace('}\n{','},\n{') +']' - if points is True: - print('requested points') - url = json.loads(res)['img'] - url = url.replace('ground_track.png','report.txt') - reppth = url.split('//', 1)[1].split('/',1)[1] - _, repname = os.path.split(reppth) - try: - tmpfname = f'/tmp/{repname}' - s3.download_file(webbuck, reppth, tmpfname) - flis = open(tmpfname, 'r').readlines() - os.remove(tmpfname) - res = fileToJsonString(flis) - except Exception: - print('report file unavailable') - res = '{"points": "unavailable"}' - - print(res) - return { - 'statusCode': 200, - 'body': res - } - - -def fileToJsonString(flis): - hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', - 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', - 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] - ptsarray='[' - gotpts = False - for fli in flis: - if 'Points' in fli: - gotpts = True - continue - elif '------' in fli or ' No' in fli: - continue - elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): - gotpts=False - break - elif gotpts is True: - spls = fli.split(',') - thisrow = '{' - for h, s in zip(hdr, spls): - thisrow = thisrow + f'"{h}": "{s.strip()}",' - thisrow = thisrow[:-1] + '},' - ptsarray = ptsarray + thisrow - ptsarray = ptsarray[:-1] + ']' - ptsarray = '{' + f'"points": {ptsarray}' + '}' - return ptsarray diff --git a/archive/terraform/ukmda/matchApiLambda.tf b/archive/terraform/ukmda/matchApiLambda.tf deleted file mode 100644 index 68be39136..000000000 --- a/archive/terraform/ukmda/matchApiLambda.tf +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2018-2023 Mark McIntyre -# Lambda for the Match data API. Need to rework this. -# -data "archive_file" "matchapizip" { - type = "zip" - source_dir = "${path.root}/files/matchDataApi/" - output_path = "${path.root}/files/matchDataApi.zip" -} - -resource "aws_lambda_function" "matchapilambda" { - function_name = "matchDataApiHandler" - description = "API to retrieve match event data" - filename = data.archive_file.matchapizip.output_path - source_code_hash = data.archive_file.matchapizip.output_base64sha256 - handler = "matchDataApiHandler.lambda_handler" - runtime = "python3.11" - memory_size = 128 - timeout = 30 - role = aws_iam_role.S3FullAccess.arn - publish = false - environment { - variables = { - "SRCHBUCKET" = "${var.sharedbucket}" - } - } - ephemeral_storage { - size = 512 - } - tags = { - Name = "matchDataApiHandler" - billingtag = "ukmda" - } -} - -data "template_file" "match_body_templ" { - template = file("files/matchApiJson/ukmdaMatchApi.json") - vars = { - thisacct = "${data.aws_caller_identity.current.account_id}" - } -} - - -resource "aws_api_gateway_rest_api" "matchapi_apigateway" { - body = data.template_file.match_body_templ.rendered # file("files/matchApiJson/ukmdaMatchApi.json") - name = "ukmdaMatchApi" - provider = aws.eu-west-1-prov - endpoint_configuration { - types = ["REGIONAL"] - } - tags = { - Name = "ukmdaMatchApi" - billingtag = "ukmda" - } -} - -resource "aws_api_gateway_base_path_mapping" "matchapi" { - api_id = aws_api_gateway_rest_api.matchapi_apigateway.id - stage_name = "prod" - domain_name = aws_api_gateway_domain_name.apigwdomain.domain_name - base_path = "matchesv1" - provider = aws.eu-west-1-prov - depends_on = [aws_api_gateway_stage.matchapistage] -} - -resource "aws_api_gateway_stage" "matchapistage" { - deployment_id = aws_api_gateway_deployment.matchapi_deployment.id - rest_api_id = aws_api_gateway_rest_api.matchapi_apigateway.id - stage_name = "prod" - provider = aws.eu-west-1-prov -} - -resource "aws_api_gateway_deployment" "matchapi_deployment" { - rest_api_id = aws_api_gateway_rest_api.matchapi_apigateway.id - provider = aws.eu-west-1-prov - triggers = { - redeployment = sha1(jsonencode(aws_api_gateway_rest_api.matchapi_apigateway.body)) - } - - lifecycle { - create_before_destroy = true - } -} - -resource "aws_lambda_permission" "perm_apigw_match" { - statement_id = "AllowExecutionFromAPIGW" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.matchapilambda.arn - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:eu-west-2:183798037734:975kbpqxg6/*/GET/" - #source_arn = "${aws_api_gateway_rest_api.matchapi_apigateway.arn}/*/*" -} From 0ec910b903ed128795e1e11bb442c9ce0f4c9baa Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Oct 2025 02:17:15 +0100 Subject: [PATCH 02/20] remove aws profile setting --- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 2705bdff1..97555ca96 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -135,7 +135,7 @@ def createExecConsolSh(matchstart, matchend, execconsolsh): outf.write('#!/bin/bash\n') outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - outf.write('export AWS_PROFILE=ukmonshared\n') + # outf.write('export AWS_PROFILE=ukmonshared\n') outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execConsol start\n') @@ -150,7 +150,7 @@ def createExecConsolSh(matchstart, matchend, execconsolsh): pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) outf.write('logger -s -t execConsol getting the image list\n') gatherUsedImageList(outf, matchstart, matchend, shbucket) - outf.write('unset AWS_PROFILE\n') + #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execConsol done\n') return @@ -164,13 +164,13 @@ def createExecReplotSh(matchstart, matchend, execconsolsh): outf.write('#!/bin/bash\n') outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - outf.write('export AWS_PROFILE=ukmonshared\n') + #outf.write('export AWS_PROFILE=ukmonshared\n') outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execReplot start\n') refreshTrajectories(outf, matchstart, matchend, outpath) createDensityPlots(outf, calcdir, enddt, False) gatherUsedImageList(outf, matchstart, matchend, shbucket) - outf.write('unset AWS_PROFILE\n') + #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execReplot done\n') return @@ -194,7 +194,7 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh): outf.write('#!/bin/bash\n') outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - outf.write('export AWS_PROFILE=ukmonshared\n') + #outf.write('export AWS_PROFILE=ukmonshared\n') outf.write(f'cd {calcdir}\n') outf.write('df -h . \n') @@ -236,7 +236,7 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh): pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket) pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) - outf.write('unset AWS_PROFILE\n') + #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execdistrib done\n') From 377f01d383e634459a5791e37ad251c73327b552 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Oct 2025 02:17:29 +0100 Subject: [PATCH 03/20] add sts permissions --- .../terraform/ukmda/files/policies/ukmda-calcserverpolicy.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/terraform/ukmda/files/policies/ukmda-calcserverpolicy.json b/archive/terraform/ukmda/files/policies/ukmda-calcserverpolicy.json index 2dd3c0d01..8b772ae2b 100644 --- a/archive/terraform/ukmda/files/policies/ukmda-calcserverpolicy.json +++ b/archive/terraform/ukmda/files/policies/ukmda-calcserverpolicy.json @@ -36,7 +36,8 @@ "ec2messages:GetEndpoint", "ec2messages:GetMessages", "ec2messages:SendReply", - "ses:SendEmail" + "ses:SendEmail", + "sts:GetCallerIdentity" ], "Effect": "Allow", "Resource": [ From a46ede259263475d06a22e734e7f019368ed0114 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Oct 2025 21:15:15 +0100 Subject: [PATCH 04/20] bugfix in fireballCollector --- fbCollector/download_events.sh | 13 ++++++------- fbCollector/fireballCollector.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/fbCollector/download_events.sh b/fbCollector/download_events.sh index d75b9a430..c491c37b4 100644 --- a/fbCollector/download_events.sh +++ b/fbCollector/download_events.sh @@ -3,18 +3,17 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" dt=$1 -basedir=$(grep basedir $here/config.ini | awk -F= '{print $2}'|tr -d '\r' | sed 's|f:|/mnt/f|g' | sed 's|c:|/mnt/c|g') -echo $basedir $dt -sleep 30 -dtns=$dt -mkdir -p $basedir/$dtns -pushd $basedir/$dtns +basedir=$2 +getall=$3 + +mkdir -p $basedir/$dt +pushd $basedir/$dt rsync -avz gmn.uwo.ca:/home/uk*/files/event_monitor/*${dt}*.bz2 . rsync -avz gmn.uwo.ca:/home/be*/files/event_monitor/*${dt}*.bz2 . rsync -avz gmn.uwo.ca:/home/ie*/files/event_monitor/*${dt}*.bz2 . rsync -avz gmn.uwo.ca:/home/nl*/files/event_monitor/*${dt}*.bz2 . -if [ "$2" == "all" ] ; then +if [ "$getall" == "all" ] ; then rsync -avz gmn.uwo.ca:/home/fr*/files/event_monitor/*${dt}*.bz2 . rsync -avz gmn.uwo.ca:/home/de*/files/event_monitor/*${dt}*.bz2 . rsync -avz gmn.uwo.ca:/home/es*/files/event_monitor/*${dt}*.bz2 . diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 078f07ad5..7b6aec9e3 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -24,7 +24,7 @@ try: from wmpl.Formats.ECSV import loadECSVs from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric - solveravailable = 'enabled' + solveravailable = 'active' except Exception: solveravailable = 'disabled' @@ -1014,7 +1014,13 @@ def getEventData(self): if len(evtdate) < 15: tkMessageBox.showinfo("Warning", f'Need seconds in the event date field {evtdate}') return - cmd = os.path.join(self.script_loc, 'download_events.sh') + f' {evtdate} 1' + fbdir = self.fb_dir + if ':' in fbdir: + drv = fbdir[0].lower() + fbdir = '/mnt/' + drv + fbdir[2:] + fbdir = fbdir.replace('\\','/') + + cmd = os.path.join(self.script_loc, 'download_events.sh') + f' {evtdate} {fbdir} 1' if ':' in cmd: drv = cmd[0].lower() cmd = '/mnt/' + drv + cmd[2:] From 3df4a454544f0e6aa5b073c10c48218f58db1e84 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Oct 2025 23:40:37 +0000 Subject: [PATCH 05/20] Cron job to remove duplicate orbits on demand --- archive/cronjobs/deleteDuplicateOrbit.sh | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 archive/cronjobs/deleteDuplicateOrbit.sh diff --git a/archive/cronjobs/deleteDuplicateOrbit.sh b/archive/cronjobs/deleteDuplicateOrbit.sh new file mode 100644 index 000000000..03d129c4d --- /dev/null +++ b/archive/cronjobs/deleteDuplicateOrbit.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $here/../config.ini >/dev/null 2>&1 +conda activate $HOME/miniconda3/envs/${WMPL_ENV} + +chkrunning=.delrunning +mkdir -p $DATADIR/manualuploads +cd $DATADIR/manualuploads +if [[ -f ./$chkrunning || -f ./running ]] ; then + echo "update already running" + exit +fi +touch ./$chkrunning +aws s3 sync $UKMONSHAREDBUCKET/fireballs/uploads . --exclude "*" --include "*.delete" --exclude "*.done" + +newfiles=$(ls -1 *.delete 2> /dev/null) +if [ "$newfiles" != "" ] ; then + for fil in $newfiles ; do + orbname=$(cat $fil) + echo $(date +%Y-%m-%dT%H:%M:%SZ) removing $orbname + $SRC/utils/deleteOrbit.sh $orbname + aws s3 mv $UKMONSHAREDBUCKET/fireballs/uploads/$fil $UKMONSHAREDBUCKET/fireballs/uploads/processed/$fil.done + rm $fil + done +fi +rm $DATADIR/manualuploads/$chkrunning +find $SRC/logs -name "deleteDuplicateOrbit*" -mtime +10 -exec rm -f {} \; From f1b24107fde42e10080447029b7f005e3932a01c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Oct 2025 23:40:51 +0000 Subject: [PATCH 06/20] typo in mergeNewOrbit --- archive/cronjobs/mergeNewOrbit.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/cronjobs/mergeNewOrbit.sh b/archive/cronjobs/mergeNewOrbit.sh index 09335d201..0738450f0 100644 --- a/archive/cronjobs/mergeNewOrbit.sh +++ b/archive/cronjobs/mergeNewOrbit.sh @@ -5,7 +5,8 @@ source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} mkdir -p $DATADIR/manualuploads -cd $DATADIR/manualuploadsif [ -f ./running ] ; then +cd $DATADIR/manualuploads +if [ -f ./running ] ; then echo "already running" exit fi @@ -62,4 +63,4 @@ EOD # echo nothing to process fi rm $DATADIR/manualuploads/.running -find $SRC/logs -name "mergeNewOrbit*" -mtime +10 -exec rm -f {} \; \ No newline at end of file +find $SRC/logs -name "mergeNewOrbit*" -mtime +10 -exec rm -f {} \; From bf03b3983e9c6fd3690db54b281e89564c2d5b61 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Oct 2025 23:41:03 +0000 Subject: [PATCH 07/20] update list of files --- archive/deployment/cronjobs.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/archive/deployment/cronjobs.yml b/archive/deployment/cronjobs.yml index 95ee560d0..6a80c1b5e 100644 --- a/archive/deployment/cronjobs.yml +++ b/archive/deployment/cronjobs.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/Dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -21,6 +21,7 @@ - {src: '{{srcdir}}/cronjobs/nightlyJob.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - {src: '{{srcdir}}/cronjobs/captureBright.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - {src: '{{srcdir}}/cronjobs/getImoWSfile.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}//cronjobs/mergeNewOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}//cronjobs/gatherMonthlyVideos.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } + - {src: '{{srcdir}}/cronjobs/mergeNewOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } + - {src: '{{srcdir}}/cronjobs/deleteDuplicateOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } + - {src: '{{srcdir}}/cronjobs/gatherMonthlyVideos.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - {src: '{{srcdir}}/cronjobs/README.md', dest: '{{destdir}}/cronjobs', mode: '644', backup: no } From 7fe3bebb90bf41f5b01ffe4219136eda3bc93e74 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Oct 2025 23:41:48 +0000 Subject: [PATCH 08/20] fbcollector - add functionality to delete duplicate or incorrect solutions and to add videos or images to an existing solution --- fbCollector/fireballCollector.py | 187 +++++++++++++++++++++---------- 1 file changed, 125 insertions(+), 62 deletions(-) diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 7b6aec9e3..4cd2383d5 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -379,8 +379,8 @@ def initUI(self): gmnavailable ='disabled' rawMenu = Menu(self.menuBar, tearoff=0) - rawMenu.add_command(label="Get Live Images", command=self.get_data) - rawMenu.add_command(label="Get Videos", command=self.get_vids) + rawMenu.add_command(label="Get Live Images", command=self.getData) + rawMenu.add_command(label="Get Videos", command=self.getVids) rawMenu.add_separator() rawMenu.add_command(label="Get ECSVs", command=self.getRequestedECSVs) rawMenu.add_command(label="Excl/Incl ECSV", command=self.ignoreCamera) @@ -416,7 +416,10 @@ def initUI(self): self.menuBar.add_cascade(label="Solve", underline=0, menu=solveMenu) otherMenu = Menu(self.menuBar, tearoff=0) - otherMenu.add_command(label="Get Traj Pickle", command=self.get_trajpickle) + otherMenu.add_command(label="Get Traj Pickle", command=self.getTrajpickle) + otherMenu.add_command(label="Add Image/Vid", command=self.addImageVideo) + solveMenu.add_separator() + otherMenu.add_command(label="Delete Orbit", command=self.delOrbit) self.menuBar.add_cascade(label="Other", underline=0, menu=otherMenu) # buttons @@ -428,7 +431,7 @@ def initUI(self): self.patt_entry = StyledEntry(self.save_panel, textvariable = self.newpatt, width = 20) self.patt_entry.grid(row = 1, column = 1, columnspan = 2, sticky = "W") - save_bmp = StyledButton(self.save_panel, text="Get Images", width = 8, command = lambda: self.get_data()) + save_bmp = StyledButton(self.save_panel, text="Get Images", width = 8, command = lambda: self.getData()) save_bmp.grid(row = 1, column = 3) save_bmp = StyledButton(self.save_panel, text="Remove", width = 8, command = lambda: self.remove_image()) @@ -600,62 +603,8 @@ def removeSolution(self): return def uploadOrbit(self): - pickles=[] - for path, _, files in os.walk(self.dir_path): - for name in files: - if '.pickle' in name and '_mc_' not in name and 'tmpzip' not in path: - log.info(f'adding pickle {name}') - pickles.append(os.path.join(path, name)) - if name.lower().endswith(".ecsv") and 'ecsvs' not in path: - log.info(f'copying {name}') - shutil.copyfile(os.path.join(path, name), os.path.join(self.dir_path, 'ecsvs', name)) - - if len(pickles) == 0: - return - pickles = list(set(pickles)) - if len(pickles) == 1: - pickfile = pickles[0] - else: - pickfile = tkFileDialog.askopenfilename(title='Select Orbit Pickle', defaultextension='*.pickle', - initialdir=self.dir_path, initialfile='*.pickle', - filetypes=[('pickles','*.pickle')]) - if not pickfile: - return - orbname = os.path.split(pickfile)[1] - tmpdir = os.path.join(self.dir_path, 'tmpzip') - os.makedirs(tmpdir, exist_ok=True) - shutil.copyfile(pickfile, os.path.join(tmpdir, orbname)) - if os.path.isdir(os.path.join(self.dir_path, 'jpgs')): - shutil.copytree(os.path.join(self.dir_path, 'jpgs'), os.path.join(tmpdir, 'jpgs'), dirs_exist_ok=True) - if os.path.isdir(os.path.join(self.dir_path, 'mp4s')): - shutil.copytree(os.path.join(self.dir_path, 'mp4s'), os.path.join(tmpdir, 'mp4s'), dirs_exist_ok=True) - if os.path.isdir(os.path.join(self.dir_path, 'ecsvs')): - shutil.copytree(os.path.join(self.dir_path, 'ecsvs'), os.path.join(tmpdir, 'ecsvs'), dirs_exist_ok=True) - for path, _, files in os.walk(self.dir_path): - for name in files: - if '_dyn_mass_fit' in name: - im = Image.open(os.path.join(path, name)).convert("RGB") - im.save(os.path.join(tmpdir,'jpgs', name[:-4] + '.jpg')) - zfname = os.path.join(self.dir_path, orbname[:15]) - shutil.make_archive(zfname,'zip',tmpdir) - try: - shutil.rmtree(tmpdir) - except Exception: - pass + return uploadOrbitGeneric(self.dir_path, self.api_key) - if self.api_key: - headers = {'Content-type': 'application/zip', 'Slug': orbname[:15], 'apikey': self.api_key} - url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbname[:15]}.zip' - r = requests.put(url, data=open(zfname+'.zip', 'rb'), headers=headers) #, auth=('username', 'pass')) - #print(r.text) - if r.status_code != 200: - tkMessageBox.showinfo('Warning', f'Problem with upload, {r.status_code}') - else: - tkMessageBox.showinfo('Info', 'Orbit Uploaded') - return - else: - tkMessageBox.showinfo('Info', 'Zip File created') - def uploadRaw(self): zfname = os.path.join(os.getenv('TMP'), os.path.basename(self.dir_path)) log.info(f'zfname is {zfname}') @@ -723,6 +672,8 @@ def get_bin_list(self): if self.dir_path is None: dirname = tkFileDialog.askdirectory(parent=root,initialdir=self.fb_dir, title='Please select a directory') + if not dirname: + return _, thispatt = os.path.split(dirname) self.dir_path = dirname self.patt = thispatt @@ -903,7 +854,7 @@ def clean_folder(self): pass return - def get_data(self): + def getData(self): thispatt = self.newpatt.get().strip() self.patt = thispatt.ljust(15,'0') self.dir_path = os.path.join(self.fb_dir, self.newpatt.get().strip()) @@ -915,10 +866,12 @@ def get_data(self): self.renameImages(self.dir_path) self.update_listbox(self.get_bin_list()) - def get_trajpickle(self): + def getTrajpickle(self): basepatt = os.path.split(self.dir_path)[1] ymd = basepatt[:8] fullpatt = askstring('Trajectory Name', 'eg 20240101_010203.345_UK', initialvalue=basepatt) + if not fullpatt: + return trajpick = f'{basepatt}_trajectory.pickle' url = f'https://archive.ukmeteors.co.uk/reports/{ymd[:4]}/orbits/{ymd[:6]}/{ymd}/{fullpatt}/{trajpick}' log.info(f'{url}') @@ -932,8 +885,60 @@ def get_trajpickle(self): else: log.info(f'unable to retrieve {trajpick}, {get_response.status_code}') return + + def delOrbit(self): + orbit = None + orbit = askstring('Trajectory Name', 'eg 20240101_010203.345_UK', initialvalue='') + if not orbit: + return + tmpfname = os.path.join(self.fb_dir, f'{orbit[:15]}.delete') + open(tmpfname, 'w').write(orbit) + if self.api_key: + headers = {'Content-type': 'text/plain', 'Slug': orbit[:15], 'apikey': self.api_key} + url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbit[:15]}.delete' + r = requests.put(url, data=open(tmpfname, 'r'), headers=headers) + if r.status_code != 200: + tkMessageBox.showinfo('Warning', f'Problem with request, {r.status_code}') + else: + tkMessageBox.showinfo("Info", f'deleted {orbit}') + return + else: + tkMessageBox.showinfo('Info', "Can't delete without api key") + os.remove(tmpfname) + return + + def addImageVideo(self): + orbit = None + dirname = tkFileDialog.askdirectory(parent=root,initialdir=self.fb_dir, + title='Please select a directory') + if not dirname: + return + _, orbit = os.path.split(dirname) + ymd = orbit[:8] + pickname = f'{orbit[:15]}_trajectory.pickle' + os.makedirs(os.path.join(dirname, 'jpgs'), exist_ok=True) + os.makedirs(os.path.join(dirname, 'mp4s'), exist_ok=True) + for ext in ['jpg','mp4']: + files = glob.glob(f'{dirname}/*.{ext}') + for fil in files: + _, barename = os.path.split(fil) + shutil.move(fil, os.path.join(dirname,f'{ext}s',barename)) + url = f'https://archive.ukmeteors.co.uk/reports/{ymd[:4]}/orbits/{ymd[:6]}/{ymd[:8]}/{orbit}/{pickname}' + count = 0 + log.info(url) + get_response = requests.get(url, stream=True) + if get_response.status_code == 200: + log.info(f'retrieved {pickname}') + count += 1 + with open(os.path.join(dirname, pickname), 'wb') as f: + for chunk in get_response.iter_content(chunk_size=4096): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + uploadOrbitGeneric(dirname, self.api_key) + tkMessageBox.showinfo("Info", f'Added data to {orbit}') + return - def get_vids(self): + def getVids(self): jpglist = glob.glob1(os.path.join(self.dir_path,'jpgs'), 'FF*.jpg') os.makedirs(os.path.join(self.dir_path, 'mp4s'), exist_ok=True) count = 0 @@ -1101,6 +1106,64 @@ def getGMNData(self): return +def uploadOrbitGeneric(orbdir, api_key): + pickles=[] + for path, _, files in os.walk(orbdir): + for name in files: + if '.pickle' in name and '_mc_' not in name and 'tmpzip' not in path: + log.info(f'adding pickle {name}') + pickles.append(os.path.join(path, name)) + if name.lower().endswith(".ecsv") and 'ecsvs' not in path: + log.info(f'copying {name}') + shutil.copyfile(os.path.join(path, name), os.path.join(orbdir, 'ecsvs', name)) + + if len(pickles) == 0: + return + pickles = list(set(pickles)) + if len(pickles) == 1: + pickfile = pickles[0] + else: + pickfile = tkFileDialog.askopenfilename(title='Select Orbit Pickle', defaultextension='*.pickle', + initialdir=orbdir, initialfile='*.pickle', + filetypes=[('pickles','*.pickle')]) + if not pickfile: + return + orbname = os.path.split(pickfile)[1] + tmpdir = os.path.join(orbdir, 'tmpzip') + os.makedirs(tmpdir, exist_ok=True) + shutil.copyfile(pickfile, os.path.join(tmpdir, orbname)) + if os.path.isdir(os.path.join(orbdir, 'jpgs')): + shutil.copytree(os.path.join(orbdir, 'jpgs'), os.path.join(tmpdir, 'jpgs'), dirs_exist_ok=True) + if os.path.isdir(os.path.join(orbdir, 'mp4s')): + shutil.copytree(os.path.join(orbdir, 'mp4s'), os.path.join(tmpdir, 'mp4s'), dirs_exist_ok=True) + if os.path.isdir(os.path.join(orbdir, 'ecsvs')): + shutil.copytree(os.path.join(orbdir, 'ecsvs'), os.path.join(tmpdir, 'ecsvs'), dirs_exist_ok=True) + for path, _, files in os.walk(orbdir): + for name in files: + if '_dyn_mass_fit' in name: + im = Image.open(os.path.join(path, name)).convert("RGB") + im.save(os.path.join(tmpdir,'jpgs', name[:-4] + '.jpg')) + zfname = os.path.join(orbdir, orbname[:15]) + shutil.make_archive(zfname,'zip',tmpdir) + try: + shutil.rmtree(tmpdir) + except Exception: + pass + + if api_key: + headers = {'Content-type': 'application/zip', 'Slug': orbname[:15], 'apikey': api_key} + url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbname[:15]}.zip' + r = requests.put(url, data=open(zfname+'.zip', 'rb'), headers=headers) #, auth=('username', 'pass')) + #print(r.text) + if r.status_code != 200: + tkMessageBox.showinfo('Warning', f'Problem with upload, {r.status_code}') + else: + tkMessageBox.showinfo('Info', 'Orbit Uploaded') + return + else: + tkMessageBox.showinfo('Info', 'Zip File created') + + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("-d", "--datepatt", type=str, help="date pattern to retrieve") From 7fdfaf3501737ecdbce3344083a463a31c396a07 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Oct 2025 23:42:44 +0000 Subject: [PATCH 09/20] Skip empty CSV files when consolidating data. Saves space --- archive/analysis/consolidateOutput.sh | 36 ++++++++++++++++----------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/archive/analysis/consolidateOutput.sh b/archive/analysis/consolidateOutput.sh index 822038e57..34743434c 100644 --- a/archive/analysis/consolidateOutput.sh +++ b/archive/analysis/consolidateOutput.sh @@ -37,23 +37,29 @@ consdir=${DATADIR}/consolidated/temp mkdir -p ${DATADIR}/single/rawcsvs ls -1 $consdir/*.csv | while read csvf do - bn=$(basename $csvf) - typ=${bn:0:3} - if [ "$typ" != "M20" ] ; then - pref="P" - yr=${bn:7:4} + flen=$(wc -l $csvf | awk '{print $1}') + if [ $flen -lt 2 ] ; then + logger -s -t consolidateOutput "skipping empty file $csvf" + rm $csvf else - pref="M" - yr=${bn:1:4} - fi - mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv - if [ ! -f $mrgfile ] ; then - cat $csvf >> $mrgfile - else - #echo $bn $mrgfile - sed '1d' $csvf >> $mrgfile + bn=$(basename $csvf) + typ=${bn:0:3} + if [ "$typ" != "M20" ] ; then + pref="P" + yr=${bn:7:4} + else + pref="M" + yr=${bn:1:4} + fi + mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv + if [ ! -f $mrgfile ] ; then + cat $csvf >> $mrgfile + else + #echo $bn $mrgfile + sed '1d' $csvf >> $mrgfile + fi + mv $csvf ${DATADIR}/single/rawcsvs fi - mv $csvf ${DATADIR}/single/rawcsvs done logger -s -t consolidateOutput "pushing consolidated information back" From 2e50150b7a0e749a5bb7d3878dc7ef254e007030 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Oct 2025 13:32:57 +0000 Subject: [PATCH 10/20] add function to purge old CSV files from server to save space --- archive/ukmon_pylib/maintenance/purgeCsvs.py | 39 ++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 archive/ukmon_pylib/maintenance/purgeCsvs.py diff --git a/archive/ukmon_pylib/maintenance/purgeCsvs.py b/archive/ukmon_pylib/maintenance/purgeCsvs.py new file mode 100644 index 000000000..e4a89feec --- /dev/null +++ b/archive/ukmon_pylib/maintenance/purgeCsvs.py @@ -0,0 +1,39 @@ +# copyright 2023- Mark McIntyre +# all rights reserved + +import boto3 +import sys + + +def findCsvsToDelete(archbucket, pref): + s3 = boto3.resource('s3') + s3c = boto3.client('s3') + bucket = s3.Bucket(archbucket) + patt = f'matches/single/rawcsvs/{pref}' + print(f'clearing down empty csvs from {archbucket}/{patt}... ', end='') + files = [obj.key for obj in bucket.objects.filter(Prefix=patt)] + filestodel = [] + for fil in files: + response = s3c.head_object(Bucket=archbucket, Key=fil) + size = response['ContentLength'] + if size < 100: + filestodel.append(fil) + numimgs = len(filestodel) + deleteZeroSizeFiles(filestodel, archbucket) + print(f'deleted {numimgs} empty CSVs') + return numimgs + + +def deleteZeroSizeFiles(flist, archbucket): + s3 = boto3.client('s3') + chunk_size = 900 + chunked_list = [flist[i:i + chunk_size] for i in range(0, len(flist), chunk_size)] + for ch in chunked_list: + delete_keys = {'Objects': []} + delete_keys['Objects'] = [{'Key': k} for k in ch] + s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + return + + +if __name__ == '__main__': + findCsvsToDelete('ukmda-shared', sys.argv[1]) From d962fdd4635a6e341a3d15c714658382c1ccc77c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Oct 2025 13:43:46 +0000 Subject: [PATCH 11/20] update GH workflows --- .github/workflows/automated-testing.yml | 2 +- .github/workflows/build_tools.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/automated-testing.yml b/.github/workflows/automated-testing.yml index 5b3f625ad..6190f53ca 100644 --- a/.github/workflows/automated-testing.yml +++ b/.github/workflows/automated-testing.yml @@ -10,7 +10,7 @@ on: branches: [dev, markmac99] pull_request: # The branches below must be a subset of the branches above - branches: [master] + branches: [dev, markmac99] jobs: conttests: runs-on: thelinux diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml index 7b2865d36..bc39e5e68 100644 --- a/.github/workflows/build_tools.yml +++ b/.github/workflows/build_tools.yml @@ -7,7 +7,7 @@ permissions: on: push: - branches: [dev, markmac99, master] + branches: [master] pull_request: # The branches below must be a subset of the branches above branches: [master] From c6ac98faff97a58f0e958d7b233514274001ccca Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Oct 2025 13:52:47 +0000 Subject: [PATCH 12/20] testing conttests --- .github/workflows/automated-testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/automated-testing.yml b/.github/workflows/automated-testing.yml index 6190f53ca..ab43f5397 100644 --- a/.github/workflows/automated-testing.yml +++ b/.github/workflows/automated-testing.yml @@ -41,7 +41,8 @@ jobs: echo "test data available" fi mkdir -p ~/.aws - cp /data/profile/credentials ~/.aws/ + #cp /data/profile/credentials ~/.aws/ + aws sys get-caller-identity /tests.sh apitests: runs-on: ubuntu-latest From cfd47c09aff2797782668c48f06cd9bea8eb25a5 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Oct 2025 13:54:09 +0000 Subject: [PATCH 13/20] typo --- .github/workflows/automated-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-testing.yml b/.github/workflows/automated-testing.yml index ab43f5397..a479ec1fc 100644 --- a/.github/workflows/automated-testing.yml +++ b/.github/workflows/automated-testing.yml @@ -42,7 +42,7 @@ jobs: fi mkdir -p ~/.aws #cp /data/profile/credentials ~/.aws/ - aws sys get-caller-identity + aws sts get-caller-identity /tests.sh apitests: runs-on: ubuntu-latest From b6d95dea506eff18f78406fb59114b2e59b53a76 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Oct 2025 14:00:45 +0000 Subject: [PATCH 14/20] attempt to fix testing --- archive/ukmon_pylib/tests/test_reports_CameraDetails.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/tests/test_reports_CameraDetails.py b/archive/ukmon_pylib/tests/test_reports_CameraDetails.py index 607bd6cfd..2e57dbd19 100644 --- a/archive/ukmon_pylib/tests/test_reports_CameraDetails.py +++ b/archive/ukmon_pylib/tests/test_reports_CameraDetails.py @@ -3,6 +3,7 @@ import datetime import os import shutil +import boto3 from reports.CameraDetails import getCamLocDirFov, updateCamLocDirFovDB from reports.CameraDetails import loadLocationDetails, findEmail, findSite @@ -25,7 +26,11 @@ def test_updateCamLocDirFovDB(): def test_loadLocationDetails(): - caminfo = loadLocationDetails() + # for testing we should precreate the DDB connection using the default role + conn = boto3.Session() + ddb = conn.resource('dynamodb', region_name='eu-west-2') + + caminfo = loadLocationDetails(ddb=ddb) caminfo = caminfo[caminfo.stationid=='UK0006'] assert len(caminfo) == 1 From d755b953dd6fcfcc21fcc088e471d95274000923 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Oct 2025 14:09:25 +0000 Subject: [PATCH 15/20] more work on automated testing --- .github/workflows/automated-testing.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/automated-testing.yml b/.github/workflows/automated-testing.yml index a479ec1fc..2f2d82c42 100644 --- a/.github/workflows/automated-testing.yml +++ b/.github/workflows/automated-testing.yml @@ -40,10 +40,18 @@ jobs: else echo "test data available" fi - mkdir -p ~/.aws - #cp /data/profile/credentials ~/.aws/ aws sts get-caller-identity - /tests.sh + pip install --upgrade MeteorTools | grep -v 'already satisfied' + pip install pytest pytest-cov | grep -v 'already satisfied' + [ -d ./ukmda-dataprocessing] && rm -Rf ./ukmda-dataprocessing + git clone https://github.com/ukmda/ukmda-dataprocessing.git + cd ./ukmda-dataprocessing/ + git checkout $BRANCH + cd ./archive/ukmon_pylib/ + echo DATADIR is $DATADIR curdir is $curdir + export PYTHONPATH=/WesternMeteorPyLib:/RMS:${curdir} + pytest -v ./tests --cov=. --cov-report=term-missing + apitests: runs-on: ubuntu-latest steps: From 97b12fb109fbd9cd71603df63dcfb62fbcb43635 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Oct 2025 14:11:28 +0000 Subject: [PATCH 16/20] more work on testing --- .github/workflows/automated-testing.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/automated-testing.yml b/.github/workflows/automated-testing.yml index 2f2d82c42..205cacd46 100644 --- a/.github/workflows/automated-testing.yml +++ b/.github/workflows/automated-testing.yml @@ -32,7 +32,6 @@ jobs: aws-region: ${{ env.AWS_REGION }} - name: remote tests run: | - chmod +x /tests.sh if [ ! -f /data/admin/cameraLocs.json ] ; then echo "getting test data" curl https://archive.ukmeteors.co.uk/browse/testdata/testdata.tar.gz -o /data/testdata.tar.gz @@ -43,7 +42,7 @@ jobs: aws sts get-caller-identity pip install --upgrade MeteorTools | grep -v 'already satisfied' pip install pytest pytest-cov | grep -v 'already satisfied' - [ -d ./ukmda-dataprocessing] && rm -Rf ./ukmda-dataprocessing + [ -d ./ukmda-dataprocessing ] && rm -Rf ./ukmda-dataprocessing git clone https://github.com/ukmda/ukmda-dataprocessing.git cd ./ukmda-dataprocessing/ git checkout $BRANCH From e9cb3b2200c5f35b054e781eacefffdf2a3f9880 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Oct 2025 14:34:11 +0000 Subject: [PATCH 17/20] improvements to automated testing --- .github/workflows/automated-testing.yml | 2 ++ archive/ukmon_pylib/reports/cameraStatusReport.py | 4 ++-- .../ukmon_pylib/tests/test_reports_cameraStatusReport.py | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/automated-testing.yml b/.github/workflows/automated-testing.yml index 205cacd46..769740d4e 100644 --- a/.github/workflows/automated-testing.yml +++ b/.github/workflows/automated-testing.yml @@ -47,6 +47,8 @@ jobs: cd ./ukmda-dataprocessing/ git checkout $BRANCH cd ./archive/ukmon_pylib/ + curdir=$(pwd -P) + export DATADIR=/data echo DATADIR is $DATADIR curdir is $curdir export PYTHONPATH=/WesternMeteorPyLib:/RMS:${curdir} pytest -v ./tests --cov=. --cov-report=term-missing diff --git a/archive/ukmon_pylib/reports/cameraStatusReport.py b/archive/ukmon_pylib/reports/cameraStatusReport.py index 328a910c3..0872feff7 100644 --- a/archive/ukmon_pylib/reports/cameraStatusReport.py +++ b/archive/ukmon_pylib/reports/cameraStatusReport.py @@ -9,7 +9,7 @@ import pandas as pd -def getLastUpdateDate(datadir=None, camfname=None): +def getLastUpdateDate(datadir=None, camfname=None, ddb=None): """ Create a status report showing the last update date of each camera that is providing data @@ -19,7 +19,7 @@ def getLastUpdateDate(datadir=None, camfname=None): includenever (bool) default false, include cameras that have never uploaded """ - caminfo = loadLocationDetails() + caminfo = loadLocationDetails(ddb=ddb) caminfo = caminfo[caminfo.active==1] caminfo = caminfo.drop(columns=['direction','oldcode','active','camtype','eMail', 'humanName']) diff --git a/archive/ukmon_pylib/tests/test_reports_cameraStatusReport.py b/archive/ukmon_pylib/tests/test_reports_cameraStatusReport.py index c67ec74b9..668f37318 100644 --- a/archive/ukmon_pylib/tests/test_reports_cameraStatusReport.py +++ b/archive/ukmon_pylib/tests/test_reports_cameraStatusReport.py @@ -3,6 +3,7 @@ import datetime import os +import boto3 from reports.cameraStatusReport import getLastUpdateDate, createStatusReportJSfile @@ -13,7 +14,11 @@ def test_createStatusReportJSfile(): - stati = getLastUpdateDate(datadir) + # for testing we should precreate the DDB connection using the default role + conn = boto3.Session() + ddb = conn.resource('dynamodb', region_name='eu-west-2') + + stati = getLastUpdateDate(datadir=datadir, ddb=ddb) assert 'UK0006' in list(stati.stationid) createStatusReportJSfile(stati, datadir) csvf = os.path.join(datadir, 'reports', 'camrep.js') From 7e199c9a5659d1352b936f1ccb9a09bd2d58f260 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 9 Nov 2025 22:35:04 +0000 Subject: [PATCH 18/20] bugfix in fireball page creator --- archive/ukmon_pylib/reports/findFireballs.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/archive/ukmon_pylib/reports/findFireballs.py b/archive/ukmon_pylib/reports/findFireballs.py index 49575426b..60b7af0c0 100644 --- a/archive/ukmon_pylib/reports/findFireballs.py +++ b/archive/ukmon_pylib/reports/findFireballs.py @@ -67,8 +67,16 @@ def createMDFiles(fbs, outdir, orbdir): targfile = os.path.join(tmpdir, 'jpgs.lst') try: s3.download_file(srcbucket, fname, targfile) - bestimg = getBestView(pickfile) - #print(bestimg) + bestimg = getBestView(pickfile) + if bestimg[0] == '[': + bestimg = bestimg.replace('[','').replace(']','') + bestimg = bestimg.split(',') + bestimg = bestimg[0] + if bestimg[0] == "'": + bestimg = bestimg[1:-1] + if type(bestimg) is list: + bestimg = bestimg[0] + print(f' bestimg is "{bestimg}"') if bestimg[:3] == 'FF_': pth = fb.url[:fb.url.find('reports/')] bestimg = f'img/single/{yr}/{ym}/{bestimg}' @@ -83,8 +91,7 @@ def createMDFiles(fbs, outdir, orbdir): bestimgurl = '' except: print('unable to collect jpgs.lst') - - fname = loctime.strftime('%Y%m%d_%H%M%S') + '.md' + fname = f'{trajdir[:15]}.md' if os.path.isfile(os.path.join(outdir,fname)): print(f'{fname} exists, not replacing it') else: From efd2beb41d680007aa62164d4bde64e3e71aabd5 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 9 Nov 2025 22:35:23 +0000 Subject: [PATCH 19/20] bugfix and improvement deleteOrbit functionality --- archive/ukmon_pylib/maintenance/manageTraj.py | 20 +++++++++++++------ archive/utils/deleteOrbit.sh | 11 +++++++++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/archive/ukmon_pylib/maintenance/manageTraj.py b/archive/ukmon_pylib/maintenance/manageTraj.py index dcb515298..fae24d735 100644 --- a/archive/ukmon_pylib/maintenance/manageTraj.py +++ b/archive/ukmon_pylib/maintenance/manageTraj.py @@ -7,7 +7,7 @@ # delete an orbit from the database -def deleteDuplicate(trajname): +def deleteDuplicate(trajname, jd=None): datadir = os.getenv('DATADIR', default='/home/pi/prod/data') yr=trajname[:4] if int(yr) > 2021: @@ -21,18 +21,26 @@ def deleteDuplicate(trajname): else: print("can only be done for 2022 onwards") return 0 - idx = df[df.orbname==trajname].index - if len(idx) > 0: + seldf = df[df.orbname==trajname] + idx = seldf.index + if jd: + idx = seldf[seldf._mjd == jd].index + if len(idx) == 1: df = df.drop(index=idx) df.to_parquet(fname, compression='snappy') csvfname = os.path.join(datadir, 'matched','matches-full-{}.csv'.format(yr)) df.to_csv(csvfname, index=False) - - deleteWebPage(trajname) + if jd is None: + deleteWebPage(trajname) + return 0 + elif len(idx) > 1: + print(f'two or more matches to {trajname}, please select MJD') + for _,rw in seldf.iterrows(): + print(f'{rw._mjd}, {rw.orbname}, {rw._amag}, {rw._stream}') return 1 else: print(f'no match for {trajname}') - return 0 + return 1 def deleteWebPage(trajname): diff --git a/archive/utils/deleteOrbit.sh b/archive/utils/deleteOrbit.sh index 71dbdf60d..0338b1fff 100644 --- a/archive/utils/deleteOrbit.sh +++ b/archive/utils/deleteOrbit.sh @@ -5,10 +5,19 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} +if [ "$1" == "" ] ; then + echo "usage: deleteOrbit.sh fulltrajname {optional jd}" + exit 0 +fi + traj=$1 ymd=${traj:0:8} -python -c "from maintenance.manageTraj import deleteDuplicate as dd; dd('$traj'); " +if [ "$2" == "" ]; then + python -c "from maintenance.manageTraj import deleteDuplicate as dd; dd('$traj'); " +else + python -c "from maintenance.manageTraj import deleteDuplicate as dd; dd('$traj', $2); " +fi if [ $? == 0 ] ; then $SRC/website/createOrbitIndex.sh $ymd From 1b4cb027d347041961dabd05d5b673b6b058cb0f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 9 Nov 2025 22:35:41 +0000 Subject: [PATCH 20/20] bugfix to make fbCollector solve orbits properly --- fbCollector/fireballCollector.py | 56 +++++++++++++++----------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 4cd2383d5..3962b66c6 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -21,12 +21,6 @@ import paramiko from scp import SCPClient -try: - from wmpl.Formats.ECSV import loadECSVs - from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric - solveravailable = 'active' -except Exception: - solveravailable = 'disabled' import tkinter as tk import tkinter.filedialog as tkFileDialog @@ -280,9 +274,12 @@ def readConfig(self): try: self.wmpl_loc = os.path.expanduser(localcfg['solver']['wmpl_loc'].replace('$HOME','~')).replace('\\','/') self.wmpl_env= localcfg['solver']['wmpl_env'] + if os.path.isdir(os.path.expanduser(self.wmpl_loc)): + self.solveravailable = 'active' + else: + self.solveravailable = 'disabled' except: - log.error('WMPL location and environment name must be configured') - quitApp() + self.solveravailable = 'disabled' try: self.rms_loc = os.path.expanduser(localcfg['reduction']['rms_loc'].replace('$HOME','~')).replace('\\','/') self.rms_env = localcfg['reduction']['rms_env'] @@ -407,7 +404,7 @@ def initUI(self): self.menuBar.add_cascade(label="Review", underline=0, menu=revMenu) solveMenu = Menu(self.menuBar, tearoff=0) - solveMenu.add_command(label="Solve", command=self.solveOrbit, state=solveravailable) + solveMenu.add_command(label="Solve", command=self.solveOrbit, state=self.solveravailable) solveMenu.add_separator() solveMenu.add_command(label="View Solution", command=self.viewSolution) solveMenu.add_command(label="Delete Solution", command=self.removeSolution) @@ -538,46 +535,47 @@ def ignoreCamera(self): def solveOrbit(self): log.info('Using ECSV files:') ecsv_names = [] - ecsv_paths = [] - os.makedirs(os.path.join(self.dir_path,'ecsvs'), exist_ok=True) + ecsv_loc = os.path.join(self.dir_path,'ecsvs') + os.makedirs(ecsv_loc, exist_ok=True) for entry in sorted(os.walk(self.dir_path), key=lambda x: x[0]): dir_name, _, file_names = entry if 'ecsvs' in dir_name: continue for fn in file_names: if fn.lower().endswith(".ecsv"): - shutil.copyfile(os.path.join(dir_name, fn), os.path.join(self.dir_path, 'ecsvs', fn)) + shutil.copyfile(os.path.join(dir_name, fn), os.path.join(ecsv_loc, fn)) if fn.lower().endswith(".ecsv") and 'REJECT' not in dir_name.upper() and 'REJECT' not in fn.upper(): # Add ECSV file, but skip duplicates if fn not in ecsv_names: - ecsv_paths.append(os.path.join(dir_name, fn)) ecsv_names.append(fn) log.info(fn) - if len(ecsv_paths) < 2: + if len(ecsv_names) < 2: tkMessageBox.showinfo('Warning', 'Need at least two ECSV files') return - jdt_ref, meteor_list = loadECSVs(ecsv_paths) - mcruns = 20 - max_toffset = 15.0 - velpart = None - vinitht = None - plotallspatial = True - uncertgeom = False - jacchia = False - traj = solveTrajectoryGeneric(jdt_ref, meteor_list, self.dir_path, - max_toffset=max_toffset, monte_carlo=True, mc_runs=mcruns, - geometric_uncert=uncertgeom, plot_all_spatial_residuals=plotallspatial, - show_plots=False, v_init_part=velpart, v_init_ht=vinitht, - show_jacchia=jacchia, enable_OSM_plot=True, mc_cores=8) + + tmpscr = os.path.join(os.getenv('TEMP'), 'solve.ps1') + with open(tmpscr, 'w') as outf: + mcruns = 20 + outf.write(f'cd {self.wmpl_loc}\nconda activate {self.wmpl_env}\npython -m wmpl.Formats.ECSV {ecsv_loc} -l -x -r {mcruns} -w -t 15\n') + _ = subprocess.run(['powershell.exe', tmpscr]) + fldrs = os.listdir(ecsv_loc) + print(fldrs) + fldrs = [f for f in fldrs if os.path.isdir(os.path.join(ecsv_loc, f))] + if len(fldrs) > 0: + log.info(f'solved into {fldrs[0]}') + self.soln_outputdir = os.path.join(self.dir_path, fldrs[0]) + if os.path.isdir(self.soln_outputdir): + shutil.rmtree(self.soln_outputdir) + log.info(f'moving {os.path.join(ecsv_loc, fldrs[0])} to {self.dir_path}') + shutil.move(os.path.join(ecsv_loc, fldrs[0]), self.dir_path) tkMessageBox.showinfo('Info', 'Solver Finished') - if traj is not None: - self.soln_outputdir = traj.output_dir return def viewSolution(self): self.review_stack = False if not self.soln_outputdir: solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') + solndir = [f for f in solndir if os.path.isdir(os.path.join(self.dir_path, f))] if len(solndir) == 0: tkMessageBox.showinfo('Warning', 'No solution to review') return