From b304efbf9c0f1b3797ce7d5615d9672a7f2d5257 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 24 Feb 2026 18:01:32 +0000 Subject: [PATCH 001/287] remove disused code --- archive/terraform/ukmda/ukmdalive_bucket.tf | 52 --------------------- 1 file changed, 52 deletions(-) diff --git a/archive/terraform/ukmda/ukmdalive_bucket.tf b/archive/terraform/ukmda/ukmdalive_bucket.tf index 7e1c405b..45ea7d50 100644 --- a/archive/terraform/ukmda/ukmdalive_bucket.tf +++ b/archive/terraform/ukmda/ukmdalive_bucket.tf @@ -9,58 +9,6 @@ resource "aws_s3_bucket" "ukmdalive" { } provider = aws.eu-west-1-prov } -/* -resource "aws_s3_bucket_policy" "ukmdalivebp" { - bucket = aws_s3_bucket.ukmdalive.id - provider = aws.eu-west-1-prov - policy = jsonencode( - { - Id = "ukmda-live-bp" - Statement = [ - { - "Sid": "DataSyncCreateS3LocationAndTaskAccess", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::${var.eeaccountid}:role/DataSyncBetweenAccounts" - }, - "Action": [ - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:AbortMultipartUpload", - "s3:DeleteObject", - "s3:GetObject", - "s3:ListMultipartUploadParts", - "s3:PutObject", - "s3:GetObjectTagging", - "s3:PutObjectTagging" - ], - "Resource": [ - "${aws_s3_bucket.ukmdalive.arn}", - "${aws_s3_bucket.ukmdalive.arn}/*" - ] - }, - { - "Sid": "replicatelive", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::${var.eeaccountid}:role/service-role/replicatelive" - }, - "Action": [ - "s3:Put*", - "s3:Get*" - ], - "Resource": [ - "${aws_s3_bucket.ukmdalive.arn}", - "${aws_s3_bucket.ukmdalive.arn}/*" - ] - } - ] - Version = "2008-10-17" - } - ) -} -*/ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalivelcp" { bucket = aws_s3_bucket.ukmdalive.id From 480de91e3f7cbca921f95ba64c1ccec03c6a2946 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 24 Feb 2026 18:01:51 +0000 Subject: [PATCH 002/287] update livestream to accept normal RMS filenames --- .../files/monitorLiveFeed/monitorLiveFeed.py | 42 ++++++++----------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py b/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py index 654ad06a..40c3a421 100644 --- a/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py +++ b/archive/terraform/ukmda/files/monitorLiveFeed/monitorLiveFeed.py @@ -61,28 +61,19 @@ def storeInDDb(evtdets, camdets): return response['ResponseMetadata']['HTTPStatusCode'] -def updateLiveTable(event, dtval): - record = event['Records'][0] - fname = record['s3']['object']['key'] - _, barefname = os.path.split(fname) - if 'P.jpg' not in barefname: - print(f'{barefname} not a jpg') - return +def updateLiveTable(evtdets, camdets, fname): + statname = camdets['camid'] + dtval = evtdets['dtval'] expdate = int((dtval + datetime.timedelta(days=90)).timestamp()) tstamp = str(int(dtval.timestamp()*1000)) yr = dtval.strftime('%Y') mth = dtval.strftime('%m') ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table('live') - if barefname[0] == 'M': - statname = barefname[17:].replace('P.jpg','').replace('_',' ') - else: - statname = barefname[3:9] - - print(f'inserting {barefname} with timestamp {dtval}') + print(f'inserting {fname} with timestamp {dtval}') response = table.put_item( Item={ - 'image_name': barefname, + 'image_name': fname, 'timestamp': tstamp, 'image_timestamp': tstamp, 'station_name': statname, @@ -94,19 +85,15 @@ def updateLiveTable(event, dtval): return response['ResponseMetadata']['HTTPStatusCode'] -def processXml(event): - record = event['Records'][0] - fname = record['s3']['object']['key'] +def processXml(remote_xmlname): buck = 'ukmda-live' s3 = boto3.resource('s3') - if '.xml' not in fname: - fname = fname.replace('P.jpg','.xml') - _, barefname = os.path.split(fname) + _, barefname = os.path.split(remote_xmlname) tmpdir = mkdtemp() xmlname = os.path.join(tmpdir, barefname) try: - s3.meta.client.download_file(buck, fname, xmlname) + s3.meta.client.download_file(buck, remote_xmlname, xmlname) except Exception: print('xml file not available') return None, None @@ -157,7 +144,12 @@ def processXml(event): def lambda_handler(event, context): - evtdets, camdets = processXml(event) - if evtdets is not None: - storeInDDb(evtdets, camdets) - updateLiveTable(event, evtdets['dtval']) + record = event['Records'][0] + fname = record['s3']['object']['key'] + _, barefname = os.path.split(fname) + if ('M' in barefname and 'P.jpg' in barefname) or ('FF' in barefname and '.jpg' in barefname): + xmlname = fname.replace('P.jpg','.xml').replace('.jpg', '.xml') + evtdets, camdets = processXml(xmlname) + if evtdets is not None: + storeInDDb(evtdets, camdets) + updateLiveTable(evtdets, camdets, barefname) From 4226d9a382f3c6e85b20b80de4bc3990f1771457 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 26 Mar 2026 12:26:08 +0000 Subject: [PATCH 003/287] search dialog - fix bug and set default date range --- archive/static_content/js/searchdialog.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/archive/static_content/js/searchdialog.js b/archive/static_content/js/searchdialog.js index 5b9e0ea5..d5716b03 100644 --- a/archive/static_content/js/searchdialog.js +++ b/archive/static_content/js/searchdialog.js @@ -70,6 +70,17 @@ form.addEventListener("submit", function (event) { op = op + "l:" + strstat + "_"; } //jQuery.support.cors = true; + if (d1 == "" ){ + console.log("missing start date"); + const stdt = new Date(Date.now()); + stdt.setHours(stdt.getHours() - 12); + d1 = stdt.toISOString(); + } + if (d2 == "" ){ + console.log("missing end date"); + const enddt = new Date(Date.now()); + d2 = enddt.toISOString(); + } payload = {"d1": d1, "d2": d2, "opts": op }; console.log(payload); document.getElementById("searchresults").innerHTML = "Searching...."; @@ -96,6 +107,9 @@ function myFunc(myObj) { txt = "" txt += ""; txt += ""; + if (myObj.length == 0){ + alert("No data available, check dates"); + } for (x in myObj) { console.log(myObj[x]); var dtarr = myObj[x].split(','); From 1503fd61f0d3e867d0ca0d6dadaed8d5b1f77ea9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 26 Mar 2026 12:26:26 +0000 Subject: [PATCH 004/287] small change to explain default dates --- archive/static_content/search/index.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/static_content/search/index.html b/archive/static_content/search/index.html index 5fa067be..391ede95 100644 --- a/archive/static_content/search/index.html +++ b/archive/static_content/search/index.html @@ -78,8 +78,9 @@

Search the Archive.



This page allows you to search the archive for specific events in the matched and single-station detections.
- Time should be entered as local time, so in the UK summer you should enter the time in BST. + Time should be entered as local time, so in the UK summer you should enter the time in BST.

+

NOTE - default date range is the last 24 hours. This might be a HUGE amount of data during busy showers.

To search the livestream, go over to this page.

From 362cac09629fa3bdb0b9f118c595b3651f863b08 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 26 Mar 2026 12:31:34 +0000 Subject: [PATCH 005/287] improvement in live image search --- archive/lambdas/getLiveImages/getLiveImages.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/archive/lambdas/getLiveImages/getLiveImages.py b/archive/lambdas/getLiveImages/getLiveImages.py index 164ecc60..5d589131 100644 --- a/archive/lambdas/getLiveImages/getLiveImages.py +++ b/archive/lambdas/getLiveImages/getLiveImages.py @@ -16,7 +16,10 @@ def getLiveImages(dtstr, ddb=None): resp = table.query(IndexName='month-image_name-index', KeyConditionExpression=Key('month').eq(dtstr[4:6]) & Key('image_name').begins_with(f'M{dtstr}'), ProjectionExpression='image_name') - return resp + resp2 = table.query(IndexName='month-image_name-index', + KeyConditionExpression=Key('month').eq(dtstr[4:6]) & Key('image_name').begins_with(f'FF_{dtstr}'), + ProjectionExpression='image_name') + return resp['Items'] + resp2['Items'] def filterImages(d1, d2, statid=None, maxitems=-1, ddb=None): @@ -103,10 +106,10 @@ def lambda_handler(event, context): patt = qs['pattern'] print(f'searching for {patt}') ecsvstr = getLiveImages(patt, ddb=ddb) - print(f"found {ecsvstr['Items']}") + print(f"found {len(ecsvstr)} items") return { 'statusCode': 200, - 'body': json.dumps(ecsvstr['Items']) + 'body': json.dumps(ecsvstr) } else: if 'dtstr' in qs: @@ -119,7 +122,7 @@ def lambda_handler(event, context): if 'enddtstr' in qs: dtstr2 = qs['enddtstr'] else: - dtstr2 = datetime.datetime.now().strftime('M%Y%m%d_%H') + dtstr2 = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.000Z') statid = None if 'statid' in qs: statid = qs['statid'] From 0728e0415627d7c9e5884dcff824eb254254a931 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 26 Mar 2026 12:31:42 +0000 Subject: [PATCH 006/287] remove redundant file --- archive/website/templates/searchdialog.js | 122 ---------------------- 1 file changed, 122 deletions(-) delete mode 100644 archive/website/templates/searchdialog.js diff --git a/archive/website/templates/searchdialog.js b/archive/website/templates/searchdialog.js deleted file mode 100644 index e85d171c..00000000 --- a/archive/website/templates/searchdialog.js +++ /dev/null @@ -1,122 +0,0 @@ -// initialize input widgets first -// Copyright (C) 2018-2023 Mark McIntyre - -$('#selectInterval .time').timepicker({ - 'showDuration': true, - 'timeFormat': 'g:ia' -}); - -$('#selectInterval .date').datepicker({ - 'format': 'd/m/yyyy', - 'autoclose': true, - 'endDate': '0' - -}); - -// initialize datepair -var res = document.getElementById("selectInterval"); -var dateSelect = new Datepair(res, { - 'defaultDateDelta': 0, // days - 'defaultTimeDelta': 900000 // milliseconds = 15 minutes -}); - -$('#selectInterval').on('rangeSelected', function(){ - var startdate = $('#selectInterval .date:first').datepicker('getDate'); - var starttime = $('#selectInterval .time:first').timepicker('getTime', startdate); - if(starttime != null){ - var dateval = starttime.getTime(); - var timediff = dateSelect.getTimeDiff(); - var endval = dateval + timediff; - endtime = new Date(); - endtime.setTime(endval); - $('#datestart').text(starttime.toISOString()); - $('#dateend').text(endtime.toISOString()); - if(timediff > 0){ - $('#statusfield').text('Valid range selected'); - }else{ - $('#statusfield').text('Date range must be > 0'); - } - } -}); - -var apiurl = 'https://api.ukmeteors.co.uk/detections'; -var form = document.querySelector("form"); -form.addEventListener("submit", function (event) { - //console.log("Saving value", form.elements.value.value); - var d1 = document.getElementById("datestart").innerHTML; - var d2 = document.getElementById("dateend").innerHTML; - var matchOnly = document.getElementById("matchesOnly").checked; - console.log(matchOnly) - var op = ""; - if (matchOnly == true ) { - op = op + "t:_"; - } - var magSelect = document.getElementById("magselect").value; - if (magSelect != 1 ) { - if(magSelect == 2) {op = op + "m:0_"} - if(magSelect == 3) {op = op + "m:-4_"} - } - var shwrSelect = document.getElementById("shwrselect").value; - if (shwrSelect != 1 ) { - var e = document.getElementById("shwrselect"); - var strshwr = e.options[e.selectedIndex].text; - op = op + "s:" + strshwr + "_"; - } - var statSelect = document.getElementById("statselect").value; - if (statSelect != 1 ) { - var e = document.getElementById("statselect"); - var strstat = e.options[e.selectedIndex].text; - op = op + "l:" + strstat + "_"; - } - //jQuery.support.cors = true; - payload = {"d1": d1, "d2": d2, "opts": op }; - console.log(payload); - document.getElementById("searchresults").innerHTML = "Searching...."; - $.ajax({ - url: apiurl, - type: "GET", - data: payload, - dataType: 'jsonp', - error: function (xhr, status, ex ) { - if (status === 'error' ) { - alert("Too much data, try a narrower range"); - console.log(xhr.status); - } - }, - complete: function (xhr, status) { - } - }); - event.preventDefault();} - ); - -function myFunc(myObj) { - console.log(myObj); - var x, txt = ""; - txt = "
DateTimeSourceShowerMagCameraLink
" - txt += ""; - txt += ""; - for (x in myObj) { - console.log(myObj[x]); - var dtarr = myObj[x].split(','); - var cams = dtarr[4].replaceAll(";", "; "); - if (dtarr[0] > 0 ){ - txt +='"; - } - } - txt += "
DateTimeSourceShowerMagCameraLink
'; - dt = new Date(dtarr[0]*1000); - txt += dt.toISOString(); - txt += ""; - txt += dtarr[1]; - txt += ""; - txt += dtarr[2]; - txt += ""; - txt += dtarr[3]; - txt += ""; - txt += cams; - txt += ""; - errimg = "onerror=\"this.onerror=null;this.src='/img/missing.png';\""; - txt += "\"Image"; - txt += "
"; - document.getElementById("searchresults").innerHTML = txt; -} From 789ad9aec5a8ea9bef27e64a306759bf040eed81 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 26 Mar 2026 12:34:35 +0000 Subject: [PATCH 007/287] update deployment scripts --- archive/deployment/website.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml index 1ee42364..3d429b59 100644 --- a/archive/deployment/website.yml +++ b/archive/deployment/website.yml @@ -37,6 +37,5 @@ - {src: '{{srcdir}}/website/templates/frontpage.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/header.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/reportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/searchdialog.js', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/shwrcsvindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - {src: '{{srcdir}}/website/templates/statreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } From 408ad856e2c4ba0a59a4fb52ca4a72c2883af829 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 26 Mar 2026 13:18:35 +0000 Subject: [PATCH 008/287] improvements in routine to convert GMN report to pandas --- archive/ukmon_pylib/converters/gmnTxtToPandas.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/converters/gmnTxtToPandas.py b/archive/ukmon_pylib/converters/gmnTxtToPandas.py index ad98886f..d274779c 100644 --- a/archive/ukmon_pylib/converters/gmnTxtToPandas.py +++ b/archive/ukmon_pylib/converters/gmnTxtToPandas.py @@ -37,7 +37,7 @@ def loadOneFile(fname): df = df.fillna(value=np.nan) df['jd_beg'] = df.jd_beg.astype(float) df['utc_beg'] = pd.to_datetime(df.utc_beg) - df['ian_uo'] = df.iau_no.astype(int) + df['iau_no'] = df.iau_no.astype(int) for c in range(5,82): try: df[colhdrs[c]] = df[colhdrs[c]].astype(float) @@ -138,4 +138,10 @@ def getStats(): if __name__ == '__main__': - doYear(sys.argv[1]) + if sys.argv[1] == 'doYear': + doYear(sys.argv[2]) + if sys.argv[1] == 'convert': + df = loadOneFile(sys.argv[2]) + fn, _ = os.path.splitext(sys.argv[2]) + df.to_parquet(f'{fn}.parquet.snap', index=False) + From bc82fe93107b421cdbb3a943bdd49ec454afe654 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 26 Mar 2026 13:18:55 +0000 Subject: [PATCH 009/287] more analysis of gmn data --- .../ukmon_pylib/analysis/analyseGmnData.py | 87 ++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/analysis/analyseGmnData.py b/archive/ukmon_pylib/analysis/analyseGmnData.py index 77920b99..749216f3 100644 --- a/archive/ukmon_pylib/analysis/analyseGmnData.py +++ b/archive/ukmon_pylib/analysis/analyseGmnData.py @@ -2,11 +2,40 @@ import pandas as pd import os +import sys +import datetime from converters.gmnTxtToPandas import dirpath -def findDuplicates(yr, mth=None): +def analyseAMonth(fulldf, yr, mth): + if mth != 0: + beg = datetime.datetime(yr, mth, 1) + if mth < 12: + end = datetime.datetime(yr, mth+1, 1) + else: + end = datetime.datetime(yr+1, 1, 1) + mthdf = fulldf[fulldf.utc_beg >= beg] + mthdf = mthdf[mthdf.utc_beg < end] + else: + mthdf = fulldf + tot_num = len(mthdf) + uktraj = mthdf[mthdf.stats.str.contains('UK')] + num_uk_traj = len(uktraj) + allstats = ','.join(uktraj.stats) + stats = list(set(allstats.split(','))) + ukstats = [x for x in stats if 'UK' in x] + otherstats = [x for x in stats if 'UK' not in x] + distinctuk = [x for x in ukstats if '_' not in x] + distinctother = [x for x in otherstats if '_' not in x] + ctrys = list(set([x[:2] for x in otherstats])) + otherctrydf = uktraj[uktraj.stats.str.contains('|'.join(ctrys))] + if mth == 0: + uktraj.to_csv('uk-trajectories.csv', index=False) + return len(distinctuk), len(distinctother), num_uk_traj, tot_num, len(otherctrydf) + + +def findDuplicatesById(yr, mth=None): if mth: datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') else: @@ -18,3 +47,59 @@ def findDuplicates(yr, mth=None): print(duperows) print(len(df)) return duperows + + +def atleastOneStation(stats,statsnext): + if stats is None or statsnext is None: + return False + stats = stats.split(',') + statsnext = statsnext.split(',') + return any(i in statsnext for i in stats) + + +def findDuplicatesByStatAndJD(yr, mth=None, df=None): + if not df: + dirpath = '.' + if mth: + datafile = os.path.join(dirpath, 'parquet', 'monthly', f'gmn_{yr:04d}{mth:02d}.parquet.snap') + else: + datafile = os.path.join(dirpath, 'parquet', f'gmn_{yr:04d}.parquet.snap') + df = pd.read_parquet(datafile) + df['statsnext'] = df.stats.shift(-1) + df['jd_next'] = df.jd_beg.shift(-1) + df['Lat1_next'] = df.Lat1.shift(-1) + df['Lon1_next'] = df.Lon1.shift(-1) + nearjd = df.query('abs(jd_beg - jd_next) < (0.5/86400)') + nearlat = nearjd.query('abs(Lat1 - Lat1_next) < 1 and abs(Lon1 - Lon1_next) < 1') + samestats = nearlat.query('stats == statsnext') + nearlat['overlapstats'] = nearlat.apply(lambda row: atleastOneStation(row.stats, row.statsnext), axis=1) + commonstats = nearlat[nearlat.overlapstats] + print(f'there are {len(df)} trajectories in the period') + print(f'there are {len(nearjd)} ({round(100*len(nearjd)/len(df),2)}%) events within 1s') + print(f'there are {len(commonstats)} ({round(100*len(commonstats)/len(df),2)}%) with at least one common station') + print(f'there are {len(samestats)} ({round(100*len(samestats)/len(df),2)}%) with the same stations') + return nearjd, commonstats, samestats + + +if __name__ == '__main__': + yr = int(sys.argv[1]) + fulldf = None + data = [] + for mth in range(1,13): + print(f'processing month {mth}') + df = pd.read_parquet(f'parquet/monthly/gmn_{yr}{mth:02d}.parquet.snap') + cams, othercams, traj, totaltraj, othertraj = analyseAMonth(df, yr, mth) + data.append([mth, cams, othercams, traj, totaltraj, othertraj]) + fulldf = df if fulldf is None else pd.concat([fulldf, df]) + + result = pd.DataFrame(data, columns=['Mth','Camcount','OtherCams','TrajCount','TotalTraj','NonUKTraj']) + tmpres = result.set_index('Mth') + plot = tmpres.plot(title='UK Statistics') + fig = plot.get_figure() + fig.savefig(f'uk-analysis-{yr}.jpg') + + cams, othercams, traj, totaltraj, otherctry, = analyseAMonth(fulldf, yr, 0) + data.append([0, cams, othercams, traj, totaltraj, otherctry]) + + result = pd.DataFrame(data, columns=['Mth','Camcount','OtherCams','TrajCount','TotalTraj','NonUKTraj']) + result.to_csv('2025-stats.csv', index=False) From 4bc700220647740c5452481c64cb70d1f93b6d2a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 6 Apr 2026 21:57:06 +0100 Subject: [PATCH 010/287] Update the terraform a little --- archive/terraform/mjmm/iam.tf | 2 +- archive/terraform/mjmm/s3.tf | 9 ++++++--- archive/terraform/mjmm/ssm_parameters.tf | 11 ++++++++++- archive/terraform/ukmda/ec2.tf | 4 ++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/archive/terraform/mjmm/iam.tf b/archive/terraform/mjmm/iam.tf index 10e44de3..1da045ac 100644 --- a/archive/terraform/mjmm/iam.tf +++ b/archive/terraform/mjmm/iam.tf @@ -198,7 +198,7 @@ resource "aws_iam_role_policy" "stsAssumeLambda" { Statement = { Action = "sts:AssumeRole" Effect = "Allow" - Resource = "arn:aws:iam::183798037734:policy/s3PolicyForRadio" + Resource = "arn:aws:iam::183798037734:role/s3AccessForRadio" } Version = "2012-10-17" } diff --git a/archive/terraform/mjmm/s3.tf b/archive/terraform/mjmm/s3.tf index 740ba4cc..5a05aef5 100644 --- a/archive/terraform/mjmm/s3.tf +++ b/archive/terraform/mjmm/s3.tf @@ -1,5 +1,6 @@ # Copyright (C) 2018-2023 Mark McIntyre ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmonarchive-co-uk" { bucket = "mjmm-ukmonarchive.co.uk" tags = { @@ -52,8 +53,9 @@ data "aws_iam_policy_document" "websiteacesspolicy" { resources = ["${aws_s3_bucket.mjmm-ukmonarchive-co-uk.arn}/*"] } } - +*/ ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmon-shared" { bucket = "mjmm-ukmon-shared" timeouts {} @@ -113,8 +115,9 @@ resource "aws_s3_bucket_versioning" "shared_versioning" { status = "Suspended" } } - +*/ ######################################################################## +/* resource "aws_s3_bucket" "mjmm-ukmon-live" { bucket = "mjmm-ukmon-live" tags = { @@ -175,4 +178,4 @@ resource "aws_s3_bucket_versioning" "live_versioning" { status = "Suspended" } } - +*/ diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 683a1367..4910397c 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -50,7 +50,16 @@ resource "aws_ssm_parameter" "prod_envname" { resource "aws_ssm_parameter" "prod_calcinstance" { name = "prod_calcinstance" type = "String" - value = "i-04cd701c3cfc980f5" # "i-08cd1d5f6e1056f6b" + value = "i-04cd701c3cfc980f5" # "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "prod_calcuser" { + name = "prod_calcuser" + type = "String" + value = "ec2-user" # "ubuntu" tags = { "billingtag" = "ukmon" } diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/ec2.tf index af11b827..fe124845 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/ec2.tf @@ -47,7 +47,7 @@ resource "aws_network_interface" "calcserver_if" { resource "aws_instance" "ubuntu_calc_server" { ami = "ami-0bdf149a42243bde8" - instance_type = "c6g.4xlarge" + instance_type = "c8g.2xlarge" iam_instance_profile = aws_iam_instance_profile.calcserverrole.name key_name = aws_key_pair.marks_key.key_name tags = { @@ -61,7 +61,7 @@ resource "aws_instance" "ubuntu_calc_server" { "Name" = "calcengine_ub" "billingtag" = "ukmda" } - volume_size = 100 + volume_size = 120 } primary_network_interface { network_interface_id = aws_network_interface.ubuntu_calcserver_if.id From f6ac16fcaffa78d0d88684ed5791a3e492d8c07b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 6 Apr 2026 23:25:11 +0100 Subject: [PATCH 011/287] remove hardcoded reference to ec2-user --- .gitignore | 1 + archive/ukmon_pylib/analysis/gatherDetectionData.py | 4 ++-- archive/ukmon_pylib/analysis/showerAnalysis.py | 2 +- archive/ukmon_pylib/analysis/stationAnalysis.py | 4 ++-- archive/ukmon_pylib/analysis/summaryAnalysis.py | 8 ++++---- archive/ukmon_pylib/converters/gmnTxtToPandas.py | 2 +- archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py | 4 ++-- archive/ukmon_pylib/maintenance/plotStationsOnMap.py | 6 +++--- archive/ukmon_pylib/maintenance/rerunFailedLambdas.py | 6 +++--- archive/ukmon_pylib/metrics/camMetrics.py | 9 +++++---- archive/ukmon_pylib/metrics/costMetrics.py | 2 +- archive/ukmon_pylib/metrics/timingMetrics.py | 4 ++-- archive/ukmon_pylib/reports/CameraDetails.py | 6 +++--- archive/ukmon_pylib/reports/cameraStatusReport.py | 4 ++-- archive/ukmon_pylib/reports/createAnnualBarChart.py | 2 +- archive/ukmon_pylib/reports/createExchangeFiles.py | 2 +- archive/ukmon_pylib/reports/createSearchableFormat.py | 2 +- archive/ukmon_pylib/reports/createSummaryTable.py | 2 +- archive/ukmon_pylib/reports/dailyReport.py | 2 +- archive/ukmon_pylib/reports/extractors.py | 8 ++++---- archive/ukmon_pylib/reports/findBestMp4s.py | 4 ++-- archive/ukmon_pylib/reports/findFireballs.py | 4 ++-- archive/ukmon_pylib/reports/getLivestreamData.py | 2 +- archive/ukmon_pylib/reports/makeCoverageMap.py | 4 ++-- archive/ukmon_pylib/reports/reportActiveShowers.py | 4 ++-- archive/ukmon_pylib/reports/reportBadCameras.py | 2 +- archive/ukmon_pylib/traj/manualBulkReruns.py | 4 ++-- archive/utils/statsToMqtt.sh | 2 +- archive/utils/updateDb.sh | 3 ++- 29 files changed, 56 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index 035e1394..0ea36ffb 100644 --- a/.gitignore +++ b/.gitignore @@ -618,3 +618,4 @@ fbcollector/config.ini usermgmt/windows/stationmaint.ini usermgmt/windows/README.md fbCollector/README.html +archive/containers/trajsolver/WesternMeteorPyLib.old/** diff --git a/archive/ukmon_pylib/analysis/gatherDetectionData.py b/archive/ukmon_pylib/analysis/gatherDetectionData.py index b9031167..d7e45b9b 100644 --- a/archive/ukmon_pylib/analysis/gatherDetectionData.py +++ b/archive/ukmon_pylib/analysis/gatherDetectionData.py @@ -12,7 +12,7 @@ def getUncalibratedImageList(dtstr=None): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') flines = open(logfile, 'r').readlines() uncal = [f for f in flines if 'not recalibrated' in f] @@ -37,7 +37,7 @@ def getRawData(idlist, outpth): def updateSingleDB(yr, consdt): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) sngls = pd.read_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv')) consdf = pd.read_csv(os.path.join(datadir, 'single', 'used', f'consumed_{consdt}.txt'), names=['Filename']) conslist = list(consdf.Filename) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index 29636148..02551d78 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -467,7 +467,7 @@ def magDistributionVis(dta, shwrname, outdir, binwidth=0.2): def showerAnalysis(shwr, dtstr): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) # set up paths, files etc # check if month was passed in diff --git a/archive/ukmon_pylib/analysis/stationAnalysis.py b/archive/ukmon_pylib/analysis/stationAnalysis.py index 519bf479..fb7a2e9d 100644 --- a/archive/ukmon_pylib/analysis/stationAnalysis.py +++ b/archive/ukmon_pylib/analysis/stationAnalysis.py @@ -140,7 +140,7 @@ def reportOneSite(yr, mth, loc, sngl, mful, idlist, outdir): when = f'{mth:02d}-{yr}' idxfile = os.path.join(outdir,'index.html') - templatedir=os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templatedir=os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) shutil.copyfile(os.path.join(templatedir, 'header.html'), idxfile) outf = open(idxfile, 'a+') @@ -291,7 +291,7 @@ def pushToWebsite(fuloutdir, outdir, websitebucket): matchcols = ['_Y_ut','_M_ut','_mag','_mjd','mjd','_stream','orbname','stations'] snglcols = ['ID','Shwr','Dtstamp','Ver', 'M', 'Y'] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camlist = loadLocationDetails() sngl = pd.read_parquet(os.path.join(datadir, 'single', f'singles-{yr}.parquet.snap'), columns=snglcols) diff --git a/archive/ukmon_pylib/analysis/summaryAnalysis.py b/archive/ukmon_pylib/analysis/summaryAnalysis.py index 1aa6117a..5df93076 100644 --- a/archive/ukmon_pylib/analysis/summaryAnalysis.py +++ b/archive/ukmon_pylib/analysis/summaryAnalysis.py @@ -13,7 +13,7 @@ def showerSummaryByPeriod(dtstr): yr = int(dtstr[:4]) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) cols = ['_stream','_M_ut'] filt = None matchfile = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) @@ -34,7 +34,7 @@ def showerSummaryByPeriod(dtstr): def createSummWebpage(dtstr, outdir=None): yr = dtstr[:4] mth = None - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if len(dtstr) > 4: mth = dtstr[4:6] if outdir is None: @@ -43,7 +43,7 @@ def createSummWebpage(dtstr, outdir=None): if outdir is None: outdir = os.path.join(datadir, 'reports', yr, 'showers') os.makedirs(outdir, exist_ok=True) - templatedir=os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templatedir=os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) idxfile = os.path.join(outdir, 'index.html') shutil.copyfile(os.path.join(templatedir, 'header.html'), idxfile) with open(idxfile, 'a+') as outf: @@ -73,7 +73,7 @@ def createSummWebpage(dtstr, outdir=None): def createSummJS(dtstr, shwrdata, maxlines=None): yr = dtstr[:4] mth = None - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if len(dtstr) > 4: mth = dtstr[4:6] outdir = os.path.join(datadir, 'reports', yr, 'showers', mth) diff --git a/archive/ukmon_pylib/converters/gmnTxtToPandas.py b/archive/ukmon_pylib/converters/gmnTxtToPandas.py index d274779c..4e759b6b 100644 --- a/archive/ukmon_pylib/converters/gmnTxtToPandas.py +++ b/archive/ukmon_pylib/converters/gmnTxtToPandas.py @@ -17,7 +17,7 @@ 'Lat1','Lat1sd','Lon1','Lon1sd','H1','H1sd','Lat2','Lat2sd','Lon2','Lon2sd','H2','H2sd', 'Dur','Amag','PkHt','F1','mass','Qc','MedianFitErr','BegIn','EndIn','NumStat','stats'] -datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') +datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) dirpath = os.path.join(datadir, 'gmndata') diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py index f1b47728..2dadf8cb 100644 --- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py +++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py @@ -25,8 +25,8 @@ csvdir = 'f:/videos/meteorcam/usermgmt/csvkeys' lnxdir='/home/ec2-user/keymgmt/csvkeys' else: - csvdir='/home/ec2-user/keymgmt/csvkeys' - lnxdir='/home/ec2-user/keymgmt/csvkeys' + csvdir=os.path.expanduser('~/keymgmt/csvkeys') + csvdir=os.path.expanduser('~/keymgmt/csvkeys') def checkIfOnGMN(camid): diff --git a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py index 28d7b9c0..59b37d3b 100644 --- a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py +++ b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py @@ -107,7 +107,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): # ax.add_feature(cfeature.LAKES, edgecolor='black') # ax.add_feature(cfeature.RIVERS) ax.gridlines() - pylib=os.getenv('PYLIB', default='/home/ec2-user/prod/ukmon_pylib') + pylib=os.getenv('PYLIB', default=os.path.expanduser('~/prod/ukmon_pylib')) os.environ['CARTOPY_USER_BACKGROUNDS'] = os.path.join(pylib, 'share','maps') ax.background_img(name='BM', resolution='high', extent= [minn, maxn, mina, maxa]) @@ -143,7 +143,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): plt.plot(yi, xi, 'bx', markersize=10, linewidth=1) plt.tight_layout() - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if datadir is None: datadir='.' plt.savefig(os.path.join(datadir, 'stations.png'), dpi=200) @@ -158,7 +158,7 @@ def plotMap(srcpath, intersect=False, plotlabels=False): plotlabels = False else: plotlabels = True - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) srcfile = os.path.join(datadir, 'admin', 'cameraLocs.json') intersect = False if len(sys.argv) > 2: diff --git a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py index feb2a507..24107d46 100644 --- a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py +++ b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py @@ -14,7 +14,7 @@ def findOtherBadEvents(): evts = [] - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'dailyreports', 'latest.txt') with open(lastf, 'r') as inf: lis = inf.readlines() @@ -33,7 +33,7 @@ def findOtherBadEvents(): def findFailedEvents(prof='ukmonshared'): #session = boto3.Session(profile_name=prof) logcli = boto3.client('logs', region_name='eu-west-2') - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') if os.path.isfile(lastf): with open(lastf, 'r') as inf: @@ -102,7 +102,7 @@ def rerunFails(fails, prof='ukmonshared'): session=boto3.Session(profile_name=prof) lambd = session.client('lambda', region_name='eu-west-2') - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') with open(lastf, 'w') as outf: rundt = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 09c961dc..e85ff3d6 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -71,7 +71,7 @@ def findRowCamTimings(stationid, uploaddate, ddb=None): def getDayCamTimings(uploaddate, ddb=None, outfile=None, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if not ddb: ddb = boto3.resource('dynamodb', region_name='eu-west-2') #, endpoint_url="http://thelinux:8000") table = ddb.Table('uploadtimes') @@ -136,9 +136,10 @@ def deleteRowCamTimings(stationid, dtstamp, ddb=None): def backPopulate(stationid): s3bucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - fldrs = glob.glob1(f'/home/ec2-user/ukmon-shared/matches/RMSCorrelate/{stationid}/', '*') + basepath = os.path.expanduser('~/prod/ukmon-shared/matches/RMSCorrelate') + fldrs = glob.glob1(os.path.join(basepath, stationid), '*') for fldr in fldrs: - s3objects = glob.glob1(f'/home/ec2-user/ukmon-shared/matches/RMSCorrelate/{stationid}/{fldr}/', 'FTPd*') + s3objects = glob.glob1(os.path.join(basepath, stationid. fldr), 'FTPd*') if len(s3objects) > 0: s3obj = s3objects[0] fullobj = f'matches/RMSCorrelate/{stationid}/{fldr}/{s3obj}' @@ -147,7 +148,7 @@ def backPopulate(stationid): if __name__ == '__main__': - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) ddb = boto3.resource('dynamodb', region_name='eu-west-2') if os.path.isfile('/sys/devices/virtual/dmi/id/board_asset_tag'): # crude check for EC2 diff --git a/archive/ukmon_pylib/metrics/costMetrics.py b/archive/ukmon_pylib/metrics/costMetrics.py index 36f83dd2..9b0236fe 100644 --- a/archive/ukmon_pylib/metrics/costMetrics.py +++ b/archive/ukmon_pylib/metrics/costMetrics.py @@ -17,7 +17,7 @@ def monthlyCostByService(dtstr, acctid): mth = int(dtstr[4:6]) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) df = pd.read_csv(os.path.join(datadir, 'costs', f'costs-{acctid}-90-days.csv')) if acctid != '183798037734': df = df[(df.Tag.str.contains('ukmda')) | (df.Tag.str.contains('ukmon'))] diff --git a/archive/ukmon_pylib/metrics/timingMetrics.py b/archive/ukmon_pylib/metrics/timingMetrics.py index 0f4606da..d88273d0 100644 --- a/archive/ukmon_pylib/metrics/timingMetrics.py +++ b/archive/ukmon_pylib/metrics/timingMetrics.py @@ -47,7 +47,7 @@ def graphOfData(logf, dtstr): plt.grid(axis='x') plt.gca().invert_yaxis() logname, _ = os.path.splitext(os.path.basename(logf)) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) os.makedirs(os.path.join(datadir, 'batchcharts'), exist_ok=True) plt.savefig(os.path.join(datadir, 'batchcharts',f'./{dtstr}-{logname}.jpg'), dpi=100) plt.close() @@ -130,7 +130,7 @@ def getLogStats(nightlogf, matchlogf, thisdy): if __name__ == '__main__': dtstr = sys.argv[1] - logdir = os.path.join(os.getenv('SRC', default='/home/ec2-user/prod'), 'logs') + logdir = os.path.join(os.getenv('SRC',default=os.path.expanduser('~/prod')), 'logs') nowdt = datetime.datetime.now().strftime('%Y%m%d') if nowdt == dtstr: nightf = os.path.join(logdir, 'nightlyJob.log') diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index 74ce7ac8..ab3fd50d 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -13,7 +13,7 @@ def getCamLocDirFov(camid, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camdb = json.load(open(os.path.join(datadir, 'admin', 'cameraLocs.json'))) if camid not in camdb.keys(): return False @@ -22,7 +22,7 @@ def getCamLocDirFov(camid, datadir=None): def updateCamLocDirFovDB(datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) camdb = {} ppfiles = glob.glob(os.path.join(datadir, 'consolidated','platepars', '*.json')) for ppf in ppfiles: @@ -80,7 +80,7 @@ def createCDCsv(targetloc): cd4csv = cd4csv[['site','stationid','oldcode','direction','camtype','dummycode','active']] cd4csv = cd4csv.rename(columns={'stationid':'camid', 'oldcode':'lid', 'direction':'sid'}) cd4csv.sort_values(by=['active','camid'], inplace=True) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data/') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) outfname = os.path.join(datadir, targetloc, 'camera-details.csv') cd4csv.to_csv(outfname, index=False) with open(os.path.join(datadir, 'activecamcount.txt'), 'w') as outf: diff --git a/archive/ukmon_pylib/reports/cameraStatusReport.py b/archive/ukmon_pylib/reports/cameraStatusReport.py index 0872feff..256ad076 100644 --- a/archive/ukmon_pylib/reports/cameraStatusReport.py +++ b/archive/ukmon_pylib/reports/cameraStatusReport.py @@ -25,7 +25,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): caminfo = caminfo.drop(columns=['direction','oldcode','active','camtype','eMail', 'humanName']) caminfo.rename(columns={'site':'Site'}, inplace=True) if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) tmplist = pd.read_csv(os.path.join(datadir,'reports','camuploadtimes.csv'), index_col=False) fldrlist = pd.merge(tmplist, caminfo, on=['stationid'], how='inner') @@ -54,7 +54,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): def createStatusReportJSfile(stati, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) os.makedirs(os.path.join(datadir, 'reports'), exist_ok=True) repfile = os.path.join(datadir, 'reports','camrep.js') with open(repfile, 'w') as outf: diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 05aa32aa..9426f5d6 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -15,7 +15,7 @@ def createBarChart(datadir=None, yr=None): if yr is None: yr=datetime.datetime.now().year if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) fname = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): print('{} missing', fname) diff --git a/archive/ukmon_pylib/reports/createExchangeFiles.py b/archive/ukmon_pylib/reports/createExchangeFiles.py index 820650ab..e11a5588 100644 --- a/archive/ukmon_pylib/reports/createExchangeFiles.py +++ b/archive/ukmon_pylib/reports/createExchangeFiles.py @@ -164,7 +164,7 @@ def createCameraFile(datadir): def createAll(targdate=None, datadir=None): if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/datas') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if targdate is None: targdate = datetime.datetime.now() createCameraFile(datadir) diff --git a/archive/ukmon_pylib/reports/createSearchableFormat.py b/archive/ukmon_pylib/reports/createSearchableFormat.py index e2df79c2..4424cc4b 100644 --- a/archive/ukmon_pylib/reports/createSearchableFormat.py +++ b/archive/ukmon_pylib/reports/createSearchableFormat.py @@ -110,7 +110,7 @@ def convertMatchToSrchable(datadir, year, newonly=True): print('usage: python createSearchableFormat.py year mode outdir') exit(1) else: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) year = sys.argv[1] mode = sys.argv[2] diff --git a/archive/ukmon_pylib/reports/createSummaryTable.py b/archive/ukmon_pylib/reports/createSummaryTable.py index cff8e40c..a0d861bb 100644 --- a/archive/ukmon_pylib/reports/createSummaryTable.py +++ b/archive/ukmon_pylib/reports/createSummaryTable.py @@ -18,7 +18,7 @@ def createSummaryTable(curryr=None, datadir=None): if curryr is None: curryr = str(datetime.datetime.now().year) if datadir is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) fname = os.path.join(datadir, 'summarytable.js') with open(fname, 'w') as f: f.write('$(function() {\n') diff --git a/archive/ukmon_pylib/reports/dailyReport.py b/archive/ukmon_pylib/reports/dailyReport.py index 8c785207..a511c405 100644 --- a/archive/ukmon_pylib/reports/dailyReport.py +++ b/archive/ukmon_pylib/reports/dailyReport.py @@ -115,7 +115,7 @@ def LookForMatchesRMS(doff, dayfile, statsfile): # now send the post to groups.io targeturl='https://archive.ukmeteors.co.uk' - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) recs = open(os.path.join(datadir, 'admin','dailyReportRecips.txt'), 'r').readlines() mailFrom = 'markmcintyre99@googlemail.com' mailRecip = recs[0].strip() diff --git a/archive/ukmon_pylib/reports/extractors.py b/archive/ukmon_pylib/reports/extractors.py index d7fc4bb8..65670b38 100644 --- a/archive/ukmon_pylib/reports/extractors.py +++ b/archive/ukmon_pylib/reports/extractors.py @@ -19,7 +19,7 @@ def createSplitMatchFile(yr, mth=None, shwr=None, matches=None): shwr (string): optional shower code. If provided, the data will be filtered """ - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if matches is None: infname = os.path.join(datadir, 'matched',f'matches-full-{yr}.parquet.snap') if not os.path.isfile(infname): @@ -58,7 +58,7 @@ def createUFOSingleMonthlyExtract(yr, mth=None, shwr=None, dta=None): """ # print('ufo singles file') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if dta is None: fname = os.path.join(datadir, 'consolidated','M_{}-unified.csv'.format(yr)) if not os.path.isfile(fname): @@ -94,7 +94,7 @@ def createRMSSingleMonthlyExtract(yr, mth=None, shwr=None, dta=None, withshower= """ #print(f'rms singles file, withshower {withshower}') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if dta is None: fname = os.path.join(datadir, 'single','singles-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): @@ -147,7 +147,7 @@ def extractAllShowersData(ymd): showerlist = sl.getMajorShowers(True, True).strip().split(' ') print(f'processing data for {ymd}') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) infname = os.path.join(datadir, 'matched',f'matches-full-{yr}.parquet.snap') if not os.path.isfile(infname): print(f'unable to open {infname}') diff --git a/archive/ukmon_pylib/reports/findBestMp4s.py b/archive/ukmon_pylib/reports/findBestMp4s.py index a07611dd..22f53370 100644 --- a/archive/ukmon_pylib/reports/findBestMp4s.py +++ b/archive/ukmon_pylib/reports/findBestMp4s.py @@ -20,7 +20,7 @@ def getBestNMatches(reqdate=None, numtoget=10): tod = reqdate + datetime.timedelta(days=1) yr = reqdate.year - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') # select only the columns we need @@ -81,7 +81,7 @@ def getUrlFromFilename(fname): def getBestNMp4s(yr, mth, numtoget): - datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') # select only the columns we need diff --git a/archive/ukmon_pylib/reports/findFireballs.py b/archive/ukmon_pylib/reports/findFireballs.py index 60b7af0c..d30949fa 100644 --- a/archive/ukmon_pylib/reports/findFireballs.py +++ b/archive/ukmon_pylib/reports/findFireballs.py @@ -19,7 +19,7 @@ # Manually mark a trajectoriy as a "fireball" # def markAsFireball(trajname, tof=True): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) yr=trajname[:4] if int(yr) > 2021: fname = os.path.join(datadir, 'matched','matches-full-{}.parquet.snap'.format(yr)) @@ -142,7 +142,7 @@ def findFBPre2020(df, mag=-4): def findFireballs(dtval, shwr, minmag=-3.99, matchdataset = None): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) orbdir = 'matches' # check if month was passed in diff --git a/archive/ukmon_pylib/reports/getLivestreamData.py b/archive/ukmon_pylib/reports/getLivestreamData.py index 3db0dfe2..18c15b1d 100644 --- a/archive/ukmon_pylib/reports/getLivestreamData.py +++ b/archive/ukmon_pylib/reports/getLivestreamData.py @@ -72,7 +72,7 @@ def getLatestLiveFiles(daysback=None, df=None): if __name__ == '__main__': - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) df = getLatestLiveFiles(-30) prvfile = os.path.join(datadir, 'ukmonlive', 'livefeed.csv') if os.path.isfile(prvfile): diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index 2bdb89c5..bbb7f0fb 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -58,8 +58,8 @@ def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): def createCoveragePage(): apikey = os.getenv('APIKEY') apikey = decodeApiKey(apikey) - templdir = os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) with open(os.path.join(templdir, 'coverage-maps.html'), 'r') as inf: lis = inf.readlines() with open(os.path.join(datadir, 'latest','coverage-maps.html'), 'w') as outf: diff --git a/archive/ukmon_pylib/reports/reportActiveShowers.py b/archive/ukmon_pylib/reports/reportActiveShowers.py index 7786eafa..7a58d91f 100644 --- a/archive/ukmon_pylib/reports/reportActiveShowers.py +++ b/archive/ukmon_pylib/reports/reportActiveShowers.py @@ -15,7 +15,7 @@ def createShowerIndexPage(dtstr, shwr, shwrname, outdir, datadir): - templdir = os.getenv('TEMPLATES', default='/home/ec2-user/prod/website/templates') + templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) idxfile = os.path.join(outdir, 'index.html') shutil.copyfile(os.path.join(templdir,'header.html'), idxfile) @@ -175,7 +175,7 @@ def reportActiveShowers(ymd, thisshower=None, thismth=None, includeMinor=False): else: shwrlist = [thisshower] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) # pltdir=os.path.join(datadir, 'showerplots') dtstr = ymd[:4] if thismth is not None: diff --git a/archive/ukmon_pylib/reports/reportBadCameras.py b/archive/ukmon_pylib/reports/reportBadCameras.py index bdf4f6ef..cc0938ee 100644 --- a/archive/ukmon_pylib/reports/reportBadCameras.py +++ b/archive/ukmon_pylib/reports/reportBadCameras.py @@ -19,7 +19,7 @@ print('usage: python reportBadCameras.py daysback') exit(0) - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) if datadir is None: print('define DATADIR first') exit(1) diff --git a/archive/ukmon_pylib/traj/manualBulkReruns.py b/archive/ukmon_pylib/traj/manualBulkReruns.py index d9451057..497310da 100644 --- a/archive/ukmon_pylib/traj/manualBulkReruns.py +++ b/archive/ukmon_pylib/traj/manualBulkReruns.py @@ -406,7 +406,7 @@ def prepDataForRange(startdt, enddt): # working out what has been used, then collecting the FTPs and platepars, # and finally removing the already-used meteors from the FTPs # - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) sdt = datetime.datetime.strptime(startdt, '%Y%m') edt = datetime.datetime.strptime(enddt, '%Y%m') smth = sdt.month @@ -430,7 +430,7 @@ def prepDataForRange(startdt, enddt): def moveInterestingCands(): - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) pickledir = os.path.join(datadir, 'historic', 'candidates') goodcandr = os.path.join(datadir, 'historic', 'goodcands') os.makedirs(goodcandr, exist_ok=True) diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh index 3235281a..7725469b 100644 --- a/archive/utils/statsToMqtt.sh +++ b/archive/utils/statsToMqtt.sh @@ -13,7 +13,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi elif [ "$(hostname)" == "ukmcalcserver" ] ; then - source /home/ec2-user/venvs/wmpl/bin/activate + source $HOME/venvs/wmpl/bin/activate python $here/statsToMqtt.py diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') if [ $diskpct -gt 80 ] ; then diff --git a/archive/utils/updateDb.sh b/archive/utils/updateDb.sh index 2b8df8c2..501497f0 100644 --- a/archive/utils/updateDb.sh +++ b/archive/utils/updateDb.sh @@ -11,10 +11,11 @@ else rundt=$1 fi +cd ~/prod/data/brightness mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; -load data local infile '/home/ec2-user/prod/data/brightness/CaptureNight_${rundt}.csv' +load data local infile './CaptureNight_${rundt}.csv' into table brightness fields terminated by ','; select count(*) from ukmon.brightness; EOD From 33044836d52017299bd4c2ab19ff71115d29c408 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 6 Apr 2026 23:28:27 +0100 Subject: [PATCH 012/287] add SERVERUSERID env /ssm variable for calcserver user id --- archive/server_setup/.bash_aliases | 4 ++-- archive/ukmon_pylib/traj/distributeCandidates.py | 11 ++++++++--- archive/utils/makeConfig.sh | 4 +++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index 7676e445..f39f446a 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -37,9 +37,9 @@ function calcserver { ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) isrunning=$(echo $sts | cut -d " " -f 1) if [ $isrunning -ne 16 ] ; then - /home/ec2-user/prod/utils/stopstart-calcengine.sh start + $HOME/prod/utils/stopstart-calcengine.sh start echo "starting server on ${ipaddr}... waiting 10s..." sleep 10 fi - ssh -i ~/.ssh/markskey.pem ec2-user@$ipaddr + ssh -i ~/.ssh/markskey.pem $SERVERUSERID@$ipaddr } diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 91f185f4..861b09bd 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -3,6 +3,8 @@ # Used when running the correlator in 'distributed' mode. # Pick up new groups of candidate trajectories and distribute them, # then wait for completion and gather the logs. + +# NB this runs on the calculation engine server # import json @@ -147,7 +149,7 @@ def monitorProgress(rundate): client = boto3.client('ecs', region_name='eu-west-2') s3 = boto3.client('s3') archbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) templdir,_ = os.path.split(__file__) clusdets = getClusterDetails(templdir) @@ -318,7 +320,7 @@ def getMissedLogs(picklefile): taskarns = dumpdata[1] loggrp = '/ecs/trajcont' contname = 'trajcont' - logdir = '/home/ec2-user/prod/logs/distrib' + logdir = os.path.expanduser('~/prod/logs/distrib') for thisarn in taskarns: tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') with open(tmpfname, 'w') as outf: @@ -364,7 +366,10 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): if __name__ == '__main__': if len(sys.argv) < 4: rundt = datetime.datetime(2022,4,21) - srcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates' # hardcoded on calcserver + # runs on the calcserver + csuser = os.getenv('SERVERUSERID', default='ec2-user') + srcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate/candidates' # hardcoded on calcserver + buck = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') targdir = f'{buck}/matches/distrib' else: diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index a5687db6..9d44d62a 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -20,6 +20,7 @@ SERVERINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_ BKPINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_backupinstance --query Parameters[0].Value | tr -d '"') SERVERSSHKEY=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_sshkey --query Parameters[0].Value | tr -d '"') NJLOGGRP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_batchloggroup --query Parameters[0].Value | tr -d '"') +SERVERUSERID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcuser --query Parameters[0].Value | tr -d '"') # hardcoded PYLIB=$SRC/ukmon_pylib @@ -50,6 +51,7 @@ echo "TEMPLATES=${TEMPLATES}" >> ${CFGFILE} echo "RCODEDIR=${RCODEDIR}" >> ${CFGFILE} echo "DATADIR=${DATADIR}" >> ${CFGFILE} echo "SERVERINSTANCEID=${SERVERINSTANCEID}" >> ${CFGFILE} +echo "SERVERUSERID=${SERVERUSERID}" >> ${CFGFILE} echo "BKPINSTANCEID=${BKPINSTANCEID}" >> ${CFGFILE} echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> ${CFGFILE} echo "RMS_ENV=${RMS_ENV}" >> ${CFGFILE} @@ -69,7 +71,7 @@ echo "export PYLIB TEMPLATES RCODEDIR DATADIR BKPINSTANCEID AWS_DEFAULT_REGION" echo "export RMS_ENV RMS_LOC WMPL_ENV WMPL_LOC" >> ${CFGFILE} echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFILE} echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} -echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID NJLOGGRP" >> ${CFGFILE} +echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID SERVERUSERID NJLOGGRP" >> ${CFGFILE} echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib" >> ${CFGFILE} # enable miniconda echo "source ~/.condaon" >> ${CFGFILE} From 0f80d43b67e3bc6794e8ed4d50a7534f90f3901a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 6 Apr 2026 23:34:49 +0100 Subject: [PATCH 013/287] update container for new distrib-processing method --- archive/containers/trajsolver/Dockerfile | 2 + archive/containers/trajsolver/build.ps1 | 2 +- archive/containers/trajsolver/trajsolver.py | 54 +++++++++++--------- archive/containers/trajsolver/update_wmpl.sh | 2 +- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/archive/containers/trajsolver/Dockerfile b/archive/containers/trajsolver/Dockerfile index ff8b6ff5..47f4ac3f 100644 --- a/archive/containers/trajsolver/Dockerfile +++ b/archive/containers/trajsolver/Dockerfile @@ -11,6 +11,8 @@ RUN conda install -y matplotlib RUN pip install boto3 RUN apt-get update && apt-get -y upgrade RUN conda install -y -c conda-forge numba +RUN conda install -y -c conda-forge libsqlite sqlite +RUN conda install -y -c conda-forge paramiko COPY WesternMeteorPyLib/ ./WesternMeteorPyLib ENV PYTHONPATH=./WesternMeteorPyLib diff --git a/archive/containers/trajsolver/build.ps1 b/archive/containers/trajsolver/build.ps1 index 4a4dac04..c5bb2a3f 100644 --- a/archive/containers/trajsolver/build.ps1 +++ b/archive/containers/trajsolver/build.ps1 @@ -15,7 +15,7 @@ $repo = "calcengine/${imagename}" write-output "building $imagename in $accid" $yn=read-host -prompt "update WMPL?" -if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } +if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } if ($env -eq "test") { move-item awskeys.test awskeys} diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py index 2d42835f..1e3de667 100644 --- a/archive/containers/trajsolver/trajsolver.py +++ b/archive/containers/trajsolver/trajsolver.py @@ -25,7 +25,7 @@ def runCorrelator(dir_path, time_beg, time_end): saveplots = True velpart = 0.40 uncerttime = False - distribute = 2 + distribute = 3 trajectory_constraints = TrajectoryConstraints() trajectory_constraints.max_toffset = maxtoffset @@ -42,8 +42,8 @@ def runCorrelator(dir_path, time_beg, time_end): event_time_range = None # Extract time range - dt_beg = datetime.datetime.strptime(time_beg, "%Y%m%d-%H%M%S") - dt_end = datetime.datetime.strptime(time_end, "%Y%m%d-%H%M%S") + dt_beg = datetime.datetime.strptime(time_beg, "%Y%m%d-%H%M%S").replace(tzinfo=datetime.timezone.utc) + dt_end = datetime.datetime.strptime(time_end, "%Y%m%d-%H%M%S").replace(tzinfo=datetime.timezone.utc) print("Custom time range:") print(" BEG: {:s}".format(str(dt_beg))) @@ -52,17 +52,7 @@ def runCorrelator(dir_path, time_beg, time_end): event_time_range = [dt_beg, dt_end] # Init the data handle - dh = RMSDataHandle(dir_path, event_time_range) - - # If there is nothing to process, stop, unless we're in distributed - # processing mode 2 - if not dh.processing_list and distribute !=2: - print() - print("Nothing to process!") - print("Probably everything is already processed.") - print("Exiting...") - sys.exit() - + dh = RMSDataHandle(dir_path, dt_range=event_time_range, mcmode=distribute) ### GENERATE MONTHLY TIME BINS ### @@ -109,12 +99,15 @@ def runCorrelator(dir_path, time_beg, time_end): print() # Load data of unprocessed observations - dh.unpaired_observations = dh.loadUnpairedObservations(dh.processing_list, dt_range=(bin_beg, bin_end)) + #dh.unpaired_observations = dh.loadUnpairedObservations(dh.processing_list, dt_range=(bin_beg, bin_end)) # Run the trajectory correlator - tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, distribute=distribute, enableOSM=True) - tc.run(event_time_range=event_time_range) + tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, enableOSM=True) + tc.run(event_time_range=event_time_range, mcmode=3) + dh.closeObservationsDatabase() + dh.closeTrajectoryDatabase() + print("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) return @@ -162,6 +155,8 @@ def getExtraArgs(fname): ctyp = 'application/json' elif file_ext=='.zip': ctyp = 'application/zip' + elif file_ext=='.db': + ctyp = 'application/vnd.sqlite3' extraargs = {'ContentType': ctyp} return extraargs @@ -249,7 +244,7 @@ def startup(srcfldr, startdt, enddt, isTest=False): print(f'fetching data from {srcbucket}/{srckey} saving to {outbucket} and {webbucket}') objlist = s3.meta.client.list_objects_v2(Bucket=srcbucket,Prefix=srckey) - print(objlist) + #print(objlist) if objlist['KeyCount'] > 0: keys = objlist['Contents'] for k in keys: @@ -257,24 +252,37 @@ def startup(srcfldr, startdt, enddt, isTest=False): if '.pickle' in fname: _, locfname = os.path.split(fname) targfile = os.path.join(canddir, locfname) - print(f'downloading {locfname}') + print(f'downloading {locfname} to {targfile}') s3.meta.client.download_file(srcbucket, fname, targfile) runCorrelator(localfldr, startdt, enddt) print('uploading data to website') trajfldr = os.path.join(localfldr,'trajectories') - # reacquire tokens just in case the 1hour time limit on chained roles is exceeded + # reacquire tokens just in case the 1 hour time limit on chained roles is exceeded s3 = getS3Client() pushToWebsite(s3, trajfldr, webbucket, webpth, outbucket, outpth) + if srcfldr[:5] == 'test/': + srcfldr = srcfldr[5:] + for dbname in ['observations', 'trajectories', 'candidates']: + + fname = f'{dbname}_{srcfldr}.db' + localfile = os.path.join(localfldr, f'{dbname}.db') + if os.path.isfile(localfile): + targkey = f'{outpth}/{fname}' + print(f'uploading {localfile} to {srcbucket}/{targkey}') + s3.meta.client.upload_file(localfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + fname = f'{srcfldr}.json' jsonfile = os.path.join(localfldr, 'processed_trajectories.json') - targkey = f'{srcpth}/{fname}' - print(f'uploading {jsonfile} to {srcbucket}/{srcpth}') - s3.meta.client.upload_file(jsonfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + if os.path.isfile(jsonfile): + targkey = f'{srcpth}/{fname}' + print(f'uploading {jsonfile} to {srcbucket}/{srcpth}') + s3.meta.client.upload_file(jsonfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) else: print('no files found') + print(f"Finished at {datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}") return diff --git a/archive/containers/trajsolver/update_wmpl.sh b/archive/containers/trajsolver/update_wmpl.sh index 5fa92c4b..1f2db1f3 100644 --- a/archive/containers/trajsolver/update_wmpl.sh +++ b/archive/containers/trajsolver/update_wmpl.sh @@ -8,7 +8,7 @@ else fi cd /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive/containers/$targ/ -ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout forcontainer && git pull" +ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout distrib_processing && git pull" rsync -avz ukmonhelper2:src/wmpldev/* ./WesternMeteorPyLib/ --exclude "build/" --exclude "*.egg*" --exclude "dist/" --exclude ".git/" --exclude "__pycache__/" ssh ukmonhelper2 "cd src/wmpldev && git checkout - && git stash apply" From 1c50cdd3119c047886f7042ae6d7376cf800b102 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 6 Apr 2026 23:36:33 +0100 Subject: [PATCH 014/287] use calcserver SERVERUSERID variable --- archive/analysis/updatePlotsAndDetStatus.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index 948846cb..479acf4b 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -72,18 +72,18 @@ done log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" updatePlotsAndDetStatus -scp -i $SERVERSSHKEY $execrerunsh ec2-user@$privip:data/distrib/$execrerun +scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" updatePlotsAndDetStatus - scp -i $SERVERSSHKEY $execrerunsh ec2-user@$privip:data/distrib/$execrerun + scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun done # push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py ec2-user@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj # now run the script -ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execrerun" +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execrerun" log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" updatePlotsAndDetStatus aws ec2 stop-instances --instance-ids $SERVERINSTANCEID From 8f1668871e5e2301b44f9e2d04e6c879b872c516 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 12:41:23 +0100 Subject: [PATCH 015/287] support for test ECS containers --- archive/terraform/ukmda/ecscluster-test.tf | 79 +++++++++++++++++++ .../traj/clusdetailstest-ukmda.txt | 6 ++ archive/ukmon_pylib/traj/taskrunner_test.json | 44 +++++++++++ 3 files changed, 129 insertions(+) create mode 100644 archive/terraform/ukmda/ecscluster-test.tf create mode 100644 archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt create mode 100644 archive/ukmon_pylib/traj/taskrunner_test.json diff --git a/archive/terraform/ukmda/ecscluster-test.tf b/archive/terraform/ukmda/ecscluster-test.tf new file mode 100644 index 00000000..9496bed2 --- /dev/null +++ b/archive/terraform/ukmda/ecscluster-test.tf @@ -0,0 +1,79 @@ +# terraform to create ECS cluster +# Copyright (C) 2018-2023 Mark McIntyre + +# create a cluster +resource "aws_ecs_cluster" "trajsolvertest" { + name = "trajsolvertest" + tags = { + "billingtag" = "ukmda" + } +} + +# declare the capacity provider type, in this case FARGATE +resource "aws_ecs_cluster_capacity_providers" "trajsolvertest_cap" { + cluster_name = aws_ecs_cluster.trajsolvertest.name + capacity_providers = ["FARGATE", "FARGATE_SPOT"] +} + +data "template_file" "tasktest_json_template" { + template = file("files/trajsolver/trajsolver_container.json") + vars = { + acctid = data.aws_caller_identity.current.account_id + regionid = "eu-west-2" + repoid = "calcengine/trajsolvertest" + contname = var.containername + loggrp = var.ecsloggroup + } +} + +# define the task +# the definition of the container it runs are in the webapp.json file +resource "aws_ecs_task_definition" "trajsolvertest_task" { + family = "trajsolvertest" + container_definitions = data.template_file.tasktest_json_template.rendered + cpu = 4096 + memory = 8192 + network_mode = "awsvpc" + tags = { + billingtag = "ukmda" + } + requires_compatibilities = ["FARGATE"] + execution_role_arn = aws_iam_role.ecstaskrole.arn + task_role_arn = aws_iam_role.ecstaskrole.arn + runtime_platform { + operating_system_family = "LINUX" + } +} + +/* +# print out some results - clustername, sec grp, subnet and role arn +output "clusname" { value = aws_ecs_cluster.trajsolver.name } +output "secgrpid" { value = aws_security_group.ecssecgrp.id } +output "subnetid" { value = aws_subnet.ecs_subnet.id } +output "taskrolearn" { value = aws_iam_role.ecstaskrole.arn } +output "loggrp" { value = var.ecsloggroup } +output "contname" { value = var.containername } +*/ +# create a local file containing the clustername and a few other details +# +resource "null_resource" "createECSdetailstest" { + triggers = { + clusname = join(",", tolist([aws_ecs_cluster.trajsolvertest.name, + aws_subnet.ecs_subnet.id, + aws_security_group.ecssecgrp.id, + aws_iam_role.ecstaskrole.arn, var.ecsloggroup, + var.containername])) + } + provisioner "local-exec" { + command = "echo $env:CLUSNAME $env:SECGRP $env:SUBNET $env:IAMROLE $env:LOGGRP $env:CONTNAME > ../../ukmon_pylib/traj/clusdetailstest-ukmda.txt" + interpreter = ["pwsh.exe", "-command"] + environment = { + CLUSNAME = "${aws_ecs_cluster.trajsolvertest.name}" + SECGRP = "${aws_security_group.ecssecgrp.id}" + SUBNET = "${aws_subnet.ecs_subnet.id}" + IAMROLE = "${aws_iam_role.ecstaskrole.arn}" + LOGGRP = "${var.ecsloggroup}" + CONTNAME = "${var.containername}" + } + } +} diff --git a/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt new file mode 100644 index 00000000..ca56c686 --- /dev/null +++ b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt @@ -0,0 +1,6 @@ +trajsolvertest +sg-06fb7bbee3b4f5e94 +subnet-019eef41d5eaf419b +arn:aws:iam::183798037734:role/ecsTaskExecutionRole +/ecs/trajcont +trajcont diff --git a/archive/ukmon_pylib/traj/taskrunner_test.json b/archive/ukmon_pylib/traj/taskrunner_test.json new file mode 100644 index 00000000..3957ccae --- /dev/null +++ b/archive/ukmon_pylib/traj/taskrunner_test.json @@ -0,0 +1,44 @@ +{ + "cluster": "trajsolvertest", + "count": 1, + "enableECSManagedTags": true, + "enableExecuteCommand": true, + "group": "trajsolvergrp", + "launchType": "FARGATE", + "networkConfiguration": { + "awsvpcConfiguration": { + "subnets":[], + "securityGroups": [], + "assignPublicIp": "ENABLED" + } + }, + "overrides": { + "containerOverrides": [ + { + "name": "trajcont", + "command": [], + "environment": [ + { + "name": "SRCPATH", + "value": "s3://ukmda-shared/matches/distrib/test" + }, + { + "name": "OUTPATH", + "value": "s3://ukmda-shared/matches/distrib/test" + }, + { + "name": "WEBPATH", + "value": "s3://ukmda-website/dummy" + } + ] + } + ], + "executionRoleArn": "", + "taskRoleArn": "" + }, + "platformVersion": "LATEST", + "propagateTags": "TASK_DEFINITION", + "referenceId": "", + "startedBy": "cli", + "taskDefinition": "trajsolvertest" +} From a7c6a8de25830f917b3b9637f2dbd78a13d84005 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 15:21:11 +0100 Subject: [PATCH 016/287] fix paths in deployment scripts --- archive/deployment/analysis.yml | 3 +-- archive/deployment/config.yml | 2 +- archive/deployment/database.yml | 2 +- archive/deployment/dev-vars.yml | 3 --- archive/deployment/prod-vars.yml | 3 --- archive/deployment/pylib.yml | 2 +- archive/deployment/server_setup.yml | 2 +- archive/deployment/shwrinfo.yml | 2 +- archive/deployment/utils.yml | 2 +- archive/deployment/website.yml | 2 +- archive/deployment/website_static.yml | 2 +- 11 files changed, 9 insertions(+), 16 deletions(-) diff --git a/archive/deployment/analysis.yml b/archive/deployment/analysis.yml index 8e217f41..f6506869 100644 --- a/archive/deployment/analysis.yml +++ b/archive/deployment/analysis.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml @@ -25,7 +25,6 @@ - {src: '{{srcdir}}/analysis/getBadStations.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/getLogData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/getRMSSingleData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/onlyConsolDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/reportActiveShowers.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/runDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - {src: '{{srcdir}}/analysis/showerReport.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } diff --git a/archive/deployment/config.yml b/archive/deployment/config.yml index b9823e58..88ad1408 100644 --- a/archive/deployment/config.yml +++ b/archive/deployment/config.yml @@ -2,7 +2,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: dev-vars.yml diff --git a/archive/deployment/database.yml b/archive/deployment/database.yml index 1b75b1f5..3abf9d78 100644 --- a/archive/deployment/database.yml +++ b/archive/deployment/database.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/dev-vars.yml b/archive/deployment/dev-vars.yml index fb60fbb1..86dfcf57 100644 --- a/archive/deployment/dev-vars.yml +++ b/archive/deployment/dev-vars.yml @@ -1,6 +1,3 @@ destdir: /home/ec2-user/dev env: DEV -websitebucket: mjmm-ukmonarchive.co.uk -#api_id: 0zbnc358p0 -#api_svc: test s3profile: default diff --git a/archive/deployment/prod-vars.yml b/archive/deployment/prod-vars.yml index d26be999..20171b91 100644 --- a/archive/deployment/prod-vars.yml +++ b/archive/deployment/prod-vars.yml @@ -1,6 +1,3 @@ destdir: /home/ec2-user/prod env: PROD -websitebucket: ukmda-website -#api_id: 40luvfh1od -#api_svc: Prod s3profile: ukmda_admin \ No newline at end of file diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml index 46a4afc7..f119ed1e 100644 --- a/archive/deployment/pylib.yml +++ b/archive/deployment/pylib.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/server_setup.yml b/archive/deployment/server_setup.yml index f621dc27..661d96de 100644 --- a/archive/deployment/server_setup.yml +++ b/archive/deployment/server_setup.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/shwrinfo.yml b/archive/deployment/shwrinfo.yml index 17a6110e..c7bbfd11 100644 --- a/archive/deployment/shwrinfo.yml +++ b/archive/deployment/shwrinfo.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/utils.yml b/archive/deployment/utils.yml index f122d7cb..73dd5f02 100644 --- a/archive/deployment/utils.yml +++ b/archive/deployment/utils.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml index 3d429b59..78956dda 100644 --- a/archive/deployment/website.yml +++ b/archive/deployment/website.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml diff --git a/archive/deployment/website_static.yml b/archive/deployment/website_static.yml index cbe14667..d8f79f70 100644 --- a/archive/deployment/website_static.yml +++ b/archive/deployment/website_static.yml @@ -3,7 +3,7 @@ vars_files: - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/archive" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" tasks: - name: import dev variables include_vars: ./dev-vars.yml From 6930fac681b5e7fd8610f9f2525f7580ca0d9e13 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 15:21:19 +0100 Subject: [PATCH 017/287] remove unused file --- archive/analysis/onlyConsolDistrib.sh | 122 -------------------------- 1 file changed, 122 deletions(-) delete mode 100644 archive/analysis/onlyConsolDistrib.sh diff --git a/archive/analysis/onlyConsolDistrib.sh b/archive/analysis/onlyConsolDistrib.sh deleted file mode 100644 index b3725425..00000000 --- a/archive/analysis/onlyConsolDistrib.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# just consolidate solutions of distributed candidates -# -# Used if consolidating a bunch of candidates that were solved outside the normal process -# -# consumes: the trajdb and solution json files -# creates: an updated trajdb and hopefully additional orbits -# - -""" -To use this function, first create the candidate pickle files, then copy them to the candidates -folder on the calcserver and submit them to the distributed processing engine with the following: - -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} -export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib -export AWS_PROFILE=ukmonshared -cd /home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates -time python -m traj.distributeCandidates 20230113 /home/ec2-user/ukmon-shared/matches/RMSCorrelate/candidates $UKMONSHAREDBUCKET/matches/distrib - -when this completes, logoff the calcserver and shut it down again -On the ukmonhelper run this script to wait for the distrib processing to finish and then -merge in the data - - -""" - - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -logger -s -t onlyConsolDistrib "starting onlyConsolDistrib" - -# load the configuration -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -# set the profile to the EE account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared - - -if [ "$1" == "" ] ; then - rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -else - rundate=$1 -fi -logger -s -t onlyConsolDistrib "consolidating for $rundate" - -python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}'); " - -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) -while [ "$privip" == "" ] ; do - sleep 5 - echo "getting ipaddress" - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) -done - -if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then - aws s3 sync $UKMONSHAREDBUCKET/matches/distrib/ $DATADIR/distrib/ --exclude "*" --include "*.json" --quiet - cp -f $DATADIR/distrib/processed_trajectories.json $DATADIR/distrib/prev_processed_trajectories.json - - numtoconsol=$(ls -1 $DATADIR/distrib/${rundate}*.json | wc -l) - if [ $numtoconsol -gt 5 ] ; then - logger -s -t onlyConsolDistrib "restarting calcserver to consolidate results" - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text --profile ukmonshared) - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - fi - logger -s -t onlyConsolDistrib "waiting for the server to be ready" - while [ "$stat" -ne 16 ]; do - sleep 30 - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - fi - echo "checking - status is ${stat}" - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text --profile ukmonshared) - done - - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - while [ $? -ne 0 ] ; do - # in case the server isn't responding to ssh sessions yet - sleep 10 - echo "server not responding yet, retrying" - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - done - scp -i $SERVERSSHKEY $DATADIR/distrib/${rundate}*.json ec2-user@$privip:data/distrib - - echo "#!/bin/bash" > /tmp/execConsol.sh - echo "export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib" >> /tmp/execConsol.sh - echo "python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json ${rundate}" >> /tmp/execConsol.sh - chmod +x /tmp/execConsol.sh - scp -i $SERVERSSHKEY /tmp/execConsol.sh ec2-user@$privip:data/distrib - ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/execConsol.sh" - - scp -i $SERVERSSHKEY ec2-user@$privip:data/distrib/processed_trajectories.json $DATADIR/distrib - - ssh -i $SERVERSSHKEY ec2-user@$privip "rm -f data/distrib/*.json" - - logger -s -t runDistrib "stopping calcserver again" - aws ec2 stop-instances --instance-ids $SERVERINSTANCEID --profile ukmonshared - - python -c "from traj.consolidateDistTraj import patchTrajDB ; patchTrajDB('$DATADIR/distrib/processed_trajectories.json','/home/ec2-user/ukmon-shared/matches/RMSCorrelate', '/home/ec2-user/data/distrib');" - else - python -m traj.consolidateDistTraj $DATADIR/distrib $DATADIR/distrib/processed_trajectories.json $rundate - fi - # push the updated traj db to the S3 bucket - aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - - logger -s -t onlyConsolDistrib "compressing the procssed data" - gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz - aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet - tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib/done/ --quiet - rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 rm $UKMONSHAREDBUCKET/matches/distrib/ --exclude "*" --include "${rundate}*.json" --exclude "test/*" --recursive -else - echo "trajectory database is size zero... not proceeding with copy" -fi -python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate processed_trajectories.json -dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) -$SRC/website/updateIndexPages.sh $dailyrep -logger -s -t onlyConsolDistrib "finished" From fb3e26adf8e6993b38a1d804e414059b422c3e15 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 19:06:31 +0100 Subject: [PATCH 018/287] missed some templates --- archive/deployment/pylib.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml index f119ed1e..31317339 100644 --- a/archive/deployment/pylib.yml +++ b/archive/deployment/pylib.yml @@ -97,6 +97,7 @@ - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mm.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } + - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetailstest-ukmda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/consolidateDistTraj.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/createDistribMatchingSh.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } @@ -107,4 +108,5 @@ - {src: '{{srcdir}}/ukmon_pylib/traj/plotOSMGroundTrack.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/ShowerAssociation.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } + - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner_test.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/traj/README.md', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } From ecf849169a3646ccd9bdbba3a0a3797cf65badc9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 20:31:53 +0100 Subject: [PATCH 019/287] update dev/test env settings --- archive/terraform/mjmm/dev_ssm_parameters.tf | 4 ++-- archive/terraform/mjmm/variables.tf | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/archive/terraform/mjmm/dev_ssm_parameters.tf b/archive/terraform/mjmm/dev_ssm_parameters.tf index e622d547..26e99f57 100644 --- a/archive/terraform/mjmm/dev_ssm_parameters.tf +++ b/archive/terraform/mjmm/dev_ssm_parameters.tf @@ -32,7 +32,7 @@ resource "aws_ssm_parameter" "dev_livebucket" { resource "aws_ssm_parameter" "dev_siteurl" { name = "dev_siteurl" type = "String" - value = "http://mjmm-ukmonarchive.co.uk.s3-website.eu-west-2.amazonaws.com" + value = "https://www.ukmeteors.co.uk/dummy/" tags = { "billingtag" = "ukmon" } @@ -50,7 +50,7 @@ resource "aws_ssm_parameter" "dev_envname" { resource "aws_ssm_parameter" "dev_calcinstance" { name = "dev_calcinstance" type = "String" - value = "i-08cd1d5f6e1056f6b" #"i-0da38ed8aea1a1d85" + value = "i-0ab47af23705beb31" tags = { "billingtag" = "ukmon" } diff --git a/archive/terraform/mjmm/variables.tf b/archive/terraform/mjmm/variables.tf index f1d68fb9..bad991dc 100644 --- a/archive/terraform/mjmm/variables.tf +++ b/archive/terraform/mjmm/variables.tf @@ -11,8 +11,8 @@ variable "webbucket" {default = "ukmda-website"} variable "sharedbucket" {default = "ukmda-shared"} variable "livebucket" {default = "ukmda-live"} -variable "dev_sharedbucket" { default = "mjmm-ukmon-shared" } -variable "dev_webbucket" { default = "mjmm-ukmonarchive.co.uk" } -variable "dev_livebucket" { default = "mjmm-ukmon-live" } +variable "dev_webbucket" { default = "ukmda-website" } +variable "dev_sharedbucket" { default = "ukmda-shared" } +variable "dev_livebucket" { default = "ukmda-live" } variable "vpc_id" { default = "vpc-a19015c8" } From 245164924180a142a5bfb3695b045c39dcbeb718 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 20:32:03 +0100 Subject: [PATCH 020/287] remove disused file --- archive/ukmon_pylib/traj/clusdetails-ee.txt | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 archive/ukmon_pylib/traj/clusdetails-ee.txt diff --git a/archive/ukmon_pylib/traj/clusdetails-ee.txt b/archive/ukmon_pylib/traj/clusdetails-ee.txt deleted file mode 100644 index 1e17ed3f..00000000 --- a/archive/ukmon_pylib/traj/clusdetails-ee.txt +++ /dev/null @@ -1,6 +0,0 @@ -trajsolver -sg-0d37a3b8ee1a3a1c6 -subnet-0c224d5642fb71023 -arn:aws:iam::822069317839:role/ecsTaskExecutionRole -/ecs/trajcont -trajcont From 0186b3b7e1b6739cac6118906b34d07c06f1dcb7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 20:32:40 +0100 Subject: [PATCH 021/287] make sure paths are expanded --- archive/ukmon_pylib/traj/pickleAnalyser.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index ddfe6fbc..c7586d6a 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -122,6 +122,8 @@ def getShowerDets(shwr): def createUFOOrbitFile(traj, outdir, amag, mass, shower_obj): + outdir = os.path.expanduser(outdir) + #print('Creating UFO Orbit style output file') orb = traj.orbit if shower_obj is None: @@ -378,6 +380,7 @@ def computeAbsoluteMagnitudes(traj, meteor_list): def draw3Dmap(traj, outdir): + outdir = os.path.expanduser(outdir) lats = [] lons = [] alts = [] @@ -493,6 +496,8 @@ def calcAdditionalValues(traj): def createAdditionalOutput(traj, outdir): + outdir = os.path.expanduser(outdir) + # calculate the values amag, vmag, mass, id, cod, shwrname, orb, shower_obj, lg, bg, vg, _ = calcAdditionalValues(traj) @@ -591,7 +596,8 @@ def __init__(self): self. min_end_ht = 20 -def getAllImages(dir_path, out_path): +def getAllImages(dir_path, outdir): + outdir = os.path.expanduser(outdir) traj_quality_params = TrajQualityParams() trajs = loadTrajectoryPickles(dir_path, traj_quality_params, verbose=True) imglist = [] @@ -604,6 +610,6 @@ def getAllImages(dir_path, out_path): except Exception: pass outfname = os.path.split(dir_path)[1] - with open(os.path.join(out_path, f'consumed_{outfname}.txt'), 'w') as outf: + with open(os.path.join(outdir, f'consumed_{outfname}.txt'), 'w') as outf: for img in imglist: outf.write(f'{img}\n') From 360ac216c912012438bdbdf10dbb4789ee68eb9d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 22:16:05 +0100 Subject: [PATCH 022/287] fixing various config issues --- archive/server_setup/.bash_aliases | 4 ++-- archive/terraform/mjmm/dev_ssm_parameters.tf | 9 +++++++++ archive/terraform/mjmm/ssm_parameters.tf | 4 ++-- archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt | 4 ++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index f39f446a..00d56993 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -16,8 +16,8 @@ alias stats='if [ "$DATADIR" == "" ] ; then echo select env first; else tail $DA alias matchstatus='if [ "$SRC" == "" ] ; then echo select env first; else grep "Running" $SRC/logs/matchJob.log && grep TRAJ $SRC/logs/matchJob.log | grep SOLVING && echo -n "Completed " && grep Observations: $SRC/logs/matchJob.log | wc -l && grep "nightlyJob" $SRC/logs/nightlyJob.log ; fi ' alias spacecalc='ls -1 | egrep -v "ukmon-shared" | while read i ; do \du -s $i ; done | sort -n' -alias startcalc='~/prod/utils/stopstart-calcengine.sh start' -alias stopcalc='~/prod/utils/stopstart-calcengine.sh stop' +alias startcalc='$SRC/utils/stopstart-calcengine.sh start' +alias stopcalc='$SRC/utils/stopstart-calcengine.sh stop' function dev { source ~/dev/config.ini >/dev/null diff --git a/archive/terraform/mjmm/dev_ssm_parameters.tf b/archive/terraform/mjmm/dev_ssm_parameters.tf index 26e99f57..a9216604 100644 --- a/archive/terraform/mjmm/dev_ssm_parameters.tf +++ b/archive/terraform/mjmm/dev_ssm_parameters.tf @@ -56,6 +56,15 @@ resource "aws_ssm_parameter" "dev_calcinstance" { } } +resource "aws_ssm_parameter" "dev_calcuser" { + name = "dev_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmon" + } +} + resource "aws_ssm_parameter" "dev_backupinstance" { name = "dev_backupinstance" type = "String" diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 4910397c..d8355217 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -50,7 +50,7 @@ resource "aws_ssm_parameter" "prod_envname" { resource "aws_ssm_parameter" "prod_calcinstance" { name = "prod_calcinstance" type = "String" - value = "i-04cd701c3cfc980f5" # "i-0ab47af23705beb31" + value = "i-04cd701c3cfc980f5" tags = { "billingtag" = "ukmon" } @@ -59,7 +59,7 @@ resource "aws_ssm_parameter" "prod_calcinstance" { resource "aws_ssm_parameter" "prod_calcuser" { name = "prod_calcuser" type = "String" - value = "ec2-user" # "ubuntu" + value = "ec2-user" # "unbuntu" tags = { "billingtag" = "ukmon" } diff --git a/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt index ca56c686..6f588b7b 100644 --- a/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt +++ b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt @@ -2,5 +2,5 @@ trajsolvertest sg-06fb7bbee3b4f5e94 subnet-019eef41d5eaf419b arn:aws:iam::183798037734:role/ecsTaskExecutionRole -/ecs/trajcont -trajcont +/ecs/trajconttest +trajconttest From 83b007bec0f9139288ef1eedd08ac86c82f7e4b2 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 22:17:42 +0100 Subject: [PATCH 023/287] updates to makeConfig --- archive/utils/makeConfig.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index 9d44d62a..c87c7c40 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -8,6 +8,8 @@ fi RUNTIME_ENV=$1 envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') +export AWS_PROFILE=default + # read from AWS SSM Parameterset SRC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_srcdir --query Parameters[0].Value | tr -d '"') SITEURL=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_siteurl --query Parameters[0].Value | tr -d '"') @@ -21,6 +23,7 @@ BKPINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_bac SERVERSSHKEY=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_sshkey --query Parameters[0].Value | tr -d '"') NJLOGGRP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_batchloggroup --query Parameters[0].Value | tr -d '"') SERVERUSERID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcuser --query Parameters[0].Value | tr -d '"') +unset AWS_PROFILE # hardcoded PYLIB=$SRC/ukmon_pylib From 5de5268b1954ae4c241b0e26159cc2770293bce7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 22:18:33 +0100 Subject: [PATCH 024/287] updates to distributed processing to align with newest wmpl --- archive/analysis/runDistrib.sh | 132 ++++++-------- .../traj/createDistribMatchingSh.py | 151 ++++++++-------- .../ukmon_pylib/traj/distributeCandidates.py | 164 +++++++++++------- 3 files changed, 236 insertions(+), 211 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 5eb4b5ab..3b1f021a 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -20,6 +20,9 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 + [ "$RUNTIME_ENV" == "DEV" ] && TESTMODE="true" + [ "$RUNTIME_ENV" == "DEV" ] && TESTSUFF="/test" + # logstream name inherited from parent environment but set it if not if [ "$NJLOGSTREAM" == "" ]; then NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) @@ -27,7 +30,7 @@ if [ "$NJLOGSTREAM" == "" ]; then fi log2cw $NJLOGGRP $NJLOGSTREAM "starting runDistrib" runDistrib -# set the profile to the EE account so we can run the server and monitor progress +# set the profile to the UKMDA account so we can run the server and monitor progress export AWS_PROFILE=ukmonshared if [ $# -gt 0 ] ; then @@ -58,12 +61,13 @@ while [ "$stat" -ne 16 ]; do done log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate}" runDistrib + conda activate $HOME/miniconda3/envs/${WMPL_ENV} log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" runDistrib execdist=execdistrib.sh execMatchingsh=/tmp/$execdist -python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh +python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE chmod +x $execMatchingsh log2cw $NJLOGGRP $NJLOGSTREAM "get server details" runDistrib @@ -76,100 +80,78 @@ done log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" runDistrib -scp -i $SERVERSSHKEY $execMatchingsh ec2-user@$privip:data/distrib/$execdist +scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $execMatchingsh ec2-user@$privip:data/distrib/$execdist + scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist done -# push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner.json ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py ec2-user@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py ec2-user@$privip:src/ukmon_pylib/traj +# push the python code and ECS templates required +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj # now run the script log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib -ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execdist" +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist" log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib aws ec2 stop-instances --instance-ids $SERVERINSTANCEID log2cw $NJLOGGRP $NJLOGSTREAM "monitoring and waiting for completion" runDistrib -python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}'); " +python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}', '${TESTMODE}'); " -log2cw $NJLOGGRP $NJLOGSTREAM "merging in the new json files" runDistrib mkdir -p $DATADIR/distrib cd $DATADIR/distrib -# make sure the database isn't corrupt before overwriting it !! -if [ -s $DATADIR/distrib/processed_trajectories.json ] ; then - aws s3 sync $UKMONSHAREDBUCKET/matches/distrib/ $DATADIR/distrib/ --exclude "*" --include "*.json" --quiet - cp -f $DATADIR/distrib/processed_trajectories.json $DATADIR/distrib/prev_processed_trajectories.json - - log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib +stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +if [ $stat -eq 80 ]; then + aws ec2 start-instances --instance-ids $SERVERINSTANCEID +fi +log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib +while [ "$stat" -ne 16 ]; do + sleep 30 if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi - log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib - while [ "$stat" -ne 16 ]; do - sleep 30 - if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID - fi - log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib - stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) - done - - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - while [ $? -ne 0 ] ; do - # in case the server isn't responding to ssh sessions yet - sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $DATADIR/distrib/processed_trajectories.json ec2-user@$privip:data/distrib - done - scp -i $SERVERSSHKEY $DATADIR/distrib/${rundate}*.json ec2-user@$privip:data/distrib - - execcons=execconsol.sh - execConsolsh=/tmp/$execcons - python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', $rundate)" - chmod +x $execConsolsh - - log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib - scp -i $SERVERSSHKEY $execConsolsh ec2-user@$privip:data/distrib/$execcons - ssh -i $SERVERSSHKEY ec2-user@$privip "data/distrib/$execcons" - - log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation" runDistrib - scp -i $SERVERSSHKEY ec2-user@$privip:data/distrib/processed_trajectories.json $DATADIR/distrib - - ssh -i $SERVERSSHKEY ec2-user@$privip "rm -f data/distrib/*.json /tmp/processed_trajectories.json" - # remote temporary files - ssh -i $SERVERSSHKEY ec2-user@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" - # prune trajdb folder on calcserver - ssh -i $SERVERSSHKEY ec2-user@$privip "find ~/ukmon-shared/matches/RMSCorrelate/trajdb -maxdepth 1 -name "*.json*" -mtime +30 -exec rm -f {} \;" - - log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib - aws ec2 stop-instances --instance-ids $SERVERINSTANCEID - - python -c "from traj.consolidateDistTraj import patchTrajDB ; patchTrajDB('$DATADIR/distrib/processed_trajectories.json','/home/ec2-user/ukmon-shared/matches/RMSCorrelate', '/home/ec2-user/data/distrib');" - - # archive older data then push the updated traj db to the S3 bucket - python -m traj.jsonDbMaintenance $DATADIR/distrib/ - aws s3 cp $DATADIR/distrib/processed_trajectories.json $UKMONSHAREDBUCKET/matches/distrib/ --quiet - - log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib - gzip < $DATADIR/distrib/processed_trajectories.json > $DATADIR/trajdb/processed_trajectories.json.${rundate}.gz - aws s3 mv $UKMONSHAREDBUCKET/matches/distrib/${rundate}.pickle $DATADIR/distrib --quiet - tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib/done/ --quiet - rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle - aws s3 rm $UKMONSHAREDBUCKET/matches/distrib/ --exclude "*" --include "${rundate}*.json" --exclude "test/*" --recursive -else - log2cw $NJLOGGRP $NJLOGSTREAM "trajectory database is size zero... not proceeding with copy" runDistrib -fi + log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) +done + +execcons=execconsol.sh +execConsolsh=/tmp/$execcons +python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', '$TESTMODE')" +chmod +x $execConsolsh + +log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib +scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$privip:data/distrib/$execcons +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execcons" + +log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation" runDistrib +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/dbs${TESTSUFF}/*.db $DATADIR/distrib + +# remote temporary files +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" + +log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib +aws ec2 stop-instances --instance-ids $SERVERINSTANCEID + +rm -Rf $DATADIR/latest/dbs/ +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/dbs/ --exclude "*" --include "traj*.db" --quiet + +log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib +tar czvf $DATADIR/distrib/databases_${rundate}.tgz $DATADIR/distrib/*.db +aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATADIR/distrib --quiet +tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle +aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet +rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle +aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "*.db" --exclude "test/*" --exclude "dbs/*" --recursive --quiet + # and then clear the profile again unset AWS_PROFILE log2cw $NJLOGGRP $NJLOGSTREAM "finished runDistrib" runDistrib diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 8ca5aec5..3b8a3bf5 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -6,21 +6,8 @@ import os import sys import datetime -import json - -# read the task template to determine the paths to write any data to -# -def getTrajsolverPaths(): - templdir,_ = os.path.split(__file__) - with open(os.path.join(templdir, 'taskrunner.json'), 'r') as inf: - taskdets = json.load(inf) - taskenv = taskdets['overrides']['containerOverrides'][0]['environment'] - srcpath = [x for x in taskenv if x['name']=='SRCPATH'][0]['value'] # path from which trajsolver will consume candidates - outpath = [x for x in taskenv if x['name']=='OUTPATH'][0]['value'] # path to which trajsolver will publish trajectories - webpath = [x for x in taskenv if x['name']=='WEBPATH'][0]['value'] # web loc to which trajsolver will publish trajectories - - return srcpath, outpath, webpath +from traj.distributeCandidates import getTrajsolverPaths # make sure the local trajectories folder is synced with the master copy @@ -36,7 +23,7 @@ def refreshTrajectories(outf, matchstart, matchend, trajpath): return -# make sure the master copy is updated with any new locally updated trajectories +# make sure the shared bucket is updated with any new locally updated trajectories # def pushUpdatedTrajectoriesShared(outf, matchstart, matchend, targpath): for d in range(matchend, matchstart+1): @@ -46,15 +33,15 @@ def pushUpdatedTrajectoriesShared(outf, matchstart, matchend, targpath): ymd=thisdt.strftime('%Y%m%d') trajloc=f'trajectories/{yr}/{ym}/{ymd}' outf.write(f'if [ -d {trajloc} ] ; then \n') - outf.write(f'aws s3 sync {trajloc} {targpath}/matches/RMSCorrelate/{trajloc} --exclude "*" --include "*.pickle" --include "*report.txt" --quiet\n') - outf.write(f'aws s3 sync {trajloc}/plots {targpath}/matches/RMSCorrelate/{trajloc}/plots --quiet\n') + outf.write(f'aws s3 sync {trajloc} {targpath}/{trajloc} --exclude "*" --include "*.pickle" --include "*report.txt" --quiet\n') + outf.write(f'aws s3 sync {trajloc}/plots {targpath}/{trajloc}/plots --quiet\n') outf.write('fi\n') - outf.write(f'aws s3 sync trajectories/{yr}/plots {targpath}/matches/RMSCorrelate/trajectories/{yr}/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {targpath}/matches/RMSCorrelate/trajectories/{yr}/{ym}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/plots {targpath}/trajectories/{yr}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {targpath}/trajectories/{yr}/{ym}/plots --quiet\n') return -# make sure the master copy is updated with any new locally updated trajectories +# make sure the website is updated with any new locally updated trajectories # def pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath): for d in range(matchend, matchstart+1): @@ -63,16 +50,18 @@ def pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath): ym=thisdt.strftime('%Y%m') ymd=thisdt.strftime('%Y%m%d') trajloc=f'trajectories/{yr}/{ym}/{ymd}' - targloc=f'reports/{yr}/orbits/{ym}/{ymd}' + targloc=f'{yr}/orbits/{ym}/{ymd}' outf.write(f'if [ -d {trajloc} ] ; then \n') outf.write(f'aws s3 sync {trajloc} {webpath}/{targloc} --quiet\n') outf.write('fi\n') outf.write(f'aws s3 sync {trajloc}/plots {webpath}/{targloc}/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/plots {webpath}/reports/{yr}/orbits/plots --quiet\n') - outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {webpath}/reports/{yr}/orbits/{ym}/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/plots {webpath}/{yr}/orbits/plots --quiet\n') + outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {webpath}/{yr}/orbits/{ym}/plots --quiet\n') return +# Create the density plots showing meteor showers +# def createDensityPlots(outf, calcdir, enddt, includeyear=True): yr = enddt.year ym = enddt.strftime('%Y%m') @@ -98,18 +87,24 @@ def createDensityPlots(outf, calcdir, enddt, includeyear=True): return -def SyncRawData(outf, matchstart, matchend, shbucket, calcdir): +# Sync the raw camera data from shared storage to the local disk +# +def SyncRawData(outf, matchstart, matchend, shbucket): # camera data - no need to replicate it for an historical date - outf.write(f'targdirs=$(aws s3 ls {shbucket}/matches/RMSCorrelate/ | egrep -v "traj|daily|test|plot|proce"|grep PRE | awk \'{{print $2}}\')\n') + outf.write(f'targdirs=$(aws s3 ls {shbucket}/ | egrep -v "traj|daily|test|plot|proce|dbs"|grep PRE | awk \'{{print $2}}\')\n') outf.write('for td in $targdirs ; do\n') for d in range(matchend+1, matchstart+2): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) trgdy=thisdt.strftime('%Y%m%d') - outf.write(f' aws s3 sync {shbucket}/matches/RMSCorrelate/$td {calcdir}/$td --exclude "*" --include "${{td:0:6}}_{trgdy}*" --quiet\n') + outf.write(f' aws s3 sync {shbucket}/$td ./$td --exclude "*" --include "${{td:0:6}}_{trgdy}*" --quiet\n') outf.write('done\n') return +# +# Get a list of images that are used by the solutions +# + def gatherUsedImageList(outf, matchstart, matchend, shbucket): for d in range(matchend, matchstart+1): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) @@ -117,18 +112,25 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): mth = thisdt.month dy = thisdt.day trajloc = f'trajectories/{yr}/{yr}{mth:02d}/{yr}{mth:02d}{dy:02d}' - out_dir = '/home/ec2-user/data/distrib' + out_dir = '~/data/distrib' outf.write(f'python -c "from traj.pickleAnalyser import getAllImages;getAllImages(\'{trajloc}\', \'{out_dir}\');"\n') outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt"\n') outf.write(f'rm {out_dir}/consumed_*.txt\n') return -def createExecConsolSh(matchstart, matchend, execconsolsh, rundt): - shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! - _, outpath, _ = getTrajsolverPaths() +# +# Create the bash script that consolidates the generated data and makes sure the website and shared area are updated + +def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): + + istest = True if istest.lower()=='true' else False + print(f'istest is {istest}') + + srcpath, shbucket, webbucket = getTrajsolverPaths(istest=istest) + csuser = os.getenv('SERVERUSERID', default='ec2-user') + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) includeyear = False if enddt.day == 30: @@ -136,16 +138,19 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, rundt): with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - # outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') + outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execConsol start\n') + outf.write(f'aws s3 sync {srcpath}/ ~/data/distrib/ --exclude "*" --include "*.db" --quiet\n') + + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/ {calcdir}/dbs/\n') - outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/ ~/data/distrib/processed_trajectories.json {rundt}\n') + outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') - refreshTrajectories(outf, matchstart, matchend, outpath) + refreshTrajectories(outf, matchstart, matchend, shbucket) outf.write('logger -s -t execConsol creating density plots\n') createDensityPlots(outf, calcdir, enddt, includeyear) outf.write('logger -s -t execConsol pushing data back to S3\n') @@ -153,34 +158,36 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, rundt): pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) outf.write('logger -s -t execConsol getting the image list\n') gatherUsedImageList(outf, matchstart, matchend, shbucket) - #outf.write('unset AWS_PROFILE\n') + outf.write('logger -s -t execConsol done\n') return +# +# Create a bash script to replot the density charts if needed + def createExecReplotSh(matchstart, matchend, execconsolsh): shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! + csuser = os.getenv('SERVERUSERID', default='ec2-user') + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + _, outpath, _ = getTrajsolverPaths() enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - #outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execReplot start\n') refreshTrajectories(outf, matchstart, matchend, outpath) createDensityPlots(outf, calcdir, enddt, False) gatherUsedImageList(outf, matchstart, matchend, shbucket) - #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execReplot done\n') return -def createDistribMatchingSh(matchstart, matchend, execmatchingsh): - shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - webbucket = os.getenv('WEBSITEBUCKET', default='s3://ukmda-website') +def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): + csuser = os.getenv('SERVERUSERID', default='ec2-user') startdt = datetime.datetime.now() + datetime.timedelta(days=-matchstart) enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) @@ -189,66 +196,66 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh): enddtstr = enddt.strftime('%Y%m%d-080000') rundatestr = enddt.strftime('%Y%m%d') - calcdir = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' # hardcoded! + calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' - srcpath, outpath, _ = getTrajsolverPaths() + _, outpath, webpath = getTrajsolverPaths(istest=istest) + srcpath = os.getenv('UKMONSHAREDBUCKET') + '/matches/RMSCorrelate' with open(execmatchingsh, 'w') as outf: outf.write('#!/bin/bash\n') - outf.write('source /home/ec2-user/venvs/wmpl/bin/activate\n') - outf.write('export PYTHONPATH=/home/ec2-user/src/WesternMeteorPyLib:/home/ec2-user/src/ukmon_pylib\n') - #outf.write('export AWS_PROFILE=ukmonshared\n') + outf.write(f'source /home/{csuser}/venvs/wmpl/bin/activate\n') + outf.write(f'export PYTHONPATH=/home/{csuser}/src/WesternMeteorPyLib:/home/{csuser}/src/ukmon_pylib\n') outf.write(f'cd {calcdir}\n') outf.write('df -h . \n') # fetch anything thats new from S3 + outf.write('logger -s -t execdistrib syncing latest trajectories from shared S3\n') refreshTrajectories(outf, matchstart, matchend, outpath) outf.write('logger -s -t execdistrib syncing the raw data from shared S3\n') - outf.write(f'aws s3 cp {srcpath}/processed_trajectories.json {calcdir}/processed_trajectories.json --quiet\n') - outf.write(f'ls -ltr {calcdir}/*.json\n') - - SyncRawData(outf, matchstart, matchend, shbucket, calcdir) + SyncRawData(outf, matchstart, matchend, srcpath) + outf.write(f'aws s3 sync {srcpath}/dbs/ ./dbs/ --quiet\n') outf.write('logger -s -t execdistrib starting correlator to update existing matches and create candidates\n') - outf.write(f'mkdir -p {calcdir}/candidates\n') - outf.write(f'rm {calcdir}/candidates/*.pickle >/dev/null 2>&1\n') - outf.write(f'time python -m wmpl.Trajectory.CorrelateRMS {calcdir} -i 1 -l -r \"({startdtstr},{enddtstr})\"\n') + outf.write('mkdir -p ./candidates/processed\n') + outf.write('rm ./candidates/*.pickle >/dev/null 2>&1\n') + outf.write(f'time python -m wmpl.Trajectory.CorrelateRMS . --dbdir ./dbs --logdir ./logs --mcmode 4 -l -r \"({startdtstr},{enddtstr})\"\n') # backup the raw candidates in case i need to reprocess some by hand - outf.write(f'mkdir -p {calcdir}/candidates/bkp\n') - outf.write(f'tar cvfz {calcdir}/candidates/bkp/{rundatestr}.tgz {calcdir}/candidates/*.pickle\n') - outf.write(f'find {calcdir}/candidates/bkp/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') + outf.write(f'tar cvfz ./candidates/processed/{rundatestr}.tgz ./candidates/*.pickle\n') + outf.write('find ./candidates/processed/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') + outf.write('find ./logs/ -mtime +28 -exec rm -f ' + '{} \\;\n') + # TODO change this to sync the SQLite databases outf.write('logger -s -t execdistrib backing up the database to trajdb\n') - outf.write(f'cp {calcdir}/processed_trajectories.json {calcdir}/trajdb/processed_trajectories.json.{rundatestr}\n') + outf.write(f'tar cvzf ./trajdb/databases_{rundatestr}.tgz dbs/observations.db dbs/trajectories.db dbs/candidates.db\n') + outf.write('find ./trajdb/ -name "*" -mtime +14 -exec rm -f ' + '{} \\;\n') - outf.write('logger -s -t execdistrib Syncing the database back to shared S3\n') - outf.write(f'if [ -s {calcdir}/processed_trajectories.json ] ; then\n') - outf.write(f'aws s3 cp {calcdir}/processed_trajectories.json {srcpath}/processed_trajectories.json --quiet\n') - outf.write('else echo "bad database file" ; fi \n') + outf.write('logger -s -t execdistrib Syncing the databases back to shared S3\n') + outf.write(f'aws s3 sync ./dbs/ {srcpath}/dbs/ --quiet\n') outf.write('logger -s -t execdistrib distributing candidates and launching containers\n') - outf.write(f'time python -m traj.distributeCandidates {rundatestr} {calcdir}/candidates {srcpath}\n') + outf.write(f'time python -m traj.distributeCandidates {rundatestr} ./candidates {istest}\n') # do this again to fetch todays results outf.write('logger -s -t execdistrib refetch latest trajectories\n') refreshTrajectories(outf, matchstart, matchend, outpath) outf.write('logger -s -t execdistrib and sync the back to S3 as well\n') - pushUpdatedTrajectoriesShared(outf, matchstart, matchend, shbucket) - pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webbucket) + pushUpdatedTrajectoriesShared(outf, matchstart, matchend, outpath) + pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath) - #outf.write('unset AWS_PROFILE\n') outf.write('logger -s -t execdistrib done\n') if __name__ == '__main__': if len(sys.argv) < 4: - print('Usage: createDistribMatchingSh day1 day2 outfile') + print('Usage: createDistribMatchingSh day1 day2 outfile optional_istest') exit(1) matchstart = int(sys.argv[1]) matchend = int(sys.argv[2]) outfname = sys.argv[3] + if len(sys.argv) > 4: + istest = True if sys.argv[4].lower()=='true' else False - createDistribMatchingSh(matchstart, matchend, outfname) + createDistribMatchingSh(matchstart, matchend, outfname, istest) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 861b09bd..58c942b6 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -18,13 +18,30 @@ import pickle -def getClusterDetails(templdir): - sts = boto3.client('sts') - accid = sts.get_caller_identity()['Account'] - if accid == '183798037734': - clusdetails = os.path.join(templdir, 'clusdetails-mda.txt') +# read the task template to determine the paths to write any data to +# +def getTrajsolverPaths(istest=False): + templdir,_ = os.path.split(__file__) + if istest: + taskjson = 'taskrunner_test.json' + else: + taskjson = 'taskrunner.json' + with open(os.path.join(templdir, taskjson), 'r') as inf: + taskdets = json.load(inf) + taskenv = taskdets['overrides']['containerOverrides'][0]['environment'] + srcpath = [x for x in taskenv if x['name']=='SRCPATH'][0]['value'] # path from which trajsolver will consume candidates + outpath = [x for x in taskenv if x['name']=='OUTPATH'][0]['value'] # path to which trajsolver will publish trajectories + webpath = [x for x in taskenv if x['name']=='WEBPATH'][0]['value'] # web loc to which trajsolver will publish trajectories + + return srcpath, outpath, webpath + + +def getClusterDetails(istest=False): + templdir,_ = os.path.split(__file__) + if istest: + clusdetails = os.path.join(templdir, 'clusdetailstest-ukmda.txt') else: - clusdetails = os.path.join(templdir, 'clusdetails-mm.txt') + clusdetails = os.path.join(templdir, 'clusdetails-mda.txt') with open(clusdetails) as inf: lis = inf.readlines() clusname = lis[0].strip() @@ -36,12 +53,16 @@ def getClusterDetails(templdir): return [clusname, secgrp, subnet, exrolearn, loggrp, contname] -def createTaskTemplate(rundate, buckname, clusdets, spandays=3): +def createTaskTemplate(rundate, buckname, clusdets, spandays=3, istest=False): d1 = (rundate + datetime.timedelta(days = -(spandays-1))).strftime('%Y%m%d') d2 = (rundate + datetime.timedelta(days = 1)).strftime('%Y%m%d') templdir,_ = os.path.split(__file__) - templatefile = os.path.join(templdir, 'taskrunner.json') + if istest: + taskjson = 'taskrunner_test.json' + else: + taskjson = 'taskrunner.json' + templatefile = os.path.join(templdir, taskjson) clusname = clusdets[0] secgrp = clusdets[1] subnet = clusdets[2] @@ -67,16 +88,25 @@ def getDebugStatus(): return False -def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): +def distributeCandidates(rundate, srcdir, maxcount=20, istest=False): + + clusdets = getClusterDetails(istest=istest) + _, targdir, _ = getTrajsolverPaths(istest=istest) clusname = clusdets[0] + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] + ecsclient = boto3.client('ecs', region_name='eu-west-2') status = ecsclient.describe_clusters(clusters=[clusname]) if len(status['clusters']) == 0: print('cluster not running!') return False + print(clusdets, targdir) + print(f'Reading from {srcdir}') # obtain a list of picklefiles and sort by name flist = glob.glob1(srcdir, '*.pickle') @@ -87,12 +117,10 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): # work out how many buckets i need numcands = len(flist) + print(f'processing {numcands} candidates') numbucks = int(math.ceil(numcands/maxcount)) # create buckets - targdir = targdir[5:] - outbucket=targdir[:targdir.find('/')] - targdir = targdir[targdir.find('/')+1:] buckroot = os.path.join(targdir, rundate.strftime('%Y%m%d')) taskarns = [None] * numbucks @@ -110,7 +138,7 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): dst = os.path.join(bucknames[i], fli) s3.upload_file(src, outbucket, dst) - taskjson = createTaskTemplate(rundate, bucknames[i], clusdets) + taskjson = createTaskTemplate(rundate, bucknames[i], clusdets, istest=istest) response = ecsclient.run_task(**taskjson) while len(response['tasks']) == 0: @@ -119,7 +147,7 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): taskarn = response['tasks'][0]['taskArn'] taskarns[i] = taskarn jsontempls[i] = taskjson - print(taskarn[51:]) + print(taskarn.split('/')[-1]) if isDbg is True and i == 3: break @@ -145,21 +173,29 @@ def distributeCandidates(rundate, srcdir, targdir, clusdets, maxcount=20): return True -def monitorProgress(rundate): +def monitorProgress(rundatestr, istest=''): + + istest = False if istest=='' else True client = boto3.client('ecs', region_name='eu-west-2') s3 = boto3.client('s3') - archbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - templdir,_ = os.path.split(__file__) - clusdets = getClusterDetails(templdir) + clusdets = getClusterDetails(istest=istest) + _, targdir, _ = getTrajsolverPaths(istest=istest) + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] # load the buckets, tasks and cluster name from the dump file - rundate = datetime.datetime.strptime(rundate, '%Y%m%d') - picklefile = os.path.join(datadir, 'distrib', rundate.strftime('%Y%m%d') + '.pickle') - rempickle = f"matches/distrib/{rundate.strftime('%Y%m%d')}.pickle" + rundate = datetime.datetime.strptime(rundatestr, '%Y%m%d') + if istest: + picklefile = os.path.join(datadir, 'distrib', 'test', rundatestr + '.pickle') + else: + picklefile = os.path.join(datadir, 'distrib', rundatestr + '.pickle') + rempickle = f"{targdir}/{rundatestr}.pickle" + os.makedirs(os.path.join(datadir, 'distrib', 'test'), exist_ok=True) try: - s3.download_file(archbucket, rempickle, picklefile) + s3.download_file(outbucket, rempickle, picklefile) except: print('no containers to monitor') return @@ -193,22 +229,24 @@ def monitorProgress(rundate): taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {tsk["arn"][51:]} completed already') + taskid = tsk["taskArn"].split('/')[-1] + print(f'task {taskid} completed already') for tsk in sts['tasks']: - print(f'checking {tsk["taskArn"][51:]}') + taskid = tsk["taskArn"].split('/')[-1] + print(f'checking {taskid}') idx = taskarns.index(tsk['taskArn']) #print(tsk['taskArn'], tsk['lastStatus']) if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': # retry the task - thisjson = createTaskTemplate(rundate, bucknames[idx]) + thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] print('task restarted') @@ -218,11 +256,11 @@ def monitorProgress(rundate): taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') print('task completed') @@ -231,9 +269,9 @@ def monitorProgress(rundate): realfname=None logdir = os.path.join(datadir, '..', 'logs', 'distrib') os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') + tmpfname = os.path.join(logdir, f'{thisarn.split("/")[-1]}.log') with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): + for events in getLogDetails(loggrp, thisarn.split("/")[-1], contname): for evt in events: evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) msg = evt['message'] @@ -243,8 +281,8 @@ def monitorProgress(rundate): locname = os.path.join(logdir, f'{realfname}.log') os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + remlog = f'{targdir}/logs/{realfname}.log' + s3.upload_file(Filename=locname, Bucket=outbucket, Key=remlog) if len(taskarns) > 99: sts = client.describe_tasks(cluster=clusname, tasks=taskarns[99:199]) for tsk in sts['failures']: @@ -255,21 +293,22 @@ def monitorProgress(rundate): taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {tsk["arn"][51:]} completed already') + print(f'task {thisarn.split("/")[-1]} completed already') for tsk in sts['tasks']: - print(f'checking {tsk["taskArn"][51:]}') + taskid = tsk["taskArn"].split('/')[-1] + print(f'checking {taskid}') idx = taskarns.index(tsk['taskArn']) if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': # retry the task - thisjson = createTaskTemplate(rundate, bucknames[idx]) + thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] print('task restarted') @@ -279,11 +318,11 @@ def monitorProgress(rundate): taskcount -= 1 _, thisbuck = os.path.split(thisbuck) try: - pref = f'matches/distrib/{thisbuck}/' - objects_to_delete = s3.list_objects(Bucket=archbucket, Prefix=pref) + pref = f'{targdir}/{thisbuck}/' + objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) delete_keys = {'Objects': []} delete_keys['Objects'] = [{'Key': k} for k in [obj['Key'] for obj in objects_to_delete.get('Contents', [])]] - s3.delete_objects(Bucket=archbucket, Delete=delete_keys) + s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') print('task completed') @@ -292,9 +331,9 @@ def monitorProgress(rundate): realfname=None logdir = os.path.join(datadir, '..', 'logs', 'distrib') os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') + tmpfname = os.path.join(logdir, f'{thisarn.split("/")[-1]}.log') with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): + for events in getLogDetails(loggrp, thisarn.split("/")[-1], contname): for evt in events: evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) msg = evt['message'] @@ -304,8 +343,8 @@ def monitorProgress(rundate): locname = os.path.join(logdir, f'{realfname}.log') os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + remlog = f'{targdir}/logs/{realfname}.log' + s3.upload_file(Filename=locname, Bucket=outbucket, Key=remlog) if taskcount > 0: # wait 60s before checking again print('sleeping for 60s') @@ -322,9 +361,10 @@ def getMissedLogs(picklefile): contname = 'trajcont' logdir = os.path.expanduser('~/prod/logs/distrib') for thisarn in taskarns: - tmpfname = os.path.join(logdir, f'{thisarn[51:]}.log') + taskid = thisarn.split('/')[-1] + tmpfname = os.path.join(logdir, f'{taskid}.log') with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn[51:], contname): + for events in getLogDetails(loggrp, taskid, contname): for evt in events: evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) msg = evt['message'] @@ -343,6 +383,8 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): """ client = boto3.client('logs', region_name=region_name) + print(contname, thisarn) + # first request response = client.get_log_events( logGroupName=loggrp, @@ -364,21 +406,15 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): if __name__ == '__main__': - if len(sys.argv) < 4: - rundt = datetime.datetime(2022,4,21) - # runs on the calcserver - csuser = os.getenv('SERVERUSERID', default='ec2-user') - srcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate/candidates' # hardcoded on calcserver - - buck = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') - targdir = f'{buck}/matches/distrib' + if len(sys.argv) < 3: + print('usage: python distributeCandidates yyyymmdd ./candidates istest\n') + print(' optional istest if True then test mode used') else: rundt = datetime.datetime.strptime(sys.argv[1], '%Y%m%d') srcdir = sys.argv[2] - targdir = sys.argv[3] - - templdir,_ = os.path.split(__file__) - clusdets = getClusterDetails(templdir) - print(clusdets) - distributeCandidates(rundt, srcdir, targdir, clusdets) + if len(sys.argv)>3: + istest = True if sys.argv[3].lower()=='true' else False + else: + istest = False + distributeCandidates(rundt, srcdir, istest=istest) From 65411e24afa453f90bd761603c90ec4ce356349f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 22:30:55 +0100 Subject: [PATCH 025/287] add a comment to explain what the code does --- archive/analysis/runDistrib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 3b1f021a..2d689d7c 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -141,6 +141,7 @@ ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickl log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib aws ec2 stop-instances --instance-ids $SERVERINSTANCEID +# grab a copy of the indvidual container trajectory dbs so we can get a list of new solutions rm -Rf $DATADIR/latest/dbs/ aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/dbs/ --exclude "*" --include "traj*.db" --quiet From 515692ef98271e3c0423586d7863b7e5b2c69288 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 22:31:23 +0100 Subject: [PATCH 026/287] remove json db --- archive/analysis/findAllMatches.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index db2b35cd..593e0408 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -79,7 +79,7 @@ python -m maintenance.rerunFailedLambdas cd $here log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches -python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate processed_trajectories.json +python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) From 38d09c600017b01eb3716892822c49a121e3f937 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 22:31:45 +0100 Subject: [PATCH 027/287] update consolidation to handle sqlite dbs --- .../ukmon_pylib/traj/consolidateDistTraj.py | 161 ++++++------------ 1 file changed, 49 insertions(+), 112 deletions(-) diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index b213f4b4..64de33c5 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -3,123 +3,60 @@ import os import sys import glob -import shutil import datetime -from wmpl.Trajectory.CorrelateRMS import TrajectoryReduced, DatabaseJSON -#from wmpl.Trajectory.CorrelateRMS import MeteorObsRMS, PlateparDummy, MeteorPointRMS # noqa: F401 - - -# support class so i can add any paired obs objects -class dummyMeteorObsRMS(object): - def __init__(self, station_code, id): - self.station_code = station_code - self.id = id - - -# merge a distributed engine DB back into the master DB -def mergeDatabases(srcdir, srcdb, masterpth, masterfile, mastdb = None): - newdb = os.path.join(srcdir, srcdb) - mergedb = DatabaseJSON(newdb) - if mastdb is None: - mastdb = DatabaseJSON(masterfile) - mastdb.db_file_path = masterfile - # merge successful trajectories - for traj in mergedb.trajectories: - traj_obj = TrajectoryReduced(traj, json_dict = mergedb.trajectories[traj].__dict__) - traj_file_path = traj_obj.traj_file_path - traj_file_path = masterpth + '/' + traj_file_path[traj_file_path.find('trajectories'):] - traj_obj.traj_file_path = traj_file_path - mastdb.addTrajectory(traj_file_path, traj_obj=traj_obj) - # merge failed trajectories - for traj in mergedb.failed_trajectories: - traj_obj = TrajectoryReduced(traj, json_dict = mergedb.failed_trajectories[traj].__dict__) - traj_file_path = traj_obj.traj_file_path - traj_file_path = masterpth + '/' + traj_file_path[traj_file_path.find('trajectories'):] - traj_obj.traj_file_path = traj_file_path - mastdb.addTrajectory(traj_file_path, traj_obj=traj_obj, failed=True) - # merge paired obs data - for p in mergedb.paired_obs: - ids = mergedb.paired_obs[p] - for id in ids: - met_obs = dummyMeteorObsRMS(p, id) - mastdb.addPairedObservation(met_obs) - # save the master DB again - mastdb.save() - return mastdb - - -# utility to patch the database to have the right trajectory folder -def patchTrajDB(dbfile, targpath, oldstr='/home/ec2-user/data/RMSCorrelate'): - - with open(dbfile, 'r') as inf: - with open(os.path.join('/tmp/processed_trajectories.json'), 'w') as outf: - for lin in inf: - outf.write(lin.replace(oldstr, targpath)) - shutil.copyfile('/tmp/processed_trajectories.json', dbfile) - return - - -# utility to count how many of each type of observation was in a database -def countDataInDb(path_to_db): - mergedb = DatabaseJSON(path_to_db) - trajs = len(mergedb.trajectories) - failed = len(mergedb.failed_trajectories) - pairs = len(mergedb.paired_obs) - print(path_to_db, trajs, failed, pairs) - return - - -# utility to dump the detection dates (JDs) to a file -def dumpJDs(path_to_db, fname): - mergedb = DatabaseJSON(path_to_db) - with open(fname,'w') as outf: - outf.write('traj\n') - for t in mergedb.trajectories: - outf.write(f'{t}\n') - outf.write('failed\n') - for t in mergedb.failed_trajectories: - outf.write(f'{t}\n') - - -# utility to compare two databases and dump the JDs of any new events -def dumpNewEntries(db1, db2, fname): - olddb = DatabaseJSON(db1) - newdb = DatabaseJSON(db2) - newtraj = set(newdb.trajectories) - set(olddb.trajectories) - newfail = set(newdb.failed_trajectories) - set(olddb.failed_trajectories) - print(len(newtraj), len(newfail)) - with open(fname,'w') as outf: - outf.write('traj\n') - for t in newtraj: - outf.write(f'{t}\n') - outf.write('failed\n') - for t in newfail: - outf.write(f'{t}\n') +from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase + + +def createLatest(srcdir, dbdir): + mergeDatabases(os.path.expanduser(srcdir), os.path.expanduser(dbdir), ignore_missing=True) + + +def mergeDatabases(srcdir, dbdir, ignore_missing=False): + targdb = os.path.join(dbdir, 'observations.db') + if os.path.isfile(targdb) or ignore_missing: + obsdb = ObservationsDatabase(dbdir) + flist = glob.glob(os.path.join(srcdir, 'observations_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if obsdb.mergeObsDatabase(fl): + os.remove(fl) + obsdb.closeObsDatabase() + + targdb = os.path.join(dbdir, 'trajectories.db') + if os.path.isfile(targdb) or ignore_missing: + trajdb = TrajectoryDatabase(dbdir) + flist = glob.glob(os.path.join(srcdir, 'trajectories_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if trajdb.mergeTrajDatabase(fl): + os.remove(fl) + trajdb.closeTrajDatabase() + + # should never need to run this part as the candidates are all made in one place + targdb = os.path.join(dbdir, 'candidates.db') + if os.path.isfile(targdb): + canddb = CandidateDatabase(dbdir) + flist = glob.glob(os.path.join(srcdir, 'candidates_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if canddb.mergeCandDatabase(fl): + os.remove(fl) + canddb.closeCandDatabase() + + print('done') if __name__ == '__main__': if len(sys.argv) < 3: - print('usage: consolidateDistTraj folder_containing_srcdbs full_path_to_targb') + print('usage: consolidateDistTraj folder_containing_srcdbs targ_dbdir') exit(0) srcdir = sys.argv[1] - masterdb = sys.argv[2] - if len(sys.argv) > 3: - rundt = sys.argv[3] - else: - rundt = datetime.datetime.now().strftime('%Y%m%d') - # real path to the trajectories as per the master database - masterpth = '/home/ec2-user/ukmon-shared/matches/RMSCorrelate' - - flist = glob.glob1(srcdir, f'{rundt}*.json') - flist.sort() - mastdb = None - for fl in flist: - #countDataInDb(os.path.join(srcdir, fl)) - tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') - print(f'{tstamp} processing {fl}') - sys.stdout.flush() - mastdb = mergeDatabases(srcdir, fl, masterpth, masterdb, mastdb) - - print('patching path in mastdb') - patchTrajDB(masterdb, masterpth, oldstr='/home/ec2-user/prod/data/distrib') + dbdir = sys.argv[2] + mergeDatabases(srcdir, dbdir) From 3310c1ba190dd9680c419885d4a0686f58793d7d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 22:50:01 +0100 Subject: [PATCH 028/287] remove unused function --- archive/ukmon_pylib/traj/consolidateDistTraj.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index 64de33c5..46c3d89c 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -8,10 +8,6 @@ from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase -def createLatest(srcdir, dbdir): - mergeDatabases(os.path.expanduser(srcdir), os.path.expanduser(dbdir), ignore_missing=True) - - def mergeDatabases(srcdir, dbdir, ignore_missing=False): targdb = os.path.join(dbdir, 'observations.db') if os.path.isfile(targdb) or ignore_missing: From f549695706909de9b1edf78aa1b0bab5ef5903ee Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Apr 2026 23:50:08 +0100 Subject: [PATCH 029/287] create data for daily report --- .../reports/reportOfLatestMatches.py | 42 ++++++++++--------- .../ukmon_pylib/traj/consolidateDistTraj.py | 6 +-- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index f7eeed01..b7010edf 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -8,13 +8,16 @@ import datetime import numpy import csv -import json import shutil import tempfile import boto3 from traj.pickleAnalyser import getVMagCodeAndStations from reports.CameraDetails import findSite, loadLocationDetails +from traj.consolidateDistTraj import mergeDatabases + +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase +from wmpl.Utils.TrajConversions import datetime2JD def processLocalFolder(trajdir, basedir): @@ -43,26 +46,30 @@ def getTrajPaths(trajdict): trajpaths=[] fullnames=[] for traj in trajdict: - fullnames.append(trajdict[traj]['traj_file_path']) - pth, _ = os.path.split(trajdict[traj]['traj_file_path']) + fullnames.append(traj['traj_file_path']) + pth, _ = os.path.split(traj['traj_file_path']) trajpaths.append(pth) return trajpaths, fullnames -def getListOfNewMatches(dir_path, tfile, prevtfile): - with open(os.path.join(dir_path, tfile), 'r') as inf: - trajs = json.load(inf) - with open(os.path.join(dir_path, prevtfile), 'r') as inf: - ptrajs = json.load(inf) - newtrajs = {k:v for k,v in trajs['trajectories'].items() if k not in ptrajs['trajectories']} - #print(len(newtrajs)) - _, newdirs = getTrajPaths(newtrajs) +def getListOfNewMatches(dir_path): + trajdir = 'matches/RMSCorrelate' + dt_range = [datetime.datetime(2000,1,1,0,0,0).replace(tzinfo=datetime.timezone.utc), + datetime.datetime.now().replace(tzinfo=datetime.timezone.utc)] + jdt_range = [datetime2JD(dt_range[0]), datetime2JD(dt_range[1])] + mergeDatabases(dir_path, '/tmp', ignore_missing=True, purge_records=True) + + tdb = TrajectoryDatabase('/tmp') + newtrajs = tdb.getTrajBasics(trajdir, jdt_range) + tdb.closeTrajDatabase() + + _, newdirs = getTrajPaths(newtrajs) return newdirs -def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): - prevdbname = 'prev_' + dbname - newdirs = getListOfNewMatches(dir_path, dbname, prevdbname) +def findNewMatches(dir_path, out_path, offset, repdtstr): + + newdirs = getListOfNewMatches(dir_path) # load camera details caminfo = loadLocationDetails() caminfo = caminfo[caminfo.active==1] @@ -134,13 +141,8 @@ def findNewMatches(dir_path, out_path, offset, repdtstr, dbname): if __name__ == '__main__': repdtstr = None - dbname = None if len(sys.argv) > 4: repdtstr = sys.argv[4] - if len(sys.argv) > 5: - dbname = sys.argv[5] - else: - dbname = 'processed_trajectories.json.bigserver' # arguments dblocation, datadir, days ago, rundate eg 20220524, full path to database - findNewMatches(sys.argv[1], sys.argv[2], sys.argv[3], repdtstr, dbname) + findNewMatches(sys.argv[1], sys.argv[2], sys.argv[3], repdtstr) diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index 46c3d89c..848feeef 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -8,10 +8,10 @@ from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase -def mergeDatabases(srcdir, dbdir, ignore_missing=False): +def mergeDatabases(srcdir, dbdir, ignore_missing=False, purge_records=False): targdb = os.path.join(dbdir, 'observations.db') if os.path.isfile(targdb) or ignore_missing: - obsdb = ObservationsDatabase(dbdir) + obsdb = ObservationsDatabase(dbdir, purge_records=purge_records) flist = glob.glob(os.path.join(srcdir, 'observations_*.db')) flist.sort() for fl in flist: @@ -23,7 +23,7 @@ def mergeDatabases(srcdir, dbdir, ignore_missing=False): targdb = os.path.join(dbdir, 'trajectories.db') if os.path.isfile(targdb) or ignore_missing: - trajdb = TrajectoryDatabase(dbdir) + trajdb = TrajectoryDatabase(dbdir, purge_records=purge_records) flist = glob.glob(os.path.join(srcdir, 'trajectories_*.db')) flist.sort() for fl in flist: From 6693f81a2c42c4703a03e76fc9758d4e0451480f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 8 Apr 2026 13:37:09 +0100 Subject: [PATCH 030/287] remove unused file --- archive/ukmon_pylib/traj/jsonDbMaintenance.py | 112 ------------------ 1 file changed, 112 deletions(-) delete mode 100644 archive/ukmon_pylib/traj/jsonDbMaintenance.py diff --git a/archive/ukmon_pylib/traj/jsonDbMaintenance.py b/archive/ukmon_pylib/traj/jsonDbMaintenance.py deleted file mode 100644 index 140d9a51..00000000 --- a/archive/ukmon_pylib/traj/jsonDbMaintenance.py +++ /dev/null @@ -1,112 +0,0 @@ -# small script to archive off old data from the JSON database -import datetime -import os -import sys -from dateutil.relativedelta import relativedelta - -from wmpl.Utils.TrajConversions import datetime2JD, jd2Date -from wmpl.Trajectory.CorrelateRMS import DatabaseJSON - -# Name of json file with the list of processed directories -JSON_DB_NAME = "processed_trajectories.json" - - -def archiveOldRecords(db, db_dir, older_than=3): - """ - Archive off old records to keep the database size down - - Keyword Arguments: - older_than: [int] number of months to keep, default 3 - """ - class DummyMetObs(): - def __init__(self, station, obs_id): - self.station_code = station - self.id = obs_id - - archdate = datetime.datetime.now(datetime.timezone.utc) - relativedelta(months=older_than) - archdate_jd = datetime2JD(archdate) - - arch_db_path = os.path.join(db_dir, 'archive', f'{archdate.strftime("%Y%m")}_{JSON_DB_NAME}') - os.makedirs(os.path.join(db_dir, 'archive'), exist_ok=True) - archdb = DatabaseJSON(arch_db_path) - - for traj in [t for t in db.trajectories if t < archdate_jd]: - if traj < archdate_jd: - archdb.addTrajectory(None, db.trajectories[traj], False) - del db.trajectories[traj] - - for traj in [t for t in db.failed_trajectories if t < archdate_jd]: - if traj < archdate_jd: - archdb.addTrajectory(None, db.failed_trajectories[traj], True) - del db.failed_trajectories[traj] - - for station in db.processed_dirs: - arch_processed = [dirname for dirname in db.processed_dirs[station] if - datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for dirname in arch_processed: - archdb.addProcessedDir(station, dirname) - db.processed_dirs[station].remove(dirname) - - for station in db.paired_obs: - arch_processed = [obs_id for obs_id in db.paired_obs[station] if - datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for obs_id in arch_processed: - archdb.addPairedObservation(DummyMetObs(station, obs_id)) - db.paired_obs[station].remove(obs_id) - - archdb.save() - db.save() - return db - - -def clearDownHistoricArchive(database_path, mthsback=1): - db = DatabaseJSON(database_path) - - if len(list(db.failed_trajectories.keys()))==0 and len(list(db.trajectories.keys()))==0: - print('nothing to do') - return - elif len(list(db.failed_trajectories.keys()))==0: - latest = jd2Date(max(list(db.trajectories.keys())), dt_obj=True) - elif len(list(db.trajectories.keys()))==0: - latest = jd2Date(max(list(db.failed_trajectories.keys())), dt_obj=True) - else: - latest = jd2Date(max(max(list(db.trajectories.keys())), max(list(db.failed_trajectories.keys()))), dt_obj=True) - print('processing', database_path) - archdate = latest - relativedelta(months=mthsback) - archdate = archdate.replace(day=1, hour=12, minute=0, second=0).replace(tzinfo=datetime.timezone.utc) - archdate_jd = datetime2JD(archdate) - - for traj in [t for t in db.trajectories if t < archdate_jd]: - if traj < archdate_jd: - del db.trajectories[traj] - - for traj in [t for t in db.failed_trajectories if t < archdate_jd]: - if traj < archdate_jd: - del db.failed_trajectories[traj] - - for station in db.processed_dirs: - arch_processed = [dirname for dirname in db.processed_dirs[station] if - datetime.datetime.strptime(dirname[14:22], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for dirname in arch_processed: - db.processed_dirs[station].remove(dirname) - - for station in db.paired_obs: - arch_processed = [obs_id for obs_id in db.paired_obs[station] if - datetime.datetime.strptime(obs_id[7:15], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) < archdate] - for obs_id in arch_processed: - db.paired_obs[station].remove(obs_id) - - db.save() - return - - -if __name__ == '__main__': - db_dir = sys.argv[1] - database_path = os.path.join(db_dir, JSON_DB_NAME) - db = DatabaseJSON(database_path) - soonest = jd2Date(min(min(list(db.trajectories.keys())), min(list(db.failed_trajectories.keys()))), dt_obj=True) - nowdt = datetime.datetime.now() - mthsback = int((nowdt - soonest).days/30) - for i in range(mthsback, 2, -1): - print(f'archiving {i} months back') - db = archiveOldRecords(db, db_dir, i) From ae06c61ed04320a03e9a9a96de4ae656417e8480 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 8 Apr 2026 13:38:55 +0100 Subject: [PATCH 031/287] update to call new version of reportOfLatestMatches --- archive/analysis/findAllMatches.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 593e0408..b5aa5092 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -79,7 +79,7 @@ python -m maintenance.rerunFailedLambdas cd $here log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches -python -m reports.reportOfLatestMatches $DATADIR/distrib $DATADIR $MATCHEND $rundate +python -m reports.reportOfLatestMatches $DATADIR/latest/dbs $DATADIR $MATCHEND $rundate log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) From 74064e632227bc9a83600ee5389e1adb731f1496 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 8 Apr 2026 14:21:22 +0100 Subject: [PATCH 032/287] add test variables --- archive/utils/makeConfig.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index c87c7c40..a0aa9d56 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -38,6 +38,15 @@ WMPL_ENV=wmpl APIKEY=$(cat ~/.ssh/gmapsapikey) KMLTEMPLATE=*70km.kml +# variables that can be used in scripts and python files to ensure test data doesnt overwrite live data +if [ "$RUNTIME_ENV" == "DEV" ] ; then + TESTMODE="true" + TESTSUFF="/test" +else + TESTMODE="false" + TESTSUFF="" +fi + # create the config file now=$(date +%Y-%m-%d-%H:%M:%S) CFGFILE=~/${envname}/config.ini @@ -67,6 +76,8 @@ echo "SERVERSSHKEY=${SERVERSSHKEY}" >> ${CFGFILE} echo "APIKEY=${APIKEY}" >> ${CFGFILE} echo "KMLTEMPLATE=${KMLTEMPLATE}" >> ${CFGFILE} echo "NJLOGGRP=${NJLOGGRP}" >> ${CFGFILE} +echo "TESTMODE=${TESTMODE}" >> ${CFGFILE} +echo "TESTSUFF=${TESTSUFF}" >> ${CFGFILE} echo "" >> ${CFGFILE} echo "export RUNTIME_ENV SRC SITEURL" >> ${CFGFILE} echo "export WEBSITEBUCKET UKMONSHAREDBUCKET" >> ${CFGFILE} @@ -76,6 +87,8 @@ echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFI echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID SERVERUSERID NJLOGGRP" >> ${CFGFILE} echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib" >> ${CFGFILE} +echo "export TESTMODE TESTSUFF" >> ${CFGFILE} + # enable miniconda echo "source ~/.condaon" >> ${CFGFILE} # function to log to cloudwatch From c24044d611bf129aa340df987711d1a86fae6346 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 8 Apr 2026 14:21:30 +0100 Subject: [PATCH 033/287] mildly improve doco --- archive/analysis/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/archive/analysis/README.md b/archive/analysis/README.md index a260be3d..a20d70e2 100644 --- a/archive/analysis/README.md +++ b/archive/analysis/README.md @@ -1,17 +1,20 @@ # analysis -This folder contains the batch scripts that perform various analyses of the data. They're mostly triggere +This folder contains the batch scripts that perform various analyses of the data. They're mostly triggered from the nightly batch but can also be triggered manually as needed. +## findAllMatches +The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. + +## runDistrib +Called by findAllMatches to execute the distributed processing engine. This file is designed to be run standalone so you can rerun distributed matching. + ## consolidateOutput Collects all single station and match data and consolidates it into two files in parquet format. These are used for all detailed analysis. ## createSearchable Creates a single file for the search engine, by consolidating the required information from the match and single station data. -## findAllMatches -The heart of the matching engine. Reads in all single station for the date ranged provided (default three days) and runs the distributed matching engine process. - ## getBadStations Checks for stations that failed quality tests such as too many detections, not uploaded for a few days etc. @@ -24,9 +27,6 @@ Creates a UFO-analyser compatible version of the RMS single-station detections. ## reportActiveShowers Creates a shower report for any active showers, by calling showerReport for each active shower year-to-date. -## runDistrib -Called by findAllMatches to execute the distributed processing engine. - ## showerReport Creates a report for one or more showers. @@ -34,4 +34,4 @@ Creates a report for one or more showers. Creates a report of data for one or all stations, for a month or year to date, which is then pushed to the website. ## Copyright -All code Copyright (C) 2018-2023 Mark McIntyre# \ No newline at end of file +All code Copyright (C) 2018- Mark McIntyre# \ No newline at end of file From 0cac5126a27f3c80395c0eea560b3b817e9c6307 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 8 Apr 2026 15:42:00 +0100 Subject: [PATCH 034/287] put test logs in the right CW log group --- archive/terraform/ukmda/ecs_vpc.tf | 1 + archive/terraform/ukmda/ecscluster-test.tf | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/terraform/ukmda/ecs_vpc.tf b/archive/terraform/ukmda/ecs_vpc.tf index 6c724359..03b77e40 100644 --- a/archive/terraform/ukmda/ecs_vpc.tf +++ b/archive/terraform/ukmda/ecs_vpc.tf @@ -7,6 +7,7 @@ data "aws_region" "current" {} # some variables for the cluster and task defns variable "ecsloggroup" { default = "/ecs/trajcont" } +variable "ecsloggrouptest" { default = "/ecs/trajcontest" } variable "containername" { default = "trajcont" } # create a VPC for the cluster diff --git a/archive/terraform/ukmda/ecscluster-test.tf b/archive/terraform/ukmda/ecscluster-test.tf index 9496bed2..badc2926 100644 --- a/archive/terraform/ukmda/ecscluster-test.tf +++ b/archive/terraform/ukmda/ecscluster-test.tf @@ -22,7 +22,7 @@ data "template_file" "tasktest_json_template" { regionid = "eu-west-2" repoid = "calcengine/trajsolvertest" contname = var.containername - loggrp = var.ecsloggroup + loggrp = var.ecsloggrouptest } } @@ -72,7 +72,7 @@ resource "null_resource" "createECSdetailstest" { SECGRP = "${aws_security_group.ecssecgrp.id}" SUBNET = "${aws_subnet.ecs_subnet.id}" IAMROLE = "${aws_iam_role.ecstaskrole.arn}" - LOGGRP = "${var.ecsloggroup}" + LOGGRP = "${var.ecsloggrouptest}" CONTNAME = "${var.containername}" } } From 030013763738e2e3054ea144d79b6b23b975daee Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 8 Apr 2026 15:42:25 +0100 Subject: [PATCH 035/287] handle missing log groups better --- .../ukmon_pylib/traj/distributeCandidates.py | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 58c942b6..8e55fe1f 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -386,24 +386,43 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): print(contname, thisarn) # first request - response = client.get_log_events( - logGroupName=loggrp, - logStreamName=f'ecs/{contname}/{thisarn}', - startFromHead=True) - yield response['events'] - - # second and later - while True: - prev_token = response['nextForwardToken'] + try: response = client.get_log_events( logGroupName=loggrp, logStreamName=f'ecs/{contname}/{thisarn}', - nextToken=prev_token) - # same token then break - if response['nextForwardToken'] == prev_token: - break + startFromHead=True) + yield response['events'] + + # second and later + while True: + prev_token = response['nextForwardToken'] + response = client.get_log_events( + logGroupName=loggrp, + logStreamName=f'ecs/{contname}/{thisarn}', + nextToken=prev_token) + # same token then break + if response['nextForwardToken'] == prev_token: + break + yield response['events'] + except Exception: + response = client.get_log_events( + logGroupName=loggrp, + logStreamName=f'ecs/trajcont/{thisarn}', + startFromHead=True) yield response['events'] + # second and later + while True: + prev_token = response['nextForwardToken'] + response = client.get_log_events( + logGroupName=loggrp, + logStreamName=f'ecs/trajcont/{thisarn}', + nextToken=prev_token) + # same token then break + if response['nextForwardToken'] == prev_token: + break + yield response['events'] + if __name__ == '__main__': if len(sys.argv) < 3: From a65af988612a61ca7877b77360b68591f08a0459 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 8 Apr 2026 15:46:05 +0100 Subject: [PATCH 036/287] extra value in TrajQualityParams --- archive/ukmon_pylib/traj/pickleAnalyser.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index c7586d6a..5afb6944 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -587,13 +587,14 @@ def getListOfImages(picklename): class TrajQualityParams(object): def __init__(self): - self. min_traj_points = 6 - self. min_qc = 5.0 - self. max_e = 1.5 - self. max_radiant_err = 2.0 - self. max_vg_err = 10.0 - self. max_begin_ht = 160 - self. min_end_ht = 20 + self.min_traj_points = 6 + self.min_qc = 5.0 + self.max_e = 1.5 + self.max_radiant_err = 2.0 + self.max_vg_err = 10.0 + self.max_vg = 120.0 + self.max_begin_ht = 160 + self.min_end_ht = 20 def getAllImages(dir_path, outdir): From 62a2d97384bfb372c8f3762a757d0f077f2bd218 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 8 Apr 2026 16:05:58 +0100 Subject: [PATCH 037/287] bugfixes revealed in testing --- archive/analysis/runDistrib.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 2d689d7c..bc9b621a 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -20,9 +20,6 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 - [ "$RUNTIME_ENV" == "DEV" ] && TESTMODE="true" - [ "$RUNTIME_ENV" == "DEV" ] && TESTSUFF="/test" - # logstream name inherited from parent environment but set it if not if [ "$NJLOGSTREAM" == "" ]; then NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) @@ -132,8 +129,8 @@ log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$privip:data/distrib/$execcons ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execcons" -log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation" runDistrib -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/dbs${TESTSUFF}/*.db $DATADIR/distrib +log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation, copying databases" runDistrib +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib # remote temporary files ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" @@ -141,17 +138,22 @@ ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickl log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -# grab a copy of the indvidual container trajectory dbs so we can get a list of new solutions -rm -Rf $DATADIR/latest/dbs/ -aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/dbs/ --exclude "*" --include "traj*.db" --quiet +# grab a copy of the indvidual container dbs so we can get a list of new solutions +rm -Rf $DATADIR/latest/contdbs/ +mkdir -p $DATADIR/latest/contdbs/ +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/contdbs/ --exclude "*" --include "*.db" --quiet +aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "*${rundate}*.db" --exclude "test/*" --exclude "dbs/*" --recursive --quiet +aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATADIR/distrib --quiet log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib + tar czvf $DATADIR/distrib/databases_${rundate}.tgz $DATADIR/distrib/*.db -aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATADIR/distrib --quiet -tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle +mkdir -p $DATADIR/trajdb +mv $DATADIR/distrib/databases_${rundate}.tgz $DATADIR/trajdb +tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet -rm -f $DATADIR/distrib/${rundate}*.json $DATADIR/distrib/${rundate}.pickle -aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "*.db" --exclude "test/*" --exclude "dbs/*" --recursive --quiet +rm -f $DATADIR/distrib/${rundate}.pickle + # and then clear the profile again unset AWS_PROFILE From f9381140a0c9c4f6445c5cd23451006f235ebec8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 8 Apr 2026 16:58:22 +0100 Subject: [PATCH 038/287] fix issues in reportOfLatestMatches --- .../reports/reportOfLatestMatches.py | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index b7010edf..3306ffb9 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -11,13 +11,12 @@ import shutil import tempfile import boto3 +import glob from traj.pickleAnalyser import getVMagCodeAndStations from reports.CameraDetails import findSite, loadLocationDetails -from traj.consolidateDistTraj import mergeDatabases from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase -from wmpl.Utils.TrajConversions import datetime2JD def processLocalFolder(trajdir, basedir): @@ -42,28 +41,32 @@ def processLocalFolder(trajdir, basedir): return outstr -def getTrajPaths(trajdict): - trajpaths=[] - fullnames=[] - for traj in trajdict: - fullnames.append(traj['traj_file_path']) - pth, _ = os.path.split(traj['traj_file_path']) - trajpaths.append(pth) - return trajpaths, fullnames - - def getListOfNewMatches(dir_path): - trajdir = 'matches/RMSCorrelate' - dt_range = [datetime.datetime(2000,1,1,0,0,0).replace(tzinfo=datetime.timezone.utc), - datetime.datetime.now().replace(tzinfo=datetime.timezone.utc)] - jdt_range = [datetime2JD(dt_range[0]), datetime2JD(dt_range[1])] - mergeDatabases(dir_path, '/tmp', ignore_missing=True, purge_records=True) + trajdb = TrajectoryDatabase('/tmp', purge_records=True) + flist = glob.glob(os.path.join(dir_path, 'trajectories_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if trajdb.mergeTrajDatabase(fl): + pass + #os.remove(fl) + else: + print('error') + + if os.getenv('TESTMODE') == 'true': + trajdir = 'matches/distrib/test' + else: + trajdir = 'matches/RMSCorrelate' + + cur = trajdb.dbhandle.execute('select traj_file_path from trajectories where status=1') + newtrajs = cur.fetchall() + trajdb.closeTrajDatabase() - tdb = TrajectoryDatabase('/tmp') - newtrajs = tdb.getTrajBasics(trajdir, jdt_range) - tdb.closeTrajDatabase() + newdirs = [] + for traj in newtrajs: + newdirs.append(f'{trajdir}/{traj[0]}') - _, newdirs = getTrajPaths(newtrajs) return newdirs From c999492e4b07e3fc15154ec397f066c0c9784005 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 9 Apr 2026 14:01:13 +0100 Subject: [PATCH 039/287] put container dbs in a sensible place --- archive/analysis/findAllMatches.sh | 2 +- archive/analysis/runDistrib.sh | 2 +- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index b5aa5092..4a8307d1 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -79,7 +79,7 @@ python -m maintenance.rerunFailedLambdas cd $here log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches -python -m reports.reportOfLatestMatches $DATADIR/latest/dbs $DATADIR $MATCHEND $rundate +python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index bc9b621a..d4a188b0 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -141,7 +141,7 @@ aws ec2 stop-instances --instance-ids $SERVERINSTANCEID # grab a copy of the indvidual container dbs so we can get a list of new solutions rm -Rf $DATADIR/latest/contdbs/ mkdir -p $DATADIR/latest/contdbs/ -aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/contdbs/ --exclude "*" --include "*.db" --quiet +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/contdbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --quiet aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "*${rundate}*.db" --exclude "test/*" --exclude "dbs/*" --recursive --quiet aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATADIR/distrib --quiet diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 3b8a3bf5..f7404d3c 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -143,9 +143,9 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): outf.write(f'cd {calcdir}\n') outf.write('logger -s -t execConsol start\n') - outf.write(f'aws s3 sync {srcpath}/ ~/data/distrib/ --exclude "*" --include "*.db" --quiet\n') + outf.write(f'aws s3 sync {srcpath}/ ~/data/distrib/canddbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --quiet\n') - outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/ {calcdir}/dbs/\n') + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/canddbs/ {calcdir}/dbs/\n') outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') @@ -223,10 +223,10 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): # backup the raw candidates in case i need to reprocess some by hand outf.write(f'tar cvfz ./candidates/processed/{rundatestr}.tgz ./candidates/*.pickle\n') + outf.write(f'aws s3 cp ./candidates/processed/{rundatestr}.tgz {outpath}/candidates/\n') outf.write('find ./candidates/processed/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') outf.write('find ./logs/ -mtime +28 -exec rm -f ' + '{} \\;\n') - # TODO change this to sync the SQLite databases outf.write('logger -s -t execdistrib backing up the database to trajdb\n') outf.write(f'tar cvzf ./trajdb/databases_{rundatestr}.tgz dbs/observations.db dbs/trajectories.db dbs/candidates.db\n') outf.write('find ./trajdb/ -name "*" -mtime +14 -exec rm -f ' + '{} \\;\n') From c71392d78822081ad6b62b654e2d79486f491952 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 9 Apr 2026 14:02:57 +0100 Subject: [PATCH 040/287] catch logging errors --- .../ukmon_pylib/traj/distributeCandidates.py | 63 +++++++------------ 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 8e55fe1f..f4884c45 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -383,45 +383,30 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): """ client = boto3.client('logs', region_name=region_name) - print(contname, thisarn) - - # first request - try: - response = client.get_log_events( - logGroupName=loggrp, - logStreamName=f'ecs/{contname}/{thisarn}', - startFromHead=True) - yield response['events'] - - # second and later - while True: - prev_token = response['nextForwardToken'] - response = client.get_log_events( - logGroupName=loggrp, - logStreamName=f'ecs/{contname}/{thisarn}', - nextToken=prev_token) - # same token then break - if response['nextForwardToken'] == prev_token: - break - yield response['events'] - except Exception: - response = client.get_log_events( - logGroupName=loggrp, - logStreamName=f'ecs/trajcont/{thisarn}', - startFromHead=True) - yield response['events'] - - # second and later - while True: - prev_token = response['nextForwardToken'] - response = client.get_log_events( - logGroupName=loggrp, - logStreamName=f'ecs/trajcont/{thisarn}', - nextToken=prev_token) - # same token then break - if response['nextForwardToken'] == prev_token: - break - yield response['events'] + for thisgrp in [loggrp, '/ecs/trajcont']: + for logstreamname in [f'ecs/{contname}/{thisarn}',f'ecs/trajcont/{thisarn}']: + # first request + print(f'looking for {logstreamname} in {thisgrp}') + try: + response = client.get_log_events( + logGroupName=thisgrp, + logStreamName=logstreamname, + startFromHead=True) + yield response['events'] + + # second and later + while True: + prev_token = response['nextForwardToken'] + response = client.get_log_events( + logGroupName=thisgrp, + logStreamName=logstreamname, + nextToken=prev_token) + # same token then break + if response['nextForwardToken'] == prev_token: + break + yield response['events'] + except Exception: + yield 'no data' if __name__ == '__main__': From 65b72f0aae2674b97bed538bf1c277f5b6888cb2 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 9 Apr 2026 14:03:15 +0100 Subject: [PATCH 041/287] tidy up file locations --- .../reports/reportOfLatestMatches.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index 3306ffb9..bcee2b98 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -49,8 +49,7 @@ def getListOfNewMatches(dir_path): tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') print(f'{tstamp} processing {fl}') if trajdb.mergeTrajDatabase(fl): - pass - #os.remove(fl) + os.remove(fl) else: print('error') @@ -82,15 +81,15 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): else: repdt = datetime.datetime.now() - datetime.timedelta(int(offset)) - os.makedirs(os.path.join(out_path, 'dailyreports'), exist_ok=True) + os.makedirs(out_path, exist_ok=True) # create filename. Allow for three reruns in a day - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_1.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_1.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_2.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_2.txt')) if os.path.isfile(matchlist) is True: - matchlist = os.path.join(out_path, 'dailyreports', repdt.strftime('%Y%m%d_3.txt')) + matchlist = os.path.join(out_path, repdt.strftime('%Y%m%d_3.txt')) s3 = boto3.client('s3') srcbucket=os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] @@ -137,7 +136,7 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): outf.write('\n') # finally, create a "latest.txt" as well - latestlist = os.path.join(out_path, 'dailyreports', 'latest.txt') + latestlist = os.path.join(out_path, 'latest.txt') shutil.copy(matchlist, latestlist) return @@ -146,6 +145,10 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): repdtstr = None if len(sys.argv) > 4: repdtstr = sys.argv[4] + + srcdir = sys.argv[1] + outdir = sys.argv[2] + offset = sys.argv[3] - # arguments dblocation, datadir, days ago, rundate eg 20220524, full path to database - findNewMatches(sys.argv[1], sys.argv[2], sys.argv[3], repdtstr) + # arguments dblocation, datadir, days ago, rundate eg 20220524 + findNewMatches(srcdir, outdir, offset, repdtstr) From f20072b9b043569cdbab7a246d8d86767430724b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 9 Apr 2026 14:19:07 +0100 Subject: [PATCH 042/287] simplify calcserver start up --- archive/analysis/runDistrib.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index d4a188b0..3becc5f5 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -46,11 +46,9 @@ begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" runDistrib -stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) -if [ $stat -eq 80 ]; then - aws ec2 start-instances --instance-ids $SERVERINSTANCEID -fi +aws ec2 start-instances --instance-ids $SERVERINSTANCEID +stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) while [ "$stat" -ne 16 ]; do sleep 5 log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" runDistrib @@ -62,6 +60,7 @@ log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate conda activate $HOME/miniconda3/envs/${WMPL_ENV} log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" runDistrib + execdist=execdistrib.sh execMatchingsh=/tmp/$execdist python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE From 6eb8a9faf632112137a46a97d0c50cedcc706dd6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 10 Apr 2026 14:03:42 +0100 Subject: [PATCH 043/287] sorting out errors in container logging --- archive/terraform/ukmda/ecs_vpc.tf | 5 ----- archive/terraform/ukmda/ecscluster-test.tf | 5 ++++- archive/terraform/ukmda/ecscluster.tf | 4 ++++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/archive/terraform/ukmda/ecs_vpc.tf b/archive/terraform/ukmda/ecs_vpc.tf index 03b77e40..3f88c92a 100644 --- a/archive/terraform/ukmda/ecs_vpc.tf +++ b/archive/terraform/ukmda/ecs_vpc.tf @@ -5,11 +5,6 @@ #data used by the code in several places data "aws_region" "current" {} -# some variables for the cluster and task defns -variable "ecsloggroup" { default = "/ecs/trajcont" } -variable "ecsloggrouptest" { default = "/ecs/trajcontest" } -variable "containername" { default = "trajcont" } - # create a VPC for the cluster resource "aws_vpc" "ecs_vpc" { cidr_block = "172.128.0.0/16" diff --git a/archive/terraform/ukmda/ecscluster-test.tf b/archive/terraform/ukmda/ecscluster-test.tf index badc2926..28a5cdbb 100644 --- a/archive/terraform/ukmda/ecscluster-test.tf +++ b/archive/terraform/ukmda/ecscluster-test.tf @@ -1,6 +1,9 @@ # terraform to create ECS cluster # Copyright (C) 2018-2023 Mark McIntyre +variable "ecsloggrouptest" { default = "/ecs/trajcontest" } +variable "containernametest" { default = "trajconttest" } + # create a cluster resource "aws_ecs_cluster" "trajsolvertest" { name = "trajsolvertest" @@ -21,7 +24,7 @@ data "template_file" "tasktest_json_template" { acctid = data.aws_caller_identity.current.account_id regionid = "eu-west-2" repoid = "calcengine/trajsolvertest" - contname = var.containername + contname = var.containernametest loggrp = var.ecsloggrouptest } } diff --git a/archive/terraform/ukmda/ecscluster.tf b/archive/terraform/ukmda/ecscluster.tf index 974d4c59..2dbb1656 100644 --- a/archive/terraform/ukmda/ecscluster.tf +++ b/archive/terraform/ukmda/ecscluster.tf @@ -1,6 +1,10 @@ # terraform to create ECS cluster # Copyright (C) 2018-2023 Mark McIntyre +# some variables for the cluster and task defns +variable "ecsloggroup" { default = "/ecs/trajcont" } +variable "containername" { default = "trajcont" } + # create a cluster resource "aws_ecs_cluster" "trajsolver" { name = "trajsolver" From c9dcdc257b3ba4e1a54b3f521ccce28d76c770b2 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 10 Apr 2026 14:08:05 +0100 Subject: [PATCH 044/287] update container logging info --- archive/terraform/ukmda/ecscluster-test.tf | 15 +++------------ .../ukmon_pylib/traj/clusdetailstest-ukmda.txt | 2 +- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/archive/terraform/ukmda/ecscluster-test.tf b/archive/terraform/ukmda/ecscluster-test.tf index 28a5cdbb..44c53265 100644 --- a/archive/terraform/ukmda/ecscluster-test.tf +++ b/archive/terraform/ukmda/ecscluster-test.tf @@ -48,15 +48,6 @@ resource "aws_ecs_task_definition" "trajsolvertest_task" { } } -/* -# print out some results - clustername, sec grp, subnet and role arn -output "clusname" { value = aws_ecs_cluster.trajsolver.name } -output "secgrpid" { value = aws_security_group.ecssecgrp.id } -output "subnetid" { value = aws_subnet.ecs_subnet.id } -output "taskrolearn" { value = aws_iam_role.ecstaskrole.arn } -output "loggrp" { value = var.ecsloggroup } -output "contname" { value = var.containername } -*/ # create a local file containing the clustername and a few other details # resource "null_resource" "createECSdetailstest" { @@ -64,8 +55,8 @@ resource "null_resource" "createECSdetailstest" { clusname = join(",", tolist([aws_ecs_cluster.trajsolvertest.name, aws_subnet.ecs_subnet.id, aws_security_group.ecssecgrp.id, - aws_iam_role.ecstaskrole.arn, var.ecsloggroup, - var.containername])) + aws_iam_role.ecstaskrole.arn, var.ecsloggrouptest, + var.containernametest])) } provisioner "local-exec" { command = "echo $env:CLUSNAME $env:SECGRP $env:SUBNET $env:IAMROLE $env:LOGGRP $env:CONTNAME > ../../ukmon_pylib/traj/clusdetailstest-ukmda.txt" @@ -76,7 +67,7 @@ resource "null_resource" "createECSdetailstest" { SUBNET = "${aws_subnet.ecs_subnet.id}" IAMROLE = "${aws_iam_role.ecstaskrole.arn}" LOGGRP = "${var.ecsloggrouptest}" - CONTNAME = "${var.containername}" + CONTNAME = "${var.containernametest}" } } } diff --git a/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt index 6f588b7b..d3be6424 100644 --- a/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt +++ b/archive/ukmon_pylib/traj/clusdetailstest-ukmda.txt @@ -2,5 +2,5 @@ trajsolvertest sg-06fb7bbee3b4f5e94 subnet-019eef41d5eaf419b arn:aws:iam::183798037734:role/ecsTaskExecutionRole -/ecs/trajconttest +/ecs/trajcontest trajconttest From 5383816f9cc17ff82ab2f68c47ecf95543b7da18 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 10 Apr 2026 16:29:58 +0100 Subject: [PATCH 045/287] avoid unnecessary error --- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index f7404d3c..55dfaf30 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -34,8 +34,10 @@ def pushUpdatedTrajectoriesShared(outf, matchstart, matchend, targpath): trajloc=f'trajectories/{yr}/{ym}/{ymd}' outf.write(f'if [ -d {trajloc} ] ; then \n') outf.write(f'aws s3 sync {trajloc} {targpath}/{trajloc} --exclude "*" --include "*.pickle" --include "*report.txt" --quiet\n') + outf.write(f'if [ -d {trajloc}/plots ] ; then \n') outf.write(f'aws s3 sync {trajloc}/plots {targpath}/{trajloc}/plots --quiet\n') outf.write('fi\n') + outf.write('fi\n') outf.write(f'aws s3 sync trajectories/{yr}/plots {targpath}/trajectories/{yr}/plots --quiet\n') outf.write(f'aws s3 sync trajectories/{yr}/{ym}/plots {targpath}/trajectories/{yr}/{ym}/plots --quiet\n') return @@ -223,7 +225,6 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): # backup the raw candidates in case i need to reprocess some by hand outf.write(f'tar cvfz ./candidates/processed/{rundatestr}.tgz ./candidates/*.pickle\n') - outf.write(f'aws s3 cp ./candidates/processed/{rundatestr}.tgz {outpath}/candidates/\n') outf.write('find ./candidates/processed/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') outf.write('find ./logs/ -mtime +28 -exec rm -f ' + '{} \\;\n') @@ -241,7 +242,7 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): outf.write('logger -s -t execdistrib refetch latest trajectories\n') refreshTrajectories(outf, matchstart, matchend, outpath) - outf.write('logger -s -t execdistrib and sync the back to S3 as well\n') + outf.write('logger -s -t execdistrib and sync back to S3 as well\n') pushUpdatedTrajectoriesShared(outf, matchstart, matchend, outpath) pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath) From 8c55c7153d0d1d4adeec77a33b545b16fac3bbb0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 10 Apr 2026 16:30:14 +0100 Subject: [PATCH 046/287] fix bugs in log capture process --- .../ukmon_pylib/traj/distributeCandidates.py | 151 ++++++++---------- 1 file changed, 70 insertions(+), 81 deletions(-) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index f4884c45..69d8c80f 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -228,6 +228,7 @@ def monitorProgress(rundatestr, istest=''): thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) + taskid = tsk["arn"].split('/')[-1] try: pref = f'{targdir}/{thisbuck}/' objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) @@ -236,8 +237,9 @@ def monitorProgress(rundatestr, istest=''): s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - taskid = tsk["taskArn"].split('/')[-1] print(f'task {taskid} completed already') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + for tsk in sts['tasks']: taskid = tsk["taskArn"].split('/')[-1] print(f'checking {taskid}') @@ -266,23 +268,8 @@ def monitorProgress(rundatestr, istest=''): print('task completed') # collect the logs from CloudWatch - realfname=None - logdir = os.path.join(datadir, '..', 'logs', 'distrib') - os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn.split("/")[-1]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn.split("/")[-1], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'{targdir}/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=outbucket, Key=remlog) + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + if len(taskarns) > 99: sts = client.describe_tasks(cluster=clusname, tasks=taskarns[99:199]) for tsk in sts['failures']: @@ -301,6 +288,8 @@ def monitorProgress(rundatestr, istest=''): except: print('folder already removed') print(f'task {thisarn.split("/")[-1]} completed already') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) + for tsk in sts['tasks']: taskid = tsk["taskArn"].split('/')[-1] print(f'checking {taskid}') @@ -326,55 +315,54 @@ def monitorProgress(rundatestr, istest=''): except: print('folder already removed') print('task completed') + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) - # collect the logs from CloudWatch - realfname=None - logdir = os.path.join(datadir, '..', 'logs', 'distrib') - os.makedirs(logdir, exist_ok=True) - tmpfname = os.path.join(logdir, f'{thisarn.split("/")[-1]}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, thisarn.split("/")[-1], contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'{targdir}/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=outbucket, Key=remlog) if taskcount > 0: # wait 60s before checking again - print('sleeping for 60s') - time.sleep(60.0) + print('sleeping for 30s') + time.sleep(30.0) return -def getMissedLogs(picklefile): +def getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir): + # collect the logs from CloudWatch + realfname=None + logdir = os.path.join(datadir, '..', 'logs', 'distrib') + os.makedirs(logdir, exist_ok=True) + tmpfname = os.path.join(logdir, f'{thisarn.split("/")[-1]}.log') + with open(tmpfname, 'w') as outf: + for events in getLogDetails(loggrp, thisarn.split("/")[-1], contname): + for evt in events: + evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) + msg = evt['message'] + outf.write(f'{evtdt} {msg}\n') + if msg[:10] == 'processing': + realfname = msg[11:].strip() + if realfname: + locname = os.path.join(logdir, f'{realfname}.log') + os.rename(tmpfname, locname) + remlog = f'{targdir}/logs/{realfname}.log' + s3.upload_file(Filename=locname, Bucket=outbucket, Key=remlog) + return + + +def getMissedLogs(picklefile, istest=False): + + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) s3 = boto3.client('s3') - archbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] + _, targdir, _ = getTrajsolverPaths(istest=istest) + targdir = targdir[5:] + outbucket=targdir[:targdir.find('/')] + targdir = targdir[targdir.find('/')+1:] + + clusdets = getClusterDetails(istest=istest) + contname = clusdets[5] + dumpdata = pickle.load(open(picklefile,'rb')) taskarns = dumpdata[1] - loggrp = '/ecs/trajcont' - contname = 'trajcont' - logdir = os.path.expanduser('~/prod/logs/distrib') + loggrp = '/ecs/trajcont' for thisarn in taskarns: - taskid = thisarn.split('/')[-1] - tmpfname = os.path.join(logdir, f'{taskid}.log') - with open(tmpfname, 'w') as outf: - for events in getLogDetails(loggrp, taskid, contname): - for evt in events: - evtdt = datetime.datetime.fromtimestamp(int(evt['timestamp']) / 1000) - msg = evt['message'] - outf.write(f'{evtdt} {msg}\n') - if msg[:10] == 'processing': - realfname = msg[11:].strip() - locname = os.path.join(logdir, f'{realfname}.log') - os.rename(tmpfname, locname) - remlog = f'matches/distrib/logs/{realfname}.log' - s3.upload_file(Filename=locname, Bucket=archbucket, Key=remlog) + getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): @@ -383,30 +371,31 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): """ client = boto3.client('logs', region_name=region_name) - for thisgrp in [loggrp, '/ecs/trajcont']: - for logstreamname in [f'ecs/{contname}/{thisarn}',f'ecs/trajcont/{thisarn}']: - # first request - print(f'looking for {logstreamname} in {thisgrp}') - try: - response = client.get_log_events( - logGroupName=thisgrp, - logStreamName=logstreamname, - startFromHead=True) - yield response['events'] - - # second and later - while True: - prev_token = response['nextForwardToken'] - response = client.get_log_events( - logGroupName=thisgrp, - logStreamName=logstreamname, - nextToken=prev_token) - # same token then break - if response['nextForwardToken'] == prev_token: - break - yield response['events'] - except Exception: - yield 'no data' + logstreamname = f'ecs/{contname}/{thisarn}' + # first request + print(f'looking for {logstreamname} in {loggrp}') + try: + response = client.get_log_events( + logGroupName=loggrp, + logStreamName=logstreamname, + startFromHead=True) + yield response['events'] + + # second and later + while True: + prev_token = response['nextForwardToken'] + response = client.get_log_events( + logGroupName=loggrp, + logStreamName=logstreamname, + nextToken=prev_token) + # same token then break + if response['nextForwardToken'] == prev_token: + break + yield response['events'] + except Exception: + dtval = int(datetime.datetime.now().timestamp() * 1000) + msg = f'log for {thisarn} not available' + yield [{'timestamp': dtval, 'message': msg}] if __name__ == '__main__': From 27edcd95109256dae41ced267affd05750e60010 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 11 Apr 2026 12:12:07 +0100 Subject: [PATCH 047/287] add MAX_STNS so i can experiement with limits --- archive/ukmon_pylib/traj/taskrunner.json | 4 ++++ archive/ukmon_pylib/traj/taskrunner_test.json | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/traj/taskrunner.json b/archive/ukmon_pylib/traj/taskrunner.json index 630b6138..be99acf1 100644 --- a/archive/ukmon_pylib/traj/taskrunner.json +++ b/archive/ukmon_pylib/traj/taskrunner.json @@ -29,6 +29,10 @@ { "name": "WEBPATH", "value": "s3://ukmda-website/reports" + }, + { + "name": "MAX_STNS", + "value": "9999" } ] } diff --git a/archive/ukmon_pylib/traj/taskrunner_test.json b/archive/ukmon_pylib/traj/taskrunner_test.json index 3957ccae..fab30ba7 100644 --- a/archive/ukmon_pylib/traj/taskrunner_test.json +++ b/archive/ukmon_pylib/traj/taskrunner_test.json @@ -15,7 +15,7 @@ "overrides": { "containerOverrides": [ { - "name": "trajcont", + "name": "trajconttest", "command": [], "environment": [ { @@ -29,6 +29,10 @@ { "name": "WEBPATH", "value": "s3://ukmda-website/dummy" + }, + { + "name": "MAX_STNS", + "value": "9999" } ] } From 8d850edc3e25758a1bbbee49ffe95da8744ebb4e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 11 Apr 2026 12:14:16 +0100 Subject: [PATCH 048/287] add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper --- archive/analysis/runDistrib.sh | 3 + archive/containers/trajsolver/trajsolver.py | 61 ++++++++++++++++----- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 3becc5f5..bae40bc9 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -94,6 +94,9 @@ rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERI log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist" +scp -i $SERVERSSHKEY $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/cands_${rundate}.log +ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" + log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib aws ec2 stop-instances --instance-ids $SERVERINSTANCEID diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py index 1e3de667..0d66c5e4 100644 --- a/archive/containers/trajsolver/trajsolver.py +++ b/archive/containers/trajsolver/trajsolver.py @@ -6,6 +6,8 @@ import boto3 import datetime import tempfile +import glob +import logging from wmpl.Trajectory.CorrelateRMS import RMSDataHandle from wmpl.Utils.Math import generateDatetimeBins @@ -15,8 +17,10 @@ from wmpl.Trajectory.CorrelateRMS import TrajectoryReduced, DatabaseJSON # noqa: F401 from wmpl.Trajectory.CorrelateRMS import MeteorObsRMS, PlateparDummy, MeteorPointRMS # noqa: F401 +log = logging.getLogger('traj_correlator') # must be the same name as the WMPL logger -def runCorrelator(dir_path, time_beg, time_end): + +def runCorrelator(dir_path, time_beg, time_end, max_stns=9999): # Init trajectory constraints maxtoffset = 10.0 maxstationdist = 600.0 @@ -34,6 +38,7 @@ def runCorrelator(dir_path, time_beg, time_end): trajectory_constraints.run_mc = not disablemc trajectory_constraints.save_plots = saveplots trajectory_constraints.geometric_uncert = not uncerttime + trajectory_constraints.max_stations = max_stns # Clock for measuring script time t1 = datetime.datetime.now(datetime.timezone.utc) @@ -98,9 +103,6 @@ def runCorrelator(dir_path, time_beg, time_end): print("-----------------------------") print() - # Load data of unprocessed observations - #dh.unpaired_observations = dh.loadUnpairedObservations(dh.processing_list, dt_range=(bin_beg, bin_end)) - # Run the trajectory correlator tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, enableOSM=True) tc.run(event_time_range=event_time_range, mcmode=3) @@ -109,6 +111,7 @@ def runCorrelator(dir_path, time_beg, time_end): dh.closeTrajectoryDatabase() print("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) + log.info("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) return @@ -129,7 +132,9 @@ def getSourceAndTargets(): webbucket = webpth[:webpth.find('/')] webpth = webpth[webpth.find('/')+1:] - return srcbucket, srcpth, outbucket, outpth, webbucket, webpth + max_stns = int(os.getenv('MAX_STNS', default='9999')) + + return srcbucket, srcpth, outbucket, outpth, webbucket, webpth, max_stns # extra args for setting the MIME type when uploading to S3. @@ -228,15 +233,37 @@ def getS3Client(): # starting point for the process def startup(srcfldr, startdt, enddt, isTest=False): + + localfldr = tempfile.mkdtemp() + + # Init the logger + log.setLevel(logging.DEBUG) + + # Init the log formatter + log_formatter = logging.Formatter( + fmt='%(asctime)s-%(levelname)-5s-%(module)-15s:%(lineno)-5d- %(message)s', + datefmt='%Y/%m/%d %H:%M:%S') + + # Init the file handler + datasetname = os.path.split(srcfldr)[-1] + log_file = os.path.join(localfldr, f'{datasetname}.log') + file_handler = logging.handlers.TimedRotatingFileHandler(log_file, when="midnight", backupCount=7) + file_handler.setFormatter(log_formatter) + log.addHandler(file_handler) + + # Init the console handler (i.e. print to console) + console_handler = logging.StreamHandler() + console_handler.setFormatter(log_formatter) + log.addHandler(console_handler) + print(f'processing {srcfldr}') print(f"Starting at {datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}") - localfldr = tempfile.mkdtemp() canddir = os.path.join(localfldr,'candidates') os.makedirs(canddir, exist_ok = True) s3 = getS3Client() - srcbucket, srcpth, outbucket, outpth, webbucket, webpth = getSourceAndTargets() + srcbucket, srcpth, outbucket, outpth, webbucket, webpth, max_stns = getSourceAndTargets() if isTest is True: outpth = os.path.join(outpth, 'test') @@ -254,7 +281,12 @@ def startup(srcfldr, startdt, enddt, isTest=False): targfile = os.path.join(canddir, locfname) print(f'downloading {locfname} to {targfile}') s3.meta.client.download_file(srcbucket, fname, targfile) - runCorrelator(localfldr, startdt, enddt) + runCorrelator(localfldr, startdt, enddt, max_stns = max_stns) + + # flush and close the logs + file_handler.flush() + console_handler.flush() + logging.shutdown() print('uploading data to website') trajfldr = os.path.join(localfldr,'trajectories') @@ -273,13 +305,14 @@ def startup(srcfldr, startdt, enddt, isTest=False): targkey = f'{outpth}/{fname}' print(f'uploading {localfile} to {srcbucket}/{targkey}') s3.meta.client.upload_file(localfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) + + lognames = glob.glob(os.path.join(localfldr, '*.log')) + if len(lognames)> 0: + fname = f'correlator_{srcfldr}.log' + targkey = f'{outpth}/{fname}' + print(f'uploading {lognames[0]} to {srcbucket}/{targkey}') + s3.meta.client.upload_file(lognames[0], srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) - fname = f'{srcfldr}.json' - jsonfile = os.path.join(localfldr, 'processed_trajectories.json') - if os.path.isfile(jsonfile): - targkey = f'{srcpth}/{fname}' - print(f'uploading {jsonfile} to {srcbucket}/{srcpth}') - s3.meta.client.upload_file(jsonfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) else: print('no files found') From 42c0407e9655ca09b9dfd7e8002b0fe88e36bde1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 11 Apr 2026 12:16:37 +0100 Subject: [PATCH 049/287] don't pull or push the databases from S3 during distribution but do back them up after consolidation --- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 55dfaf30..5ce87b29 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -149,7 +149,7 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/canddbs/ {calcdir}/dbs/\n') - outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') + outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db"\n') outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') refreshTrajectories(outf, matchstart, matchend, shbucket) @@ -216,7 +216,6 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): outf.write('logger -s -t execdistrib syncing the raw data from shared S3\n') SyncRawData(outf, matchstart, matchend, srcpath) - outf.write(f'aws s3 sync {srcpath}/dbs/ ./dbs/ --quiet\n') outf.write('logger -s -t execdistrib starting correlator to update existing matches and create candidates\n') outf.write('mkdir -p ./candidates/processed\n') @@ -232,9 +231,6 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): outf.write(f'tar cvzf ./trajdb/databases_{rundatestr}.tgz dbs/observations.db dbs/trajectories.db dbs/candidates.db\n') outf.write('find ./trajdb/ -name "*" -mtime +14 -exec rm -f ' + '{} \\;\n') - outf.write('logger -s -t execdistrib Syncing the databases back to shared S3\n') - outf.write(f'aws s3 sync ./dbs/ {srcpath}/dbs/ --quiet\n') - outf.write('logger -s -t execdistrib distributing candidates and launching containers\n') outf.write(f'time python -m traj.distributeCandidates {rundatestr} ./candidates {istest}\n') From 3d1a5c164ce8b2a6ef596aba089b9b922595f423 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Apr 2026 16:11:29 +0100 Subject: [PATCH 050/287] catch error if trajpickle can't be found --- archive/ukmon_pylib/reports/reportOfLatestMatches.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index bcee2b98..b6050a1c 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -53,7 +53,7 @@ def getListOfNewMatches(dir_path): else: print('error') - if os.getenv('TESTMODE') == 'true': + if os.getenv('TESTMODE').lower() == 'true': trajdir = 'matches/distrib/test' else: trajdir = 'matches/RMSCorrelate' @@ -64,7 +64,7 @@ def getListOfNewMatches(dir_path): newdirs = [] for traj in newtrajs: - newdirs.append(f'{trajdir}/{traj[0]}') + newdirs.append(os.path.join(trajdir, traj[0])) return newdirs @@ -99,10 +99,11 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): trajdir = trajdir[trajdir.find('matches'):] trajpath, picklename = os.path.split(trajdir) localpick = os.path.join(tmpdir, picklename) - s3.download_file(srcbucket, trajdir, localpick) try: + s3.download_file(srcbucket, trajdir, localpick) bestvmag, shwr, stationids = getVMagCodeAndStations(localpick) except: + print(f'unable to find {trajdir}') bestvmag, shwr, stationids = 0,'',[''] stations=[] for statid in stationids: From 13d110c473c44f82207b9b9f97bbe3b781f4e2f7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Apr 2026 16:12:53 +0100 Subject: [PATCH 051/287] set rundate more safely move some processes into nightlyJob to make testing easier --- archive/analysis/findAllMatches.sh | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 4a8307d1..093e4c25 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -29,7 +29,8 @@ if [ "$NJLOGSTREAM" == "" ]; then aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared fi log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" findAllMatches -rundate=$(cat $DATADIR/rundate.txt) + +[ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d) # read start/end dates from commandline if rerunning for historical date if [ $# -gt 0 ] ; then @@ -47,15 +48,7 @@ if [ $# -gt 0 ] ; then fi # folder for logs -mkdir -p $SRC/logs > /dev/null 2>&1 - -log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" findAllMatches -# this creates the parquet table for Athena -$SRC/analysis/getRMSSingleData.sh - -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" findAllMatches -yr=$(date +%Y) -$SRC/analysis/createSearchable.sh $yr singles +mkdir -p $SRC/logs/distrib > /dev/null 2>&1 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000') @@ -98,9 +91,6 @@ if [ "$RUNTIME_ENV" == "PROD" ] ; then aws s3 sync $DATADIR/dailyreports/ $UKMONSHAREDBUCKET/matches/RMSCorrelate/dailyreports/ --quiet fi -log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" findAllMatches -$SRC/website/updateIndexPages.sh $dailyrep - log2cw $NJLOGGRP $NJLOGSTREAM "start purgeLogs" findAllMatches find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \; find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \; From 55de1283a60eb485b9b2a78de76e5678c3595d87 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Apr 2026 16:13:27 +0100 Subject: [PATCH 052/287] tidy up backup file names and housekeep backup folder --- archive/analysis/runDistrib.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index bae40bc9..973d3bc0 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -152,8 +152,9 @@ log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib tar czvf $DATADIR/distrib/databases_${rundate}.tgz $DATADIR/distrib/*.db mkdir -p $DATADIR/trajdb mv $DATADIR/distrib/databases_${rundate}.tgz $DATADIR/trajdb -tar czvf $DATADIR/distrib/${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle -aws s3 cp $DATADIR/distrib/${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet +tar czvf $DATADIR/distrib/contdbs_${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle +aws s3 cp $DATADIR/distrib/contdbs_${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet +find $DATADIR/distrib/ -name "contdbs_*.tgz" -mtime +30 -exec rm -f {} \; rm -f $DATADIR/distrib/${rundate}.pickle From a90bee38079473bda7b2de9f957f33e3b1f6fa87 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Apr 2026 16:14:07 +0100 Subject: [PATCH 053/287] move some processes from findAllMatches to nightlyJob to make testing easier --- archive/cronjobs/nightlyJob.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index 7f184a5f..2ff8b759 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -47,9 +47,21 @@ if [ -f $SRC/logs/$matchlog ] ; then suff=$(stat matchJob.log -c %X) mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff fi + +log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" nightlyJob +# this creates the parquet table for Athena +$SRC/analysis/getRMSSingleData.sh + +log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" nightlyJob + +$SRC/analysis/createSearchable.sh $yr singles + # Run the match process - run this only once as it scoops up all unprocessed data ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 +log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" nightlyJob +$SRC/website/updateIndexPages.sh + # from here down, we're creating reports # consolidate the output of the match process for further analysos log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob From 2265a18079bdf9b852fb1247c5288c480e25dd45 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Apr 2026 16:15:17 +0100 Subject: [PATCH 054/287] support for testing of createExecReplot --- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 5ce87b29..102c3161 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -168,12 +168,15 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): # Create a bash script to replot the density charts if needed -def createExecReplotSh(matchstart, matchend, execconsolsh): +def createExecReplotSh(matchstart, matchend, execconsolsh, istest=''): + + istest = True if istest.lower()=='true' else False + shbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared') csuser = os.getenv('SERVERUSERID', default='ec2-user') calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' + _, outpath, _ = getTrajsolverPaths(istest=istest) - _, outpath, _ = getTrajsolverPaths() enddt = datetime.datetime.now() + datetime.timedelta(days=-matchend) with open(execconsolsh, 'w') as outf: outf.write('#!/bin/bash\n') @@ -201,6 +204,7 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): calcdir = f'/home/{csuser}/ukmon-shared/matches/RMSCorrelate' _, outpath, webpath = getTrajsolverPaths(istest=istest) + srcpath = os.getenv('UKMONSHAREDBUCKET') + '/matches/RMSCorrelate' with open(execmatchingsh, 'w') as outf: @@ -212,6 +216,7 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): # fetch anything thats new from S3 outf.write('logger -s -t execdistrib syncing latest trajectories from shared S3\n') + # in test, we seeded the test area with prod trajectories in April 2026 refreshTrajectories(outf, matchstart, matchend, outpath) outf.write('logger -s -t execdistrib syncing the raw data from shared S3\n') @@ -237,7 +242,7 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): # do this again to fetch todays results outf.write('logger -s -t execdistrib refetch latest trajectories\n') refreshTrajectories(outf, matchstart, matchend, outpath) - + outf.write('logger -s -t execdistrib and sync back to S3 as well\n') pushUpdatedTrajectoriesShared(outf, matchstart, matchend, outpath) pushUpdatedTrajectoriesWeb(outf, matchstart, matchend, webpath) From 4535a89186b0f61812a8f871592214c062b93beb Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Apr 2026 16:15:47 +0100 Subject: [PATCH 055/287] pass testmode to createExecReplot --- archive/analysis/updatePlotsAndDetStatus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index 479acf4b..beed3337 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -59,7 +59,7 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" updatePlotsAndDetStatus execrerun=execreplot.sh execrerunsh=/tmp/$execrerun -python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh')" +python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh', '$TESTMODE')" chmod +x $execrerunsh log2cw $NJLOGGRP $NJLOGSTREAM "get server details" updatePlotsAndDetStatus From 38fba7727c5864628fa389b1a028033801795445 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Apr 2026 16:59:54 +0100 Subject: [PATCH 056/287] ensure logs are unique and capture container logs --- archive/analysis/runDistrib.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 973d3bc0..2ceee370 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -84,17 +84,17 @@ while [ $? -ne 0 ] ; do scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist done # push the python code and ECS templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj # now run the script log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist" -scp -i $SERVERSSHKEY $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/cands_${rundate}.log +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib @@ -147,14 +147,20 @@ aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/contd aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "*${rundate}*.db" --exclude "test/*" --exclude "dbs/*" --recursive --quiet aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATADIR/distrib --quiet +# grab a copy of the indvidual container logs - duplicated in monitorProgress, but never mind +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $SRC/logs/distrib/ --exclude "*" --include "correl*.log" --exclude "logs/" --quiet +aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "correl*.log" --exclude "logs/" --recursive --quiet + log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib -tar czvf $DATADIR/distrib/databases_${rundate}.tgz $DATADIR/distrib/*.db mkdir -p $DATADIR/trajdb -mv $DATADIR/distrib/databases_${rundate}.tgz $DATADIR/trajdb +tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db tar czvf $DATADIR/distrib/contdbs_${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle aws s3 cp $DATADIR/distrib/contdbs_${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet -find $DATADIR/distrib/ -name "contdbs_*.tgz" -mtime +30 -exec rm -f {} \; + +tar czvf $DATADIR/distrib/contlogs_${rundate}.tgz $SRC/logs/distrib/correl*.log $SRC/logs/distrib/${rundate}_*.log + +find $DATADIR/distrib/ -name "cont_*.tgz" -mtime +30 -exec rm -f {} \; rm -f $DATADIR/distrib/${rundate}.pickle From daf8255f97244075a1a7b47ecfc149746b2f27d3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Apr 2026 21:33:50 +0100 Subject: [PATCH 057/287] realigning createFireballPage with prod. Updating gitignore --- .gitignore | 4 ++-- archive/website/createFireballPage.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 0ea36ffb..06dacff5 100644 --- a/.gitignore +++ b/.gitignore @@ -609,11 +609,11 @@ usermgmt/server/ukmon*.json **/awskeys.test **/trajsolver_old/* archive/tmp/* -servercopy/files/* +servercopy/* tests/testing/RMS/ tests/testing/WesternMeteorPyLib/ archive/ukmon_pylib/tests/testdata.tar.gz -servercopybkp/* +servercopydev/* fbcollector/config.ini usermgmt/windows/stationmaint.ini usermgmt/windows/README.md diff --git a/archive/website/createFireballPage.sh b/archive/website/createFireballPage.sh index 7c63ba4b..396aeb65 100644 --- a/archive/website/createFireballPage.sh +++ b/archive/website/createFireballPage.sh @@ -88,7 +88,7 @@ echo "\$(document).ready(function() {" >> reportindex.js # need single quotes here to allow the hash to be printed echo '$("#fbtableid").DataTable({' >> reportindex.js echo "columnDefs : [" >> reportindex.js -echo "{ Type : \"numeric\", targets : [2]}" >> reportindex.js +echo "{ Type : \"numeric\", targets : [1]}" >> reportindex.js echo " ]," >> reportindex.js echo "order : [[ 1, \"asc\"],[2,\"desc\"]]," >> reportindex.js echo "paging: false" >> reportindex.js From 982b067c44522f58d6d13455ec4cca9b0f186f0e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Apr 2026 12:59:31 +0100 Subject: [PATCH 058/287] remove verbosity flag from one tar operation --- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 102c3161..158f6dda 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -228,7 +228,7 @@ def createDistribMatchingSh(matchstart, matchend, execmatchingsh, istest=False): outf.write(f'time python -m wmpl.Trajectory.CorrelateRMS . --dbdir ./dbs --logdir ./logs --mcmode 4 -l -r \"({startdtstr},{enddtstr})\"\n') # backup the raw candidates in case i need to reprocess some by hand - outf.write(f'tar cvfz ./candidates/processed/{rundatestr}.tgz ./candidates/*.pickle\n') + outf.write(f'tar czf ./candidates/processed/{rundatestr}.tgz ./candidates/*.pickle\n') outf.write('find ./candidates/processed/ -name "*.tgz" -mtime +14 -exec rm -f ' + '{} \\;\n') outf.write('find ./logs/ -mtime +28 -exec rm -f ' + '{} \\;\n') From a62ec243160c86c0ed10e8a89277b433486f5d00 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Apr 2026 13:21:22 +0100 Subject: [PATCH 059/287] fixup to handle new style log with datestamps --- archive/ukmon_pylib/metrics/getMatchStats.py | 37 ++++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index 8114c4e3..3ffb6c04 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -34,27 +34,42 @@ def getDailyObsCounts(logf): def getMatchStats(logf): - with open(logf) as inf: - loglines = inf.readlines() + loglines = open(logf).readlines() + + events = [line.strip() for line in loglines if 'Analysing' in line and 'observations' in line] + if len(events) > 0: + # new-style log + addlines = [x for x in events if '0 observations' not in x] + addoffs = -5 + oldstyle = False + else: + addlines = [line.strip() for line in loglines if 'Added' in line and 'observations' in line] + addoffs = 1 + oldstyle = True - addlines = [line.strip() for line in loglines if 'Added' in line and 'observations' in line] - nocallines = [line.strip() for line in loglines if 'Skipping' in line and 'recalibrated' in line] - misdflines = [line.strip() for line in loglines if 'Skipping' in line and 'missing data' in line] - uncal = len(nocallines) - missdf = len(misdflines) + uncal = len([line.strip() for line in loglines if 'Skipping' in line and 'recalibrated' in line]) + missdf = len([line.strip() for line in loglines if 'Skipping' in line and 'missing data' in line]) + added=0 for li in addlines: spls=li.split(' ') - added = added + int(spls[1]) + added += int(spls[addoffs]) beglowr = len([line.strip() for line in loglines if 'Begin height lower than the end height' in line]) badalti = len([line.strip() for line in loglines if 'Meteor heights outside allowed range' in line]) badvelo = len([line.strip() for line in loglines if 'Velocity difference too high' in line]) badangl = len([line.strip() for line in loglines if 'Max convergence angle too small' in line]) - trajs = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] - spls = trajs[0].split(' ') - trajs = int(spls[1]) + if oldstyle: + trajs = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] + spls = trajs[0].split(' ') + trajs = int(spls[1]) + else: + trajs = 0 + trajlines = [line.strip() for line in loglines if 'Saved' in line and '0 candidates' not in line] + for li in trajlines: + spls=li.split(' ') + trajs += int(spls[-2]) nonphys = beglowr + badalti + badvelo + badangl tot = added + uncal + missdf From c8d8fa81733e67fa90e43742251889a6f7483125 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Apr 2026 13:21:47 +0100 Subject: [PATCH 060/287] get privip after restarting calcserver --- archive/analysis/runDistrib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 2ceee370..2fd6d493 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -121,6 +121,7 @@ while [ "$stat" -ne 16 ]; do log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done +privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) execcons=execconsol.sh execConsolsh=/tmp/$execcons From e5779d4419a11e202c8c6f59feabbab7979af9b7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Apr 2026 14:12:38 +0100 Subject: [PATCH 061/287] temporarily don't run rerunFailedLambdas in test --- archive/analysis/findAllMatches.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 093e4c25..0c6622ef 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -68,7 +68,10 @@ fi log2cw $NJLOGGRP $NJLOGSTREAM "Solving Run Done" findAllMatches log2cw $NJLOGGRP $NJLOGSTREAM "start rerunFailedLambdas" findAllMatches -python -m maintenance.rerunFailedLambdas +if [ "$RUNTIME_ENV" == "PROD" ] ; then + # don't do this yet outside Prod as the code doesn't handle it + python -m maintenance.rerunFailedLambdas +fi cd $here log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches From 431f6ec8d6654f0a9b281d1037788ca71517604b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Apr 2026 15:06:02 +0100 Subject: [PATCH 062/287] update to work in a test env too --- .../pythoncode/getExtraFiles.py | 23 +++++++++++++++---- .../getExtraFilesV2/tests/testEvent2.json | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py index 8c71bd7b..de54d28f 100644 --- a/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py +++ b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py @@ -36,10 +36,15 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): yr = orbname[:4] ym = orbname[:6] ymd= orbname[:8] + istest = False if int(yr) > 2020: webpth = f'reports/{yr}/orbits/{ym}/{ymd}/{orbname}/' else: webpth = f'reports/{yr}/orbits/{ym}/{orbname}/' + + if 'test' in key: + webpth = f'dummy/{yr}/orbits/{ym}/{ymd}/{orbname}/' + istest = True outdir = os.path.join(tmpdir, orbname) try: @@ -162,7 +167,7 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): pass print('pushing files back') - pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3) + pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3, istest=istest) print('updating the index page') createOrbitPageIndex(outdir, websitebucket, s3) @@ -241,7 +246,7 @@ def findOtherFiles(evtdt, archbucket, websitebucket, outdir, fldr, s3): return -def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): +def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3, istest=False): flist = os.listdir(outdir) _, pth =os.path.split(outdir) @@ -253,6 +258,10 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): else: webpth = f'reports/{yr}/orbits/{ym}/{pth}/' + if istest: + webpth = f'dummy/{yr}/orbits/{ym}/{ymd}/{pth}/' + + for f in flist: print(f) locfname = os.path.join(outdir, f) @@ -262,7 +271,10 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, websitebucket, key, ExtraArgs=extraargs) elif '.lst' in f: - key = os.path.join(f'matches/RMSCorrelate/trajectories/{yr}/{ym}/{ymd}/{pth}', f) + if istest: + key = os.path.join(f'matches/distrib/test/trajectories/{yr}/{ym}/{ymd}/{pth}', f) + else: + key = os.path.join(f'matches/RMSCorrelate/trajectories/{yr}/{ym}/{ymd}/{pth}', f) extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'summary' in f: @@ -271,7 +283,10 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'orbit_full.csv' in f: - key = os.path.join(f'matches/{yr}/fullcsv', f) + if istest: + key = os.path.join(f'matches/distrib/test/{yr}/fullcsv', f) + else: + key = os.path.join(f'matches/{yr}/fullcsv', f) extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) diff --git a/archive/lambdas/getExtraFilesV2/tests/testEvent2.json b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json index fd940082..9aa49b7f 100644 --- a/archive/lambdas/getExtraFilesV2/tests/testEvent2.json +++ b/archive/lambdas/getExtraFilesV2/tests/testEvent2.json @@ -7,7 +7,7 @@ "arn": "arn:aws:s3:::ukmda-shared" }, "object": { - "key": "matches/RMSCorrelate/trajectories/2024/202407/20240714/20240714_000041.319_UK/20240714_000041_trajectory.pickle" + "key": "matches/distrib/test/trajectories/2026/202604/20260412/20260412_232324.162_UK/20260412_232324_trajectory.pickle" } } } From f2440eab409dc8ac7d2d895237a36d1e49e53b9c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Apr 2026 16:23:14 +0100 Subject: [PATCH 063/287] add 'period' to the match summary api --- archive/deployment/website_static.yml | 1 - archive/lambdas/matchDataApi/testSumPeriod.json | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 archive/lambdas/matchDataApi/testSumPeriod.json diff --git a/archive/deployment/website_static.yml b/archive/deployment/website_static.yml index d8f79f70..c15044f4 100644 --- a/archive/deployment/website_static.yml +++ b/archive/deployment/website_static.yml @@ -27,7 +27,6 @@ - {src: '{{srcdir}}/static_content/js/', dest: '{{destdir}}/static_content/js', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/latest/', dest: '{{destdir}}/static_content/latest', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/live/', dest: '{{destdir}}/static_content/live', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/reports/', dest: '{{destdir}}/static_content/reports', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/search/', dest: '{{destdir}}/static_content/search', mode: '754', backup: no, directory_mode: yes } - {src: '{{srcdir}}/static_content/templates/', dest: '{{destdir}}/static_content/templates', mode: '754', backup: no, directory_mode: yes } diff --git a/archive/lambdas/matchDataApi/testSumPeriod.json b/archive/lambdas/matchDataApi/testSumPeriod.json new file mode 100644 index 00000000..81c4b419 --- /dev/null +++ b/archive/lambdas/matchDataApi/testSumPeriod.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "summary", + "reqval": "20251213", + "period": "0-3" + } +} \ No newline at end of file From 7a5374c3c441e4e1f92c14ff392660e03cd068c8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Apr 2026 16:23:41 +0100 Subject: [PATCH 064/287] add period to match summary api --- archive/lambdas/matchDataApi/matchDataApi.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/archive/lambdas/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py index da2048d1..e5084d2a 100644 --- a/archive/lambdas/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -78,13 +78,14 @@ def getStationData(statid, dtstr, period=None): return res -def getSummaryData(dtstr): +def getSummaryData(dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() + perfrag = periodToSqlFragment(period) if period is not None else "" connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' - expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%'" + expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%' {perfrag}" result=[] try: with connection.cursor() as cursor: @@ -137,8 +138,9 @@ def lambda_handler(event, context): res = getStationData(statid, dtstr) elif reqtyp == 'summary': dtstr = qs['reqval'] + period = qs['period'] if 'period' in qs else None print(f'summary data requested for {dtstr}') - res = getSummaryData(dtstr) + res = getSummaryData(dtstr, period) elif reqtyp == 'matches': dtstr = qs['reqval'] period = qs['period'] if 'period' in qs else None From f091a3940f76e41460e050ad65c659ab49b0945a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Apr 2026 22:44:03 +0100 Subject: [PATCH 065/287] re-enabling rerunFailedLambdas --- archive/analysis/findAllMatches.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 0c6622ef..093e4c25 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -68,10 +68,7 @@ fi log2cw $NJLOGGRP $NJLOGSTREAM "Solving Run Done" findAllMatches log2cw $NJLOGGRP $NJLOGSTREAM "start rerunFailedLambdas" findAllMatches -if [ "$RUNTIME_ENV" == "PROD" ] ; then - # don't do this yet outside Prod as the code doesn't handle it - python -m maintenance.rerunFailedLambdas -fi +python -m maintenance.rerunFailedLambdas cd $here log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches From 5763ceda945f0e3610561d00a8700da7e997d7d6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 14 Apr 2026 13:12:22 +0100 Subject: [PATCH 066/287] quieten an S3 transfer --- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 158f6dda..faafc6de 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -149,7 +149,7 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/canddbs/ {calcdir}/dbs/\n') - outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db"\n') + outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') refreshTrajectories(outf, matchstart, matchend, shbucket) From e14177700578bb976f7782504761be30c6d7caaf Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 14 Apr 2026 13:14:42 +0100 Subject: [PATCH 067/287] include the foldername when reporting on container progress --- .../ukmon_pylib/traj/distributeCandidates.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 69d8c80f..fb08dc4b 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -237,13 +237,14 @@ def monitorProgress(rundatestr, istest=''): s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {taskid} completed already') + print(f'{thisbuck}: task {taskid} completed already') getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) for tsk in sts['tasks']: taskid = tsk["taskArn"].split('/')[-1] - print(f'checking {taskid}') idx = taskarns.index(tsk['taskArn']) + _, thisbuck = os.path.split(bucknames[idx]) + print(f'{thisbuck}: checking {taskid} ') #print(tsk['taskArn'], tsk['lastStatus']) if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': @@ -251,7 +252,7 @@ def monitorProgress(rundatestr, istest=''): thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] - print('task restarted') + print(f'{thisbuck}: task restarted') else: thisarn=taskarns.pop(idx) thisbuck = bucknames.pop(idx) @@ -265,7 +266,7 @@ def monitorProgress(rundatestr, istest=''): s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print('task completed') + print(f'{thisbuck}: task completed') # collect the logs from CloudWatch getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) @@ -279,6 +280,7 @@ def monitorProgress(rundatestr, istest=''): thisbuck = bucknames.pop(idx) taskcount -= 1 _, thisbuck = os.path.split(thisbuck) + taskid = tsk["arn"].split('/')[-1] try: pref = f'{targdir}/{thisbuck}/' objects_to_delete = s3.list_objects(Bucket=outbucket, Prefix=pref) @@ -287,20 +289,21 @@ def monitorProgress(rundatestr, istest=''): s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print(f'task {thisarn.split("/")[-1]} completed already') + print(f'{thisbuck}: task {taskid} completed already') getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) for tsk in sts['tasks']: taskid = tsk["taskArn"].split('/')[-1] - print(f'checking {taskid}') idx = taskarns.index(tsk['taskArn']) + _, thisbuck = os.path.split(bucknames[idx]) + print(f'{thisbuck} : checking {taskid} ') if tsk['lastStatus'] == 'STOPPED': if tsk['stopCode'] != 'EssentialContainerExited': # retry the task thisjson = createTaskTemplate(rundate, bucknames[idx], clusdets, istest=istest) response = client.run_task(**thisjson) taskarns[idx] = response['tasks'][0]['taskArn'] - print('task restarted') + print(f'{thisbuck}: task restarted') else: thisarn=taskarns.pop(idx) thisbuck = bucknames.pop(idx) @@ -314,7 +317,7 @@ def monitorProgress(rundatestr, istest=''): s3.delete_objects(Bucket=outbucket, Delete=delete_keys) except: print('folder already removed') - print('task completed') + print(f'{thisbuck}: task completed') getContainerLog(thisarn, loggrp, contname, outbucket, s3, targdir, datadir) if taskcount > 0: @@ -373,7 +376,7 @@ def getLogDetails(loggrp, thisarn, contname, region_name='eu-west-2'): logstreamname = f'ecs/{contname}/{thisarn}' # first request - print(f'looking for {logstreamname} in {loggrp}') + #print(f'looking for {logstreamname} in {loggrp}') try: response = client.get_log_events( logGroupName=loggrp, From 7ffa200558f72e5a6fb68b8528a082db42390597 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 14 Apr 2026 14:18:05 +0100 Subject: [PATCH 068/287] put daily trajdb in a safe place --- .../reports/reportOfLatestMatches.py | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index b6050a1c..4737c00b 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -16,7 +16,7 @@ from traj.pickleAnalyser import getVMagCodeAndStations from reports.CameraDetails import findSite, loadLocationDetails -from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase, ObservationsDatabase def processLocalFolder(trajdir, basedir): @@ -41,8 +41,10 @@ def processLocalFolder(trajdir, basedir): return outstr -def getListOfNewMatches(dir_path): - trajdb = TrajectoryDatabase('/tmp', purge_records=True) +def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): + os.makedirs(db_path, exist_ok=True) + db_name = f'{rundate}_trajectories.db' if rundate else 'trajectories.db' + trajdb = TrajectoryDatabase(db_path = db_path, db_name=db_name, purge_records=True) flist = glob.glob(os.path.join(dir_path, 'trajectories_*.db')) flist.sort() for fl in flist: @@ -69,9 +71,29 @@ def getListOfNewMatches(dir_path): return newdirs -def findNewMatches(dir_path, out_path, offset, repdtstr): +def updatePairedDB(dir_path, db_path='/tmp', rundate=None): + os.makedirs(db_path, exist_ok=True) + db_name = f'{rundate}_observations.db' if rundate else 'observations.db' + obsdb = ObservationsDatabase(db_path=db_path, db_name=db_name, purge_records=True) + flist = glob.glob(os.path.join(dir_path, 'observations_*.db')) + flist.sort() + for fl in flist: + tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') + print(f'{tstamp} processing {fl}') + if obsdb.mergeObsDatabase(fl): + os.remove(fl) + else: + print('error') + + cur = obsdb.dbhandle.execute('select count(*) from paired_obs where status=1') + obscount = cur.fetchall() - newdirs = getListOfNewMatches(dir_path) + return obscount + + +def findNewMatches(dir_path, out_path, offset, repdtstr): + daily_path = os.path.join(os.path.split(dir_path)[0], 'dailydbs') + newdirs = getListOfNewMatches(dir_path, daily_path, rundate=repdtstr) # load camera details caminfo = loadLocationDetails() caminfo = caminfo[caminfo.active==1] @@ -147,9 +169,11 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): if len(sys.argv) > 4: repdtstr = sys.argv[4] - srcdir = sys.argv[1] - outdir = sys.argv[2] + cand_db_dir = sys.argv[1] + daily_db_dir = sys.argv[2] offset = sys.argv[3] # arguments dblocation, datadir, days ago, rundate eg 20220524 - findNewMatches(srcdir, outdir, offset, repdtstr) + findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) + # update the daily database of paired observations + updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) From bb4f9fb8c9474c7f29f1ae1fdcb3af836e96e8cf Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 15 Apr 2026 09:23:39 +0100 Subject: [PATCH 069/287] be more specific when finding the daily log --- archive/analysis/findAllMatches.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 093e4c25..cd2c33b7 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -75,7 +75,7 @@ log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches -dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) +dailyrep=$(ls -1tr $DATADIR/dailyreports/20*.txt | tail -1) trajlist=$(cat $dailyrep | awk -F, '{print $2}') matchlog=${SRC}/logs/matchJob.log From 19aad0ef84ce48db35014e8e1399486bcafb3cca Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 15 Apr 2026 09:24:05 +0100 Subject: [PATCH 070/287] only tar the current container logs --- archive/analysis/runDistrib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 2fd6d493..9ea55020 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -159,7 +159,7 @@ tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db tar czvf $DATADIR/distrib/contdbs_${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle aws s3 cp $DATADIR/distrib/contdbs_${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet -tar czvf $DATADIR/distrib/contlogs_${rundate}.tgz $SRC/logs/distrib/correl*.log $SRC/logs/distrib/${rundate}_*.log +tar czvf $DATADIR/distrib/contlogs_${rundate}.tgz $SRC/logs/distrib/correlator_${rundate}*.log $SRC/logs/distrib/${rundate}_*.log find $DATADIR/distrib/ -name "cont_*.tgz" -mtime +30 -exec rm -f {} \; rm -f $DATADIR/distrib/${rundate}.pickle From 28080f67fb97254fd55c72d84d673169987c435e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 15 Apr 2026 09:24:32 +0100 Subject: [PATCH 071/287] save daily obsdb in the right place --- archive/ukmon_pylib/reports/reportOfLatestMatches.py | 1 + 1 file changed, 1 insertion(+) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index 4737c00b..4c020947 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -176,4 +176,5 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): # arguments dblocation, datadir, days ago, rundate eg 20220524 findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) # update the daily database of paired observations + daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) From b02c7dfe1c39c6a09b85e45563d2a4c7003e8fd0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 16 Apr 2026 12:26:24 +0100 Subject: [PATCH 072/287] rename ec2.tf to calcserver.tf --- archive/terraform/ukmda/{ec2.tf => calcserver.tf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename archive/terraform/ukmda/{ec2.tf => calcserver.tf} (100%) diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/calcserver.tf similarity index 100% rename from archive/terraform/ukmda/ec2.tf rename to archive/terraform/ukmda/calcserver.tf From 70359b8ffc4567d015975a498048e513fd581018 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 16 Apr 2026 12:26:41 +0100 Subject: [PATCH 073/287] quieten some s3 transfers --- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index faafc6de..d3b6c99c 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -116,7 +116,7 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): trajloc = f'trajectories/{yr}/{yr}{mth:02d}/{yr}{mth:02d}{dy:02d}' out_dir = '~/data/distrib' outf.write(f'python -c "from traj.pickleAnalyser import getAllImages;getAllImages(\'{trajloc}\', \'{out_dir}\');"\n') - outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt"\n') + outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt --quiet"\n') outf.write(f'rm {out_dir}/consumed_*.txt\n') return From 4a3b726051e687ebd07e578fed3d71d37b5dffa1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 16 Apr 2026 12:29:58 +0100 Subject: [PATCH 074/287] rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is --- archive/terraform/mjmm/{ec2.tf => ukmonhelper_ec2.tf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename archive/terraform/mjmm/{ec2.tf => ukmonhelper_ec2.tf} (100%) diff --git a/archive/terraform/mjmm/ec2.tf b/archive/terraform/mjmm/ukmonhelper_ec2.tf similarity index 100% rename from archive/terraform/mjmm/ec2.tf rename to archive/terraform/mjmm/ukmonhelper_ec2.tf From b7badb9db40c6871bf8b356ac86949393ab76cea Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 16 Apr 2026 12:34:12 +0100 Subject: [PATCH 075/287] update SSM params to reflect new serverid --- archive/terraform/mjmm/ssm_parameters.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index d8355217..666e08e3 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -50,7 +50,8 @@ resource "aws_ssm_parameter" "prod_envname" { resource "aws_ssm_parameter" "prod_calcinstance" { name = "prod_calcinstance" type = "String" - value = "i-04cd701c3cfc980f5" + value = "i-0ab47af23705beb31" + #value = "i-04cd701c3cfc980f5" tags = { "billingtag" = "ukmon" } @@ -59,7 +60,8 @@ resource "aws_ssm_parameter" "prod_calcinstance" { resource "aws_ssm_parameter" "prod_calcuser" { name = "prod_calcuser" type = "String" - value = "ec2-user" # "unbuntu" + value = "unbuntu" + #value = "ec2-user" # "unbuntu" tags = { "billingtag" = "ukmon" } From dd92ad0bec98d4abc5708ec331e6c4a09036979c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 16 Apr 2026 17:55:43 +0100 Subject: [PATCH 076/287] bugfix in distributeCandidates - using wrong path --- archive/ukmon_pylib/traj/distributeCandidates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index fb08dc4b..fea43d62 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -91,7 +91,8 @@ def getDebugStatus(): def distributeCandidates(rundate, srcdir, maxcount=20, istest=False): clusdets = getClusterDetails(istest=istest) - _, targdir, _ = getTrajsolverPaths(istest=istest) + # the target for distribution is the source for the trajsolver + targdir, _, _ = getTrajsolverPaths(istest=istest) clusname = clusdets[0] From 9d3464ea64d09b78bc6078e6c68b57d5667e0466 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 16 Apr 2026 17:56:00 +0100 Subject: [PATCH 077/287] add newline to getCostMetrics --- archive/utils/getCostMetrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/utils/getCostMetrics.sh b/archive/utils/getCostMetrics.sh index 699bb257..99e7fc8a 100644 --- a/archive/utils/getCostMetrics.sh +++ b/archive/utils/getCostMetrics.sh @@ -26,4 +26,4 @@ python $PYLIB/metrics/costMetrics.py . eu-west-1 $thismth #export AWS_PROFILE=realukms #python $PYLIB/metrics/costMetrics.py . eu-west-1 $thismth -export AWS_PROFILE= \ No newline at end of file +export AWS_PROFILE= From 3d173c24e294749b4787124b94bdee6527264718 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 16 Apr 2026 17:56:18 +0100 Subject: [PATCH 078/287] weird bug in statsToMqtt --- archive/utils/statsToMqtt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh index 7725469b..e2334bfe 100644 --- a/archive/utils/statsToMqtt.sh +++ b/archive/utils/statsToMqtt.sh @@ -5,7 +5,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then source /home/bitnami/venvs/openhabstuff/bin/activate python $here/statsToMqtt.py diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') - if [ $diskpct -gt 9 ] ; then + if [ $diskpct -gt 90 ] ; then /home/bitnami/src/maintenance/mysql_checks.sh msg="$(hostname) diskspace was ${diskpct}%" subj="Diskspace alert for $(hostname)" From 78217c43c58c5c6319c665701b63d07b3141382b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 16 Apr 2026 17:57:52 +0100 Subject: [PATCH 079/287] bugfix in update_wmpl.sh for traj container --- archive/containers/trajsolver/update_wmpl.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/containers/trajsolver/update_wmpl.sh b/archive/containers/trajsolver/update_wmpl.sh index 1f2db1f3..1be9f5dc 100644 --- a/archive/containers/trajsolver/update_wmpl.sh +++ b/archive/containers/trajsolver/update_wmpl.sh @@ -8,9 +8,9 @@ else fi cd /mnt/e/dev/meteorhunting/ukmda-dataprocessing/archive/containers/$targ/ -ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout distrib_processing && git pull" +ssh ukmonhelper2 "cd ~/src/wmpldev && git stash && git checkout distrib_processing && git pull" rsync -avz ukmonhelper2:src/wmpldev/* ./WesternMeteorPyLib/ --exclude "build/" --exclude "*.egg*" --exclude "dist/" --exclude ".git/" --exclude "__pycache__/" -ssh ukmonhelper2 "cd src/wmpldev && git checkout - && git stash apply" +ssh ukmonhelper2 "cd ~/src/wmpldev && git checkout - && git stash apply" # remove modules and code that don't work in a container because theres no GUI rm -Rf ./WesternMeteorPyLib/wmpl/MetSim From 2e20b0f62259a66aeb838cecb8dd7c7f3123c60a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 16 Apr 2026 18:04:51 +0100 Subject: [PATCH 080/287] Script to create an AMI for the calcengine --- utils/reimageCalcServer.ps1 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 utils/reimageCalcServer.ps1 diff --git a/utils/reimageCalcServer.ps1 b/utils/reimageCalcServer.ps1 new file mode 100644 index 00000000..9f776af3 --- /dev/null +++ b/utils/reimageCalcServer.ps1 @@ -0,0 +1,9 @@ +# script to re-image the UKMON calc server +# copyright Mark McIntyre, 2026- + +$dtstr=(get-date -uformat "%Y%m%d") + +$instdetails=(aws ec2 describe-instances --filters Name="tag:Name",Values="calcengine_ub" --profile ukmonshared --region eu-west-2) +$instanceid= (Write-Output $instdetails| convertfrom-json)[0].reservations.instances.instanceid + +aws ec2 create-image --instance-id $instanceid --name "calcengine_${dtstr}" --description "Latest Calc Engine image" --profile ukmonshared --region eu-west-2 \ No newline at end of file From fdf117a016a7871ae61ffe686e5808511deecae6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Apr 2026 12:16:19 +0100 Subject: [PATCH 081/287] exclude test data when syncing from s3 --- archive/analysis/runDistrib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 9ea55020..6c0537ac 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -144,7 +144,7 @@ aws ec2 stop-instances --instance-ids $SERVERINSTANCEID # grab a copy of the indvidual container dbs so we can get a list of new solutions rm -Rf $DATADIR/latest/contdbs/ mkdir -p $DATADIR/latest/contdbs/ -aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/contdbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --quiet +aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $DATADIR/latest/contdbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --exclude "test/*" --quiet aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "*${rundate}*.db" --exclude "test/*" --exclude "dbs/*" --recursive --quiet aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATADIR/distrib --quiet From c834da0b04ea116a422f8dd7234b1e4efc43431a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Apr 2026 12:16:43 +0100 Subject: [PATCH 082/287] writing databases and logs to the wrong target --- archive/containers/trajsolver/trajsolver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py index 0d66c5e4..6bc0c19d 100644 --- a/archive/containers/trajsolver/trajsolver.py +++ b/archive/containers/trajsolver/trajsolver.py @@ -295,6 +295,8 @@ def startup(srcfldr, startdt, enddt, isTest=False): s3 = getS3Client() pushToWebsite(s3, trajfldr, webbucket, webpth, outbucket, outpth) + + # databases get pushed back to source location not target if srcfldr[:5] == 'test/': srcfldr = srcfldr[5:] for dbname in ['observations', 'trajectories', 'candidates']: @@ -302,14 +304,14 @@ def startup(srcfldr, startdt, enddt, isTest=False): fname = f'{dbname}_{srcfldr}.db' localfile = os.path.join(localfldr, f'{dbname}.db') if os.path.isfile(localfile): - targkey = f'{outpth}/{fname}' + targkey = f'{srcpth}/{fname}' print(f'uploading {localfile} to {srcbucket}/{targkey}') s3.meta.client.upload_file(localfile, srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) lognames = glob.glob(os.path.join(localfldr, '*.log')) if len(lognames)> 0: fname = f'correlator_{srcfldr}.log' - targkey = f'{outpth}/{fname}' + targkey = f'{srcpth}/{fname}' print(f'uploading {lognames[0]} to {srcbucket}/{targkey}') s3.meta.client.upload_file(lognames[0], srcbucket, targkey, ExtraArgs = getExtraArgs(fname)) From 0168530c4c7e7f55aaca958e985ea1204650192c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Apr 2026 12:16:55 +0100 Subject: [PATCH 083/287] bad test on istest --- archive/ukmon_pylib/traj/distributeCandidates.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index fea43d62..e81d2f1e 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -176,13 +176,13 @@ def distributeCandidates(rundate, srcdir, maxcount=20, istest=False): def monitorProgress(rundatestr, istest=''): - istest = False if istest=='' else True + istest = False if (istest=='' or istest.lower() == 'false') else True client = boto3.client('ecs', region_name='eu-west-2') s3 = boto3.client('s3') datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) clusdets = getClusterDetails(istest=istest) - _, targdir, _ = getTrajsolverPaths(istest=istest) + targdir, _, _ = getTrajsolverPaths(istest=istest) targdir = targdir[5:] outbucket=targdir[:targdir.find('/')] targdir = targdir[targdir.find('/')+1:] @@ -198,7 +198,7 @@ def monitorProgress(rundatestr, istest=''): try: s3.download_file(outbucket, rempickle, picklefile) except: - print('no containers to monitor') + print(f'no containers to monitor in {outbucket}/{rempickle}') return dumpdata = pickle.load(open(picklefile,'rb')) bucknames = dumpdata[0] From ad1b19bf78dd7440a61bc559d6654d0c8196bf2e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Apr 2026 12:17:07 +0100 Subject: [PATCH 084/287] typo in serveruserid --- archive/terraform/mjmm/ssm_parameters.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 666e08e3..73dff36b 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -60,8 +60,8 @@ resource "aws_ssm_parameter" "prod_calcinstance" { resource "aws_ssm_parameter" "prod_calcuser" { name = "prod_calcuser" type = "String" - value = "unbuntu" - #value = "ec2-user" # "unbuntu" + value = "ubuntu" + #value = "ec2-user" # "ubuntu" tags = { "billingtag" = "ukmon" } From b2ca89d931f81d40ab9c44ca39c8645cf1a88d52 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Apr 2026 12:26:26 +0100 Subject: [PATCH 085/287] adding to server migration notes --- archive/server_setup/migratingBatchServer.md | 25 ++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index cd413ef7..584a4d90 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -1,4 +1,21 @@ -# how to move accounts to a new server +# Replacing the UKMON helper server +The ukmon helper server provides two services. +* camera authentication and key management +* batch processing + +# how to move batch processing +The best approach is to reinstall all the code using the ansible deployment scripts. +Both dev and prod envs should be created, though arguably the dev env should be on a separate server. +Additionally, WMPL will need to be reinstalled. +After doing this, the additional UKMON python requirements can be added. + +The contents of ~/prod/data should be replicated to the new server. +The contents of ~/keymgmt should be replicated to the new server. + +Finally, double check that all required SSM variables exist in the account holding the server. These +are used to build the config file, but originally were only in Mark McIntyre's account. + +## how to move accounts to a new server The basic process is to extract the user accounts from the current server along with group and password info, then import it back in on the new server. Its important to avoid accidentally overwriting system or otherwise-existing accounts on the new server. @@ -8,7 +25,7 @@ new accounts starting at 1000 and working both upwards and downwards! Steps in brief. NB must all be done as root, of course. -## On the old server +### On the old server ``` bash mkdir /root/move/ export UGIDLIMIT=500 @@ -24,7 +41,7 @@ scp /root/move/* newserver:/tmp ``` -## On the new server +### On the new server In summary: backup the existing files, remove any accounts from the .mig files that are already present in the target, then append the filtered data. @@ -33,7 +50,7 @@ When comparing groups, remember new ids will get added to the sftp group. ``` bash mkdir -p /root/move/bkp -mv /tmp/*.mig /tmp/arsftp.tar.gz /root/move +mv /tmp/*.mig /tmp/varsftp.tar.gz /root/move cp /etc/passwd /etc/group /etc/shadow /etc/gshadow /root/move/bkp cd / From 5502f231e1d6a91c145ba5ac912ccc8457afa51b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Apr 2026 15:25:26 +0100 Subject: [PATCH 086/287] purge old rawcsvs and fullcsvs older than 180 days --- archive/analysis/consolidateOutput.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/archive/analysis/consolidateOutput.sh b/archive/analysis/consolidateOutput.sh index 34743434..5c2b8ac9 100644 --- a/archive/analysis/consolidateOutput.sh +++ b/archive/analysis/consolidateOutput.sh @@ -62,6 +62,10 @@ do fi done +logger -s -t consolidateOutput "purging older raw data which is on S3 anyway" +find ${DATADIR}/single/rawcsvs -mtime +180 -exec rm -f {} \; + + logger -s -t consolidateOutput "pushing consolidated information back" aws s3 sync ${DATADIR}/consolidated ${UKMONSHAREDBUCKET}/consolidated/ --exclude 'UKMON*' --quiet @@ -90,6 +94,12 @@ cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/matched/matches-full-$y cat ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv >> ${DATADIR}/searchidx/matches-full-$yr-new.csv mv ${DATADIR}/orbits/$yr/fullcsv/$yr*.csv ${DATADIR}/orbits/${yr}/fullcsv/processed +logger -s -t consolidateOutput "purging older raw data" +find ${DATADIR}/orbits/${yr}/fullcsv/processed -mtime +180 -exec rm -f {} \; +# and last year, because the data is split across dated folders +lyr=$(date -d 'last year' +%Y) +find ${DATADIR}/orbits/${lyr}/fullcsv/processed -mtime +180 -exec rm -f {} \; + python << EOD3 import pandas as pd df = pd.read_csv('${DATADIR}/matched/matches-full-${yr}.csv', skipinitialspace=True) From d1ffcbb918db4581017889a0b68247aeebd471e4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Apr 2026 15:43:42 +0100 Subject: [PATCH 087/287] improved housekeeping of old data --- archive/analysis/getRMSSingleData.sh | 5 ++++- archive/analysis/runDistrib.sh | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/archive/analysis/getRMSSingleData.sh b/archive/analysis/getRMSSingleData.sh index 4879e4b3..0174a815 100644 --- a/archive/analysis/getRMSSingleData.sh +++ b/archive/analysis/getRMSSingleData.sh @@ -56,7 +56,10 @@ fi # push to S3 bucket for future use by AWS tools logger -s -t getRMSSingleData "copy to S3 bucket" -aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --quiet +aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/single/ --exclude "*" --include "*.csv" --exclude "new/*" --exclude "rawcsvs/*" --exclude "used/*" --quiet aws s3 sync $SRC/data/single/ $UKMONSHAREDBUCKET/matches/singlepq/ --exclude "*" --include "*.parquet.snap" --exclude "*new.parquet.snap" --quiet +logger -s -t getRMSSingleData "purge processed data" +find $outdir/processed -mtime +180 -exec rm -f {} \; + logger -s -t getRMSSingleData "finished" \ No newline at end of file diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 6c0537ac..cd58a0ba 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -94,6 +94,8 @@ rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist" +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates + rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" @@ -156,12 +158,14 @@ log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib mkdir -p $DATADIR/trajdb tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db -tar czvf $DATADIR/distrib/contdbs_${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle -aws s3 cp $DATADIR/distrib/contdbs_${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet +mkdir -p $DATADIR/distrib/containers +tar czvf $DATADIR/distrib/containers/contdbs_${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle +aws s3 cp $DATADIR/distrib/containers/contdbs_${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet tar czvf $DATADIR/distrib/contlogs_${rundate}.tgz $SRC/logs/distrib/correlator_${rundate}*.log $SRC/logs/distrib/${rundate}_*.log -find $DATADIR/distrib/ -name "cont_*.tgz" -mtime +30 -exec rm -f {} \; +find $DATADIR/distrib/containers/ -name "cont_*.tgz" -mtime +30 -exec rm -f {} \; +find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \; rm -f $DATADIR/distrib/${rundate}.pickle From e98ea05a22d7ca798ac1916cf02dfd47f22c1fdc Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Apr 2026 15:47:56 +0100 Subject: [PATCH 088/287] make sure we scoop up data for 31/12 --- archive/cronjobs/nightlyJob.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index 2ff8b759..a9f83112 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -51,6 +51,10 @@ fi log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" nightlyJob # this creates the parquet table for Athena $SRC/analysis/getRMSSingleData.sh +if [ "$(date +%m%d)" == "0101" ] ; then + # catch any data uploaded on 01/01 that is for 31/12 the previous year + $SRC/analysis/getRMSSingleData.sh $(date -d 'last year' +%Y) +fi log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" nightlyJob From 8da59543293b3d65fd895d2120967a0779032dbc Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 17 Apr 2026 16:39:29 +0100 Subject: [PATCH 089/287] stop creation of an unused set of files --- archive/website/createOrbitIndex.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/website/createOrbitIndex.sh b/archive/website/createOrbitIndex.sh index 2d92cec2..3b05e1e8 100644 --- a/archive/website/createOrbitIndex.sh +++ b/archive/website/createOrbitIndex.sh @@ -90,9 +90,9 @@ else j=0 echo "" >> $idxfile fi - if [ $domth -eq 1 ] ; then - echo "$i" >> $DATADIR/orbits/$yr/$ym.txt - fi + #if [ $domth -eq 1 ] ; then + # echo "$i" >> $DATADIR/orbits/$yr/$ym.txt + #fi done fi echo "" >> $idxfile From 70761a4f4e6c180508f5c05fb15aea7beaadc9d1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 21 Apr 2026 09:41:29 +0100 Subject: [PATCH 090/287] change default locations for container data --- archive/containers/trajsolver/trajsolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py index 6bc0c19d..e9fdd9a9 100644 --- a/archive/containers/trajsolver/trajsolver.py +++ b/archive/containers/trajsolver/trajsolver.py @@ -117,12 +117,12 @@ def runCorrelator(dir_path, time_beg, time_end, max_stns=9999): # read the source bucket + folder and target buckets + folders from the environment def getSourceAndTargets(): - srcpth = os.getenv('SRCPATH', default='s3://ukmda-shared/matches/distrib') + srcpth = os.getenv('SRCPATH', default='s3://ukmda-shared/test/matches/distrib') srcpth = srcpth[5:] srcbucket = srcpth[:srcpth.find('/')] srcpth = srcpth[srcpth.find('/')+1:] - outpth = os.getenv('OUTPATH', default='s3://ukmda-shared/matches/distrib') + outpth = os.getenv('OUTPATH', default='s3://ukmda-shared/test/matches/RMSCorrelate') outpth = outpth[5:] outbucket = outpth[:outpth.find('/')] outpth = outpth[outpth.find('/')+1:] From 0f861e9cb5eca3c9f849b9fc211eecadfdac4645 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 21 Apr 2026 09:41:57 +0100 Subject: [PATCH 091/287] put container logs in the right place --- archive/analysis/runDistrib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index cd58a0ba..9350e5db 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -162,9 +162,9 @@ mkdir -p $DATADIR/distrib/containers tar czvf $DATADIR/distrib/containers/contdbs_${rundate}.tgz $DATADIR/latest/contdbs/*.db $DATADIR/distrib/${rundate}.pickle aws s3 cp $DATADIR/distrib/containers/contdbs_${rundate}.tgz $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/done/ --quiet -tar czvf $DATADIR/distrib/contlogs_${rundate}.tgz $SRC/logs/distrib/correlator_${rundate}*.log $SRC/logs/distrib/${rundate}_*.log +tar czvf $DATADIR/distrib/containers/contlogs_${rundate}.tgz $SRC/logs/distrib/correlator_${rundate}*.log $SRC/logs/distrib/${rundate}_*.log -find $DATADIR/distrib/containers/ -name "cont_*.tgz" -mtime +30 -exec rm -f {} \; +find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \; find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \; rm -f $DATADIR/distrib/${rundate}.pickle From b99e6bafa1c0c1084c485a582809252f9efa7f2f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 21 Apr 2026 09:42:32 +0100 Subject: [PATCH 092/287] update dockerfile syntax --- archive/lambdas/matchPickle/pythoncode/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/lambdas/matchPickle/pythoncode/Dockerfile b/archive/lambdas/matchPickle/pythoncode/Dockerfile index a7caae45..cb6745d5 100644 --- a/archive/lambdas/matchPickle/pythoncode/Dockerfile +++ b/archive/lambdas/matchPickle/pythoncode/Dockerfile @@ -3,13 +3,13 @@ FROM public.ecr.aws/lambda/python:3.8 COPY requirements.txt ./ RUN python3.8 -m pip install -r requirements.txt -t . -ENV LD_LIBRARY_PATH ./ +ENV LD_LIBRARY_PATH=./ # WMPL required even tho only one function called, because # pickle files reference structures defined in the library COPY WesternMeteorPyLib/ ./WesternMeteorPyLib -COPY *.npy ./WesternMeteorPyLib/wmpl/share/ +#COPY *.npy ./WesternMeteorPyLib/wmpl/share/ COPY *.py ./ -ENV PYTHONPATH ./WesternMeteorPyLib -ENV PROJ_LIB ./ +ENV PYTHONPATH=./WesternMeteorPyLib +ENV PROJ_LIB=./ CMD ["matchPickleApi.lambda_handler"] From b7bd0665bd3a46478ba18016d17da4235a788f24 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 21 Apr 2026 09:43:07 +0100 Subject: [PATCH 093/287] change locations of test container data and output --- archive/ukmon_pylib/traj/taskrunner_test.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/traj/taskrunner_test.json b/archive/ukmon_pylib/traj/taskrunner_test.json index fab30ba7..d834b6e2 100644 --- a/archive/ukmon_pylib/traj/taskrunner_test.json +++ b/archive/ukmon_pylib/traj/taskrunner_test.json @@ -20,11 +20,11 @@ "environment": [ { "name": "SRCPATH", - "value": "s3://ukmda-shared/matches/distrib/test" + "value": "s3://ukmda-shared/test/matches/distrib" }, { "name": "OUTPATH", - "value": "s3://ukmda-shared/matches/distrib/test" + "value": "s3://ukmda-shared/test/matches/RMSCorrelate" }, { "name": "WEBPATH", From c9df31934b9e412a14a3723664f8c02f3229a57b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 21 Apr 2026 22:30:21 +0100 Subject: [PATCH 094/287] remove incorrect comments --- archive/utils/loadMatchCsvMDB.sh | 1 - archive/utils/loadSingleCsvMDB.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/archive/utils/loadMatchCsvMDB.sh b/archive/utils/loadMatchCsvMDB.sh index 94363039..b2fd845f 100644 --- a/archive/utils/loadMatchCsvMDB.sh +++ b/archive/utils/loadMatchCsvMDB.sh @@ -1,7 +1,6 @@ #!/bin/bash # Copyright (C) 2018-2023 Mark McIntyre # -# script to clear out old log files and other old data here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 diff --git a/archive/utils/loadSingleCsvMDB.sh b/archive/utils/loadSingleCsvMDB.sh index a3b9c801..09fdaaaa 100644 --- a/archive/utils/loadSingleCsvMDB.sh +++ b/archive/utils/loadSingleCsvMDB.sh @@ -1,7 +1,6 @@ #!/bin/bash # Copyright (C) 2018-2023 Mark McIntyre # -# script to clear out old log files and other old data here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 From 41756b76bb25f69cd3de516601a8659acec54a3d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 00:48:46 +0100 Subject: [PATCH 095/287] Add script to clean up deleted/duplicate trajs --- archive/analysis/findAllMatches.sh | 3 ++ archive/utils/cleanupDeletedTrajs.sh | 51 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 archive/utils/cleanupDeletedTrajs.sh diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index cd2c33b7..3c788991 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -57,6 +57,9 @@ log2cw $NJLOGGRP $NJLOGSTREAM "solving for ${startdt} to ${enddt}" findAllMatche log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches $SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND +log2cw $NJLOGGRP $NJLOGSTREAM "clean duplicate/deleted trajs" findAllMatches +$SRC/utils/cleanupDeletedTrajs.sh + log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches success=$(grep "Total run time:" $SRC/logs/matchJob.log) diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh new file mode 100644 index 00000000..526c643c --- /dev/null +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre +# +# this script reads a list of logically-deleted trajectories from the sqlite database +# and moves the corresponding pickle and report data to a backup area +# These trajectories are generally duplicated events + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $here/../config.ini >/dev/null 2>&1 +conda activate $HOME/miniconda3/envs/${WMPL_ENV} + +cd ${DATADIR}/distrib + +startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') +jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") + +echo "checking main storage" +sqlite3 trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + trajdir=$(dirname $traj) + echo $trajdir + aws s3 ls ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/$fname ${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir/$fname + done +done + +echo "checking website" +sqlite3 trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + trajdir=$(dirname $traj) + yr=${trajdir:13:4} + trajpth=${trajdir:18} + echo $trajdir + webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth + aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv $webloc/$fname $newloc/$fname + done +done + +echo "checking fullcsv" +sqlite3 trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + trajdir=$(dirname $traj) + yr=${trajdir:13:4} + trajpth=${trajdir:34:19} + csvname=$(echo $trajpth | sed 's/_/-/g') + echo $csvname + csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} + aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do + aws s3 mv $csvloc/$fname $newloc/$fname + done +done From fc4fd1a5e0c7fb6edbae32b1ad0dc55c91f5164e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 11:41:38 +0100 Subject: [PATCH 096/287] simplify the stats gathering and move to python fix bug in getMatchStats --- archive/analysis/findAllMatches.sh | 14 +---- archive/ukmon_pylib/metrics/getMatchStats.py | 56 ++++++++++++++++---- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 3c788991..2fba436f 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -75,19 +75,9 @@ python -m maintenance.rerunFailedLambdas cd $here log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches -python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate - -log2cw $NJLOGGRP $NJLOGSTREAM "start getMatchStats" findAllMatches -dailyrep=$(ls -1tr $DATADIR/dailyreports/20*.txt | tail -1) -trajlist=$(cat $dailyrep | awk -F, '{print $2}') - matchlog=${SRC}/logs/matchJob.log -vals=$(python -m metrics.getMatchStats $matchlog ) -evts=$(echo $vals | awk '{print $2}') -trajs=$(echo $vals | awk '{print $6}') -matches=$(wc -l $dailyrep | awk '{print $1}') -rtim=$(echo $vals | awk '{print $7}') -echo $(basename $dailyrep) $evts $trajs $matches $rtim >> $DATADIR/dailyreports/stats.txt +python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate +python -m metrics.getMatchStats $matchlog # copy stats to S3 so the daily report can run if [ "$RUNTIME_ENV" == "PROD" ] ; then diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index 3ffb6c04..e1dcd4f9 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -5,6 +5,9 @@ from datetime import datetime import sys +import os +import glob +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase def getDailyObsCounts(logf): @@ -39,7 +42,7 @@ def getMatchStats(logf): events = [line.strip() for line in loglines if 'Analysing' in line and 'observations' in line] if len(events) > 0: # new-style log - addlines = [x for x in events if '0 observations' not in x] + addlines = [x for x in events if ' 0 observations' not in x] addoffs = -5 oldstyle = False else: @@ -60,16 +63,16 @@ def getMatchStats(logf): badvelo = len([line.strip() for line in loglines if 'Velocity difference too high' in line]) badangl = len([line.strip() for line in loglines if 'Max convergence angle too small' in line]) + cands = 0 if oldstyle: - trajs = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] - spls = trajs[0].split(' ') - trajs = int(spls[1]) + cands = [line.strip() for line in loglines if 'SAVING' in line and 'CANDIDATES' in line] + spls = cands[0].split(' ') + cands = int(spls[1]) else: - trajs = 0 - trajlines = [line.strip() for line in loglines if 'Saved' in line and '0 candidates' not in line] + trajlines = [line.strip() for line in loglines if 'Saved' in line and ' 0 candidates' not in line] for li in trajlines: spls=li.split(' ') - trajs += int(spls[-2]) + cands += int(spls[-2]) nonphys = beglowr + badalti + badvelo + badangl tot = added + uncal + missdf @@ -88,9 +91,42 @@ def getMatchStats(logf): d2=datetime.strptime(etim,'%H:%M:%S') cstime = str(d2 - d1) - return tot, added, uncal, missdf, nonphys, trajs, runtime, cstime + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + rundate=os.getenv('rundate') + if not rundate: + rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() + dailydbdir = os.path.join(datadir, 'latest','dailydbs') + trajdb = TrajectoryDatabase(dailydbdir, f'{rundate}_trajectories.db') + trajs = len(trajdb.getTrajBasics('.',[0,9999999])) + + return tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime + + +def updateStats(obscount, candcount, trajcount, runtime): + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + rundate=os.getenv('rundate') + if not rundate: + rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() + statsdir = os.path.join(datadir, 'dailyreports') + reports = glob.glob(os.path.join(statsdir, f'{rundate}*.txt')) + + # only update stats if the daily report exists already + if len(reports) > 0: + dailyrep = os.path.split(reports[-1])[1] + statsdata = open(os.path.join(statsdir,'stats.txt'), 'r').readlines() + + # remove any existing entry for today + currentstats = [li for li in statsdata if rundate in li] + if len(currentstats) > 0: + statsdata.pop(statsdata.index(currentstats[0])) + # add the new entry and save + statsdata.append(f'{dailyrep} {obscount} {candcount} {trajcount} {runtime}\n') + open(os.path.join(statsdir,'stats.txt'), 'w').writelines(statsdata) + return if __name__ == '__main__': - tot, added, uncal, missdf, nonphys, trajs, runtime, cstime = getMatchStats(sys.argv[1]) - print(tot, added, uncal, missdf, nonphys, trajs, runtime, cstime) + tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(sys.argv[1]) + if len(sys.argv) < 3: + updateStats(added, cands, trajs, runtime) + print(tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime) From 0df6c0a24525804469f6b17cd7dd746acfc56a2a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 11:42:03 +0100 Subject: [PATCH 097/287] comment out unused files --- archive/deployment/server_setup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/deployment/server_setup.yml b/archive/deployment/server_setup.yml index 661d96de..f53ea3a7 100644 --- a/archive/deployment/server_setup.yml +++ b/archive/deployment/server_setup.yml @@ -24,10 +24,10 @@ - {src: '{{srcdir}}/server_setup/install-proj4.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - {src: '{{srcdir}}/server_setup/install-sqlite3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - {src: '{{srcdir}}/server_setup/install_opencv.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } + #- {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - {src: '{{srcdir}}/server_setup/migratingBatchServer.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } - {src: '{{srcdir}}/server_setup/newserver.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - {src: '{{srcdir}}/server_setup/remount-s3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } From fd68a6714810bb60ae071556a0be3ac14624503f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 12:25:07 +0100 Subject: [PATCH 098/287] update reportOfLatestMatches to exclude deleted trajectories --- .../reports/reportOfLatestMatches.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index 4c020947..c8ffdc04 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -44,13 +44,13 @@ def processLocalFolder(trajdir, basedir): def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): os.makedirs(db_path, exist_ok=True) db_name = f'{rundate}_trajectories.db' if rundate else 'trajectories.db' - trajdb = TrajectoryDatabase(db_path = db_path, db_name=db_name, purge_records=True) + dailydb = TrajectoryDatabase(db_path=db_path, db_name=db_name, purge_records=True) flist = glob.glob(os.path.join(dir_path, 'trajectories_*.db')) flist.sort() for fl in flist: tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') print(f'{tstamp} processing {fl}') - if trajdb.mergeTrajDatabase(fl): + if dailydb.mergeTrajDatabase(fl): os.remove(fl) else: print('error') @@ -60,10 +60,35 @@ def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): else: trajdir = 'matches/RMSCorrelate' - cur = trajdb.dbhandle.execute('select traj_file_path from trajectories where status=1') + cur = dailydb.dbhandle.execute('select traj_file_path from trajectories where status=1') newtrajs = cur.fetchall() - trajdb.closeTrajDatabase() + if len(newtrajs) > 0: + # now get a list of logically-deleted trajs from the current master DB + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + + # get the date range for the new trajectories + cur = dailydb.dbhandle.execute('select min(jdt_ref), max(jdt_ref) from trajectories where status=1') + vals = cur.fetchall() + jdt_beg = float(vals[0][0]) + jdt_end = float(vals[0][1]) + + # retrieve a list of logically-deleted trajectories from within that date range + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + # iterate over the delete list and update the daily db and new traj list accordingly + for testtr in deltrajs: + if testtr in newtrajs: + sqlstr = f'update trajectories set status=0 where "traj_file_path={testtr[0]}";' + dailydb.dbhandle.execute(sqlstr) + newtrajs.pop(newtrajs.index(testtr)) + + dailydb.closeTrajDatabase() + newdirs = [] for traj in newtrajs: newdirs.append(os.path.join(trajdir, traj[0])) From 78ef3fd949321f90f41acc4be6b2a3dc0a5f7d0d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 17:39:21 +0100 Subject: [PATCH 099/287] add scp to requirements --- archive/ukmon_pylib/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/requirements.txt b/archive/ukmon_pylib/requirements.txt index aeaba31b..8b9f56c6 100644 --- a/archive/ukmon_pylib/requirements.txt +++ b/archive/ukmon_pylib/requirements.txt @@ -48,4 +48,5 @@ dynamodb_json pytest pytest-cov requests -paho-mqtt \ No newline at end of file +paho-mqtt +scp \ No newline at end of file From a40ecee756517e3b10a25378307466f047aff161 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 17:40:42 +0100 Subject: [PATCH 100/287] add SQL database maintenance to dataMaintenance --- .../maintenance/dataMaintenance.py | 96 +++++++++++++++++++ archive/utils/cleanupDeletedTrajs.sh | 63 ++++++++++-- 2 files changed, 153 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index 96d24dab..911abcb3 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -9,6 +9,13 @@ import paramiko from scp import SCPClient from time import sleep +import pandas as pd +import datetime + +import pymysql.cursors + +from wmpl.Utils.TrajConversions import datetime2JD +from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase def findInputDataByMonth(yyyymm, archbucket, outdir): @@ -97,6 +104,95 @@ def deleteFromCalcServerByMonth(outfname): return +def removeDeletedTraj(csvfile): + """ + Remove deleted trajectories from the consolidated match CSV and Parquet files + and from the search index. + """ + + csvdata = open(csvfile, 'r').readlines() + if 'search' in csvfile: + ts_end = float(csvdata[-1].split(',')[0]) + dt_end = datetime.datetime.fromtimestamp(ts_end, tz=datetime.timezone.utc) + jdt_end = datetime2JD(dt_end) + else: + jdt_end = float(csvdata[-1].split(',')[3]) + 2400000.5 + + jdt_beg =jdt_end - 21 + + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + i=0 + for traj in deltrajs: + fldr = os.path.basename(os.path.dirname(traj[0])) + match = [tr for tr in csvdata if fldr in tr] + if len(match) > 0: + for thismtch in match: + print(f'removing {fldr}') + idx = csvdata.index(thismtch) + _ = csvdata.pop(idx) + i += 1 + print(f'removed {i} trajectories') + + open(csvfile, 'w').writelines(csvdata) + + if 'search' not in csvfile: + df = pd.read_csv(csvfile, skipinitialspace=True) + df = df.drop_duplicates(subset=['_mjd','_sol','_ID1','_ra_o','_dc_o','_amag','_ra_t','_dc_t']) + df.to_csv(csvfile, index=False) + + return + + +def getSqlLoginDetails(): + # retrieve password and host from SSM. This allows me to manage them from Terraform + ssm = boto3.client('ssm', region_name='eu-west-1') + res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) + password = res['Parameter']['Value'] + res = ssm.get_parameter(Name='prod_dbhost') + host = res['Parameter']['Value'] + # should really do these too but they won't change often if at all + user = 'batch' + db = 'ukmon' + return host, user, password, db + + +def removeDelTrajFromDb(): + dt_end = datetime.datetime.now(tz=datetime.timezone.utc) + jdt_end = datetime2JD(dt_end) + jdt_beg =jdt_end - 7 + + # get list of deleted trajectories + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + masterdb.closeTrajDatabase() + + # get connection to the SQL database + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + + count = 0 + for traj in deltrajs: + fldr = os.path.basename(os.path.dirname(traj[0])) + sqlstr = f"delete from matches where orbname like '{fldr}%'" + with connection.cursor() as cursor: + cursor.execute(sqlstr) + result = cursor.fetchall() + count += len(result) + connection.commit() + connection.close() + print(f'cleaned up {count} trajectories') + return + + if __name__ == '__main__': arg_parser = argparse.ArgumentParser(description="Find and clear down historical raw data.") diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh index 526c643c..ed5fa0d3 100644 --- a/archive/utils/cleanupDeletedTrajs.sh +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -15,29 +15,35 @@ startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") echo "checking main storage" -sqlite3 trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do trajdir=$(dirname $traj) echo $trajdir + moved=0 aws s3 ls ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/$fname ${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir/$fname + aws s3 mv ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/$fname ${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir/$fname --quiet + moved=1 done + [ $moved == 1 ] && echo moved $trajdir done echo "checking website" -sqlite3 trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do trajdir=$(dirname $traj) yr=${trajdir:13:4} trajpth=${trajdir:18} echo $trajdir webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth + moved=0 aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv $webloc/$fname $newloc/$fname + aws s3 mv $webloc/$fname $newloc/$fname --quiet + moved=1 done + [ $moved == 1 ] && echo moved $trajdir done echo "checking fullcsv" -sqlite3 trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do trajdir=$(dirname $traj) yr=${trajdir:13:4} trajpth=${trajdir:34:19} @@ -45,7 +51,52 @@ sqlite3 trajectories.db "select traj_file_path from trajectories where status=0 echo $csvname csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} + moved=0 aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do - aws s3 mv $csvloc/$fname $newloc/$fname + aws s3 mv $csvloc/$fname $newloc/$fname --quiet + moved=1 done + [ $moved == 1 ] && echo moved $trajdir done + +echo "checking historic fullcsv just in case" +sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + trajdir=$(dirname $traj) + yr=${trajdir:13:4} + trajpth=$(basename $trajdir) + csvname=$(echo $trajpth | sed 's/_/-/g') + echo $csvname + csvloc=$DATADIR/orbits/${yr}/fullcsv/processed + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} + moved=0 + ls -1 $csvloc/ | grep $csvname | while read fname; do + aws s3 mv $csvloc/$fname $newloc/$fname --quiet + moved=1 + done + [ $moved == 1 ] && echo moved $trajdir + export trajpth +done + +echo "checking consolidated matches" +lasttraj=$(sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | tail -1) +trajdir=$(dirname $lasttraj) +yr=${trajdir:13:4} +trajpth=$(basename $trajdir) +matchcsv=$DATADIR/matched/matches-full-${yr}.csv +if [ $? == 0 ] ; then + python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$matchcsv')" + python -m converters.toParquet $matchcsv + aws s3 sync $DATADIR/matched/ $UKMONSHAREDBUCKET/matches/matched/ --include "*" --exclude "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + aws s3 sync $DATADIR/matched/ $UKMONSHAREDBUCKET/matches/matchedpq/ --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + aws s3 sync $DATADIR/matched/ $WEBSITEBUCKET/browse/parquet/ --exclude "*" --include "*.snap" --exclude "*.bkp" --exclude "*.gzip" --quiet + + srchcsv=$DATADIR/searchidx/${yr}-allevents.csv + python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$srchcsv')" + aws s3 sync $DATADIR/searchidx/ $WEBSITEBUCKET/search/indexes/ --exclude "*" --include "*allevents.csv" --quiet +fi + +export AWS_PROFILE=ukmonshared # needed for mariadb connection details +echo "checking Mariadb Database" +python -c "from maintenance.dataMaintenance import removeDelTrajFromDb;removeDelTrajFromDb()" +unset AWS_PROFILE +done \ No newline at end of file From a8c664501b3fe37481b003e05da1a34309b9e026 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 17:41:56 +0100 Subject: [PATCH 101/287] incorrect default for datadir --- archive/ukmon_pylib/maintenance/manageTraj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/maintenance/manageTraj.py b/archive/ukmon_pylib/maintenance/manageTraj.py index fae24d73..0a684bab 100644 --- a/archive/ukmon_pylib/maintenance/manageTraj.py +++ b/archive/ukmon_pylib/maintenance/manageTraj.py @@ -8,7 +8,7 @@ # delete an orbit from the database def deleteDuplicate(trajname, jd=None): - datadir = os.getenv('DATADIR', default='/home/pi/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) yr=trajname[:4] if int(yr) > 2021: fname = os.path.join(datadir, 'matched','matches-full-{}.parquet.snap'.format(yr)) From d9ec08fabac10397414aba056f8472e73806a0a8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 17:42:12 +0100 Subject: [PATCH 102/287] update call to pymysql.connect --- archive/lambdas/matchDataApi/matchDataApi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/lambdas/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py index e5084d2a..3000292d 100644 --- a/archive/lambdas/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -62,7 +62,7 @@ def periodToSqlFragment(period): def getStationData(statid, dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() - connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) try: statfrag = f"and s.stations like '%{statid}%' " if statid is not None else "" perfrag = periodToSqlFragment(period) if period is not None else "" From a3e8752dcf5656a403d0223308a1b0b059cdb458 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 18:43:41 +0100 Subject: [PATCH 103/287] ensure consolidateOutput rerunnable for prior years --- archive/analysis/consolidateOutput.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/analysis/consolidateOutput.sh b/archive/analysis/consolidateOutput.sh index 5c2b8ac9..51aa0418 100644 --- a/archive/analysis/consolidateOutput.sh +++ b/archive/analysis/consolidateOutput.sh @@ -46,12 +46,12 @@ do typ=${bn:0:3} if [ "$typ" != "M20" ] ; then pref="P" - yr=${bn:7:4} + snglyr=${bn:7:4} else pref="M" - yr=${bn:1:4} + snglyr=${bn:1:4} fi - mrgfile=${DATADIR}/consolidated/${pref}_${yr}-unified.csv + mrgfile=${DATADIR}/consolidated/${pref}_${snglyr}-unified.csv if [ ! -f $mrgfile ] ; then cat $csvf >> $mrgfile else From fd719d7fc4eccd8455e0d636aef09b907c1488ac Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 22:04:11 +0100 Subject: [PATCH 104/287] upgrading WMPL --- .../getExtraFilesV2/pythoncode/Dockerfile | 5 ++- .../WesternMeteorPyLib/requirements.txt | 22 ---------- .../pythoncode/getExtraFiles.py | 40 ++----------------- .../getExtraFilesV2/tests/remoteTest.ps1 | 1 - .../lambdas/getExtraFilesV2/update_wmpl.sh | 8 ++-- 5 files changed, 10 insertions(+), 66 deletions(-) delete mode 100644 archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt diff --git a/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile index 26f9034f..03a07a13 100644 --- a/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile +++ b/archive/lambdas/getExtraFilesV2/pythoncode/Dockerfile @@ -6,10 +6,11 @@ COPY requirements.txt ./ RUN python -m pip install -r requirements.txt -t . RUN python -m pip install numba watchdog basemap-data-hires -t . RUN python -m pip install cartopy -t . -#RUN yum update -y +RUN python -m pip install paramiko -t . + ENV LD_LIBRARY_PATH=./ COPY WesternMeteorPyLib/ ./WesternMeteorPyLib -#COPY *.npy ./WesternMeteorPyLib/wmpl/share/ + COPY *.py ./ ENV PYTHONPATH=./WesternMeteorPyLib ENV PROJ_LIB=./ diff --git a/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt b/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt deleted file mode 100644 index 81af2bc9..00000000 --- a/archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -numpy -cython -scipy -matplotlib==3.1.2 ; platform_machine != 'aarch64' -matplotlib==3.3.2 ; platform_machine == 'aarch64' -jplephem -pyephem -https://github.com/matplotlib/basemap/archive/master.zip ; sys_platform != 'win32' and platform_machine != 'aarch64' -basemap ; sys_platform == 'win32' or platform_machine == 'aarch64' -PyQt5 ; platform_machine != 'aarch64' -pyyaml -pyswarms -ml-dtypes -keras -pytz -pandas -cartopy -basemap-data-hires -gitpython -numba -watchdog==3.0.0; python_version == '3.7' -watchdog; python_version >= '3.8' \ No newline at end of file diff --git a/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py index de54d28f..83fb956c 100644 --- a/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py +++ b/archive/lambdas/getExtraFilesV2/pythoncode/getExtraFiles.py @@ -265,29 +265,25 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3, istest=False): for f in flist: print(f) locfname = os.path.join(outdir, f) + extraargs = getExtraArgs(locfname) # some files need to be pushed to the website, some to the archive bucket if '3dtrack' in f or '2dtrack' in f: key = os.path.join(webpth, f) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, websitebucket, key, ExtraArgs=extraargs) elif '.lst' in f: if istest: key = os.path.join(f'matches/distrib/test/trajectories/{yr}/{ym}/{ymd}/{pth}', f) else: key = os.path.join(f'matches/RMSCorrelate/trajectories/{yr}/{ym}/{ymd}/{pth}', f) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'summary' in f: key = os.path.join(fldr, f) - # print(locfname, key) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) elif 'orbit_full.csv' in f: if istest: key = os.path.join(f'matches/distrib/test/{yr}/fullcsv', f) else: key = os.path.join(f'matches/{yr}/fullcsv', f) - extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) return @@ -360,38 +356,8 @@ def getExtraArgs(fname): def lambda_handler(event, context): - # NB: IN THE CONSOLE, ADD TRUST RELATIONSHIP - # "AWS": "arn:aws:iam::317976261112:root" - # on the target role (replace by this lambda's ARN). - - # Note: if the lambda is connected to a VPC you must - # create a private subnet for the lambda - # create a NAT gateway on a public subnet - # create a routing table on the VPC to send 0.0.0.0/0 to the NAT gw - # attach the new subnet to the NAT gw - # - - sts_client = boto3.client('sts') - response = sts_client.get_caller_identity()['Account'] - if response == '317976261112': - assumed_role_object=sts_client.assume_role( - RoleArn="arn:aws:iam::183798037734:role/service-role/S3FullAccess", - RoleSessionName="GetExtraFilesV2") - - credentials=assumed_role_object['Credentials'] - - # Use the temporary credentials that AssumeRole returns to connections - s3 = boto3.resource('s3', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) - ddb = boto3.resource('dynamodb', region_name='eu-west-2', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) - else: - s3 = boto3.resource('s3') - ddb = boto3.resource('dynamodb', region_name='eu-west-2') + s3 = boto3.resource('s3') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') websitebucket = os.getenv('WEBSITEBUCKET') if websitebucket[:3] == 's3:': diff --git a/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 index 0a809831..fa8c832d 100644 --- a/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 +++ b/archive/lambdas/getExtraFilesV2/tests/remoteTest.ps1 @@ -15,4 +15,3 @@ else { aws lambda invoke --profile ukmonshared --function-name getExtraOrbitFilesV2 --log-type Tail --cli-binary-format raw-in-base64-out --payload file://tests/dummy.json --region eu-west-2 ./ftpdetect.log Remove-Item tests\dummy.json } - diff --git a/archive/lambdas/getExtraFilesV2/update_wmpl.sh b/archive/lambdas/getExtraFilesV2/update_wmpl.sh index 57d3fb6f..002e1f93 100644 --- a/archive/lambdas/getExtraFilesV2/update_wmpl.sh +++ b/archive/lambdas/getExtraFilesV2/update_wmpl.sh @@ -4,14 +4,14 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" cd $here/pythoncode -ssh ukmonhelper2 "cd src/wmpldev && git stash && git checkout forcontainer && git pull" +ssh ukmonhelper2 "cd ~/src/wmpldev && git stash && git checkout distrib_processing && git pull" rsync -avz ukmonhelper2:src/wmpldev/* ./WesternMeteorPyLib/ --exclude "build/" --exclude "*.egg*" --exclude "dist/" --exclude ".git/" --exclude "__pycache__/" ssh ukmonhelper2 "cd src/wmpldev && git checkout - && git stash apply" -mv -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so.0 ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so +rm -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so +cp -f ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so.0 ./WesternMeteorPyLib/wmpl/Trajectory/lib/trajectory/libtrajectorysolution.so # remove modules and code that don't work in a container because theres no GUI rm -Rf ./WesternMeteorPyLib/wmpl/MetSim rm -Rf ./WesternMeteorPyLib/wmpl/CAMO rm -f ./WesternMeteorPyLib/wmpl/Utils/DynamicMassFit.py -rm -f ./WesternMeteorPyLib/wmpl/Utils/PlotMap_OSM.py -cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py +#cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py From 769e68557e42cb41ae37bb340548ee037f434364 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 22:04:46 +0100 Subject: [PATCH 105/287] update gitignore a bit --- .gitignore | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 06dacff5..d266d959 100644 --- a/.gitignore +++ b/.gitignore @@ -556,11 +556,10 @@ archive/unused/** ukmon_pylib/tests/20220217_032206.832_UK/* unused/* *tfstate.lock.info -archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/* -archive/lambdas/getExtraFilesForEE/pythoncode/WesternMeteorPyLib/* -archive/containers/trajsolver/WesternMeteorPyLib/* +archive/lambdas/getExtraFilesV2/pythoncode/WesternMeteorPyLib/** +archive/containers/trajsolver/WesternMeteorPyLib/** archive/containers/trajsolver/awskeys -archive/containers/trajsolvertest/WesternMeteorPyLib/* +archive/containers/trajsolvertest/WesternMeteorPyLib/** archive/containers/trajsolvertest/awskeys #ukmon_pylib/traj/clusdetails.txt **/*.pickle @@ -618,4 +617,3 @@ fbcollector/config.ini usermgmt/windows/stationmaint.ini usermgmt/windows/README.md fbCollector/README.html -archive/containers/trajsolver/WesternMeteorPyLib.old/** From 6c26cf422119e94b1c1effd47f720bc5bf920d18 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 22:15:31 +0100 Subject: [PATCH 106/287] bugfix to catch data from 31/12 each year --- archive/cronjobs/nightlyJob.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index a9f83112..1c9eb120 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -66,15 +66,21 @@ ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" nightlyJob $SRC/website/updateIndexPages.sh -# from here down, we're creating reports +# FIRST CONSOLIDATE THE DATA AND CREATE SEARCH INDEXES # consolidate the output of the match process for further analysos log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob $SRC/analysis/consolidateOutput.sh ${yr} +if [ "$(date +%m%d)" == "0101" ] ; then + # catch any data uploaded on 01/01 that is for 31/12 the previous year + $SRC/analysis/consolidateOutput.sh $(date -d 'last year' +%Y) +fi # create the search indexes used on the website log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 2" nightlyJob $SRC/analysis/createSearchable.sh $yr matches +# FROM HERE DOWN WE'RE CREATING REPORTS + # add daily report to the website log2cw $NJLOGGRP $NJLOGSTREAM "start publishDailyReport" nightlyJob $SRC/website/publishDailyReport.sh From f89c69c803fdb53fc60a3f799abcd5c38af936d4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 22:19:13 +0100 Subject: [PATCH 107/287] add paramiko to dependencies for matchPickle to support wmpl upgrade --- archive/lambdas/matchPickle/pythoncode/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/lambdas/matchPickle/pythoncode/requirements.txt b/archive/lambdas/matchPickle/pythoncode/requirements.txt index 58aa7aa9..92834776 100644 --- a/archive/lambdas/matchPickle/pythoncode/requirements.txt +++ b/archive/lambdas/matchPickle/pythoncode/requirements.txt @@ -12,4 +12,5 @@ PyQt5 ; platform_machine != 'aarch64' pyyaml pyswarms keras -pytz \ No newline at end of file +pytz +paramiko \ No newline at end of file From b4b4f2826a4e795d9c4bdfd63ff16208c2e8f166 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 22:40:16 +0100 Subject: [PATCH 108/287] update getpickle to newest WMPL --- archive/lambdas/matchPickle/remoteTest.ps1 | 5 + archive/lambdas/matchPickle/testresult.json | 4157 +++++++++++++++++++ 2 files changed, 4162 insertions(+) create mode 100644 archive/lambdas/matchPickle/remoteTest.ps1 create mode 100644 archive/lambdas/matchPickle/testresult.json diff --git a/archive/lambdas/matchPickle/remoteTest.ps1 b/archive/lambdas/matchPickle/remoteTest.ps1 new file mode 100644 index 00000000..861de301 --- /dev/null +++ b/archive/lambdas/matchPickle/remoteTest.ps1 @@ -0,0 +1,5 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# test + + curl "https://api.ukmeteors.co.uk/pickle/getpickle?reqval=20260421_000017.759_UK" > ./testresult.json + diff --git a/archive/lambdas/matchPickle/testresult.json b/archive/lambdas/matchPickle/testresult.json new file mode 100644 index 00000000..7c1230d2 --- /dev/null +++ b/archive/lambdas/matchPickle/testresult.json @@ -0,0 +1,4157 @@ +{ + "jdt_ref": 2461151.5002055545, + "meastype": 2, + "output_dir": ".", + "max_toffset": 10.0, + "verbose": false, + "v_init_part": 0.4, + "v_init_ht": null, + "fixed_time_offsets": {}, + "fixed_time_offsets_copy": {}, + "estimate_timing_vel": true, + "fixed_times": null, + "monte_carlo": true, + "mc_runs": 10, + "mc_runs_max": 20, + "mc_pick_multiplier": 1, + "mc_noise_std": 1.0, + "enable_OSM_plot": true, + "geometric_uncert": true, + "filter_picks": false, + "calc_orbit": true, + "show_plots": false, + "show_jacchia": false, + "save_results": false, + "gravity_correction": true, + "gravity_factor": 1.0, + "plot_all_spatial_residuals": false, + "plot_file_type": "png", + "traj_id": "20260421000017_W5MFS", + "reject_n_sigma_outliers": 2, + "mc_cores": 2, + "file_name": "20260421_000017", + "meas_count": 19, + "observations": [ + { + "ObservedPoints.UK002Z": { + "meas1": [ + 2.3968813154069024, + 2.398794550755332, + 2.401040877732498, + 2.403248309562171, + 2.405262341887118, + 2.407007598336504, + 2.4083661435454724, + 2.4104380988088447, + 2.4124495492369737, + 2.414010003498822, + 2.4151889792750527, + 2.4171917886793013, + 2.417909396727613, + 2.42033600533781, + 2.4223435953587114, + 2.427942162403076, + 2.4300097550164326, + 2.4318363974955406, + 2.433899657791362, + 2.436516452224417, + 2.437238417883184 + ], + "meas2": [ + 0.44034695311399874, + 0.4392383275873981, + 0.4382869800294502, + 0.43791610200467623, + 0.43703226417084373, + 0.4362760437170148, + 0.4357724143366082, + 0.4343759101176623, + 0.4336724668163452, + 0.4324539154277791, + 0.4328902971191608, + 0.43031636173280585, + 0.43144285337433486, + 0.43049915328211963, + 0.4295252455056575, + 0.42636260506048096, + 0.4263728887871292, + 0.4259131817698758, + 0.4244646349323693, + 0.42476734916852354, + 0.4238170894671063 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 0.2905716440411789, + 0.33053564404117924, + 0.41047964404117904, + 0.4504116440411789, + 0.4903636440411791, + 0.5703196440411791, + 0.6102876440411793, + 0.6502516440411792, + 0.690195644041179, + 0.7301756440411791, + 0.7701196440411788, + 0.8101196440411789, + 0.850059644041179, + 0.889999644041179, + 0.929951644041179, + 1.0498356440411787, + 1.0897596440411788, + 1.169703644041179, + 1.209671644041179, + 1.249563644041179, + 1.2895636440411786 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 21, + "JD_data": [ + 2461151.500208891, + 2461151.5002093534, + 2461151.5002102787, + 2461151.500210741, + 2461151.5002112035, + 2461151.5002121287, + 2461151.500212591, + 2461151.500213054, + 2461151.500213516, + 2461151.500213979, + 2461151.500214441, + 2461151.500214904, + 2461151.5002153665, + 2461151.500215829, + 2461151.5002162913, + 2461151.5002176785, + 2461151.500218141, + 2461151.500219066, + 2461151.5002195286, + 2461151.5002199905, + 2461151.5002204534 + ], + "lat": 0.9344075681486413, + "lon": -0.006680073179083097, + "ele": 34.0, + "station_id": "UK002Z", + "azim_data": [ + 4.737799374816502, + 4.735428543040449, + 4.732911263205364, + 4.730867396557772, + 4.728583356038361, + 4.726610236803339, + 4.725138590050284, + 4.722411691527732, + 4.720262091239325, + 4.718078528987862, + 4.717466569845322, + 4.713889250782277, + 4.7141741061193105, + 4.711500673159142, + 4.709139283371754, + 4.702204122090407, + 4.700540864881191, + 4.698711574889019, + 4.695932771463546, + 4.69404324715463, + 4.692730245169177 + ], + "elev_data": [ + 0.5397212284763535, + 0.540021754973832, + 0.5406386453085916, + 0.5416703853261442, + 0.5422000352665981, + 0.5426661854062742, + 0.5430931842237807, + 0.5432695876107108, + 0.5439333938247444, + 0.5439396765993363, + 0.5449702160484455, + 0.5442139525051555, + 0.54549253181179, + 0.5462208350060278, + 0.5466770226216804, + 0.5476093934732065, + 0.5488458699002796, + 0.549583029220384, + 0.5497111435989721, + 0.5514955495795086, + 0.5512040280201354 + ], + "ra_data": [ + 2.396881315406902, + 2.3987945507553325, + 2.401040877732498, + 2.403248309562171, + 2.4052623418871177, + 2.407007598336504, + 2.4083661435454733, + 2.4104380988088443, + 2.4124495492369737, + 2.4140100034988214, + 2.4151889792750527, + 2.417191788679302, + 2.417909396727613, + 2.4203360053378105, + 2.422343595358711, + 2.427942162403076, + 2.430009755016433, + 2.431836397495541, + 2.433899657791363, + 2.4365164522244163, + 2.4372384178831847 + ], + "dec_data": [ + 0.44034695311399874, + 0.43923832758739784, + 0.4382869800294501, + 0.43791610200467607, + 0.43703226417084373, + 0.4362760437170146, + 0.43577241433660796, + 0.4343759101176626, + 0.4336724668163451, + 0.43245391542777895, + 0.432890297119161, + 0.43031636173280596, + 0.43144285337433513, + 0.4304991532821195, + 0.4295252455056574, + 0.42636260506048085, + 0.42637288878712926, + 0.42591318176987575, + 0.4244646349323693, + 0.4247673491685236, + 0.4238170894671061 + ], + "magnitudes": [ + 3.6, + 5.02, + 3.44, + 3.24, + 4.28, + 5.1, + 5.72, + 3.45, + 3.98, + 5.66, + 4.45, + 5.81, + 3.55, + 4.2, + 5.66, + 4.01, + 4.53, + 5.21, + 3.72, + 5.72, + 6.0 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK002Z_20260421-000018.239614_8851", + "comment": "{\"ff_name\": \"FF_UK002Z_20260421_000016_194_0375808.fits\"}", + "incident_angle": 0.5002051863784102, + "weight": 0.23002152819250413, + "h_residuals": [ + -3.8052913962245976, + 20.398349033358095, + 20.60146887676896, + -17.142668686892375, + -12.648791991966329, + -13.029467986359393, + -17.831075981704, + 20.861748677458234, + 11.816255498445493, + 51.257765767545706, + -14.989881371214677, + 106.20898562360125, + 5.1376366928819195, + 0.8473289069201014, + 9.980312274554471, + 63.8895794707902, + 3.432361870992362, + -21.53507262422365, + 17.085715282877228, + -78.20881756637142, + -36.77494843909438 + ], + "h_res_rms": 29.722498774935328, + "v_residuals": [ + 6.309529854796034, + -33.877011893083605, + -34.22656685546831, + 28.4211657875748, + 20.979912654651937, + 21.618339676041263, + 29.581808972747485, + -34.69571283606128, + -19.644738726302496, + -85.4334144263602, + 24.889885886707884, + -177.67230920382528, + -8.544025956103757, + -1.4090186969998264, + -16.60820767834652, + -106.71982725101019, + -5.713621282471512, + 35.80745108911481, + -28.480033854767836, + 129.65104400143866, + 61.12507390508546 + ], + "v_res_rms": 49.431213522476874, + "velocities": [ + 0.0, + 16249.106425374774, + 8967.863527321228, + 15682.132184664548, + 15881.2387686525, + 6875.364245819561, + 10338.63052400254, + 17524.6942293953, + 14943.888606668643, + 13556.02313277404, + 6169.26635208947, + 20213.790509050847, + 1179.9960627586386, + 17843.383954063283, + 15237.361660009743, + 14427.322364972191, + 12382.078647459066, + 6167.611210861438, + 16369.803445691245, + 14542.207740346299, + 6974.687234953151 + ], + "velocities_prev_point": [ + 11765.84594020067, + 11765.84594020067, + 11765.84594020067, + 11765.84594020067, + 12515.334991192343, + 11548.839904206608, + 11165.164595519736, + 11379.517422838991, + 11657.45887779439, + 11886.958654011225, + 11826.65374047157, + 12038.458636794907, + 11892.690809748638, + 11932.525875899712, + 12027.59652074097, + 12255.042346574837, + 12396.814201326875, + 12294.978224650129, + 12284.761091501927, + 12304.68522942524, + 12262.39262157771 + ], + "length": [ + 0.0, + 649.3792891836829, + 1366.3061710118493, + 1992.5250734098718, + 2627.0123246950798, + 3176.7389483338284, + 3589.953333117164, + 4290.310213300716, + 4887.228899805485, + 5429.198704653792, + 5675.623879821653, + 6484.175500183687, + 6531.304542930267, + 7243.9692980555565, + 7852.7323710962655, + 9582.337485498587, + 10076.679593419743, + 10569.743104060852, + 11224.01140817824, + 11804.129159356135, + 12083.116648754258 + ], + "state_vect_dist": [ + 3587.0536516011, + 4236.432940784783, + 4953.35982261295, + 5579.5787250109715, + 6214.0659762961795, + 6763.792599934928, + 7177.006984718264, + 7877.363864901816, + 8474.282551406584, + 9016.252356254892, + 9262.677531422753, + 10071.229151784788, + 10118.358194531367, + 10831.022949656655, + 11439.786022697366, + 13169.391137099687, + 13663.733245020843, + 14156.796755661951, + 14811.06505977934, + 15391.182810957234, + 15670.170300355358 + ], + "lag": [ + 63.37447151468132, + 194.5118468950268, + -125.25258238414244, + -16.860626788108675, + 99.54032331926737, + -387.5799767748267, + -492.65937666999525, + -310.54441028977544, + -231.60828321249573, + -208.08787566819592, + -479.64525992782546, + -189.8023912456356, + -660.6040370513838, + -465.86997047842124, + -375.1931986155396, + -200.2100838728802, + -223.59118100337764, + -767.2189814696103, + -631.2444620307251, + -568.4349489031392, + -808.1562111848561 + ], + "lag_line": [ + 11370.947134066559, + -3165.029756068875 + ], + "v_init": 11370.947134066559, + "v_init_stddev": null, + "jacchia_fit": [ + 63.688001193520016, + 1.8900846614547042 + ], + "model_ra": [ + 2.396881593167252, + 2.3989304099368014, + 2.4011844016151613, + 2.4031860875196687, + 2.4052274845571198, + 2.406976087071311, + 2.4083103572784346, + 2.4106035391564897, + 2.412566930909117, + 2.4143572664202093, + 2.41515920660715, + 2.417869111447515, + 2.418000831463778, + 2.420405613794107, + 2.4224695120777024, + 2.428399440764061, + 2.430109452127189, + 2.4317929026796112, + 2.4340906148854335, + 2.4361376027592625, + 2.4371099642448937 + ], + "model_dec": [ + 0.440302678909872, + 0.4394764728451123, + 0.43852840306686475, + 0.4377147344407611, + 0.43688315067561767, + 0.4361219869381554, + 0.4355611737515261, + 0.4346241691042197, + 0.4338134534871204, + 0.4330683131149763, + 0.43271070454145943, + 0.431599113939899, + 0.43150463507581566, + 0.4305093179638242, + 0.4296461972321007, + 0.4271459610813783, + 0.4264149242154197, + 0.42564839310551755, + 0.42467559658948373, + 0.42380137844967836, + 0.4233614958813678 + ], + "model_azim": [ + 4.73778196828684, + 4.735521732614828, + 4.7330053965160985, + 4.730788912250429, + 4.728525435294195, + 4.726550570784113, + 4.725056923942933, + 4.722507072008819, + 4.720316125854996, + 4.7183126731808445, + 4.717397902721562, + 4.714373400873156, + 4.714197584063138, + 4.711504522278045, + 4.709184901171225, + 4.702495517776749, + 4.700556524882403, + 4.69861306303917, + 4.696010730582082, + 4.6936850389188365, + 4.692562119253269 + ], + "model_elev": [ + 0.5396755224600758, + 0.5402679008661397, + 0.5408884116776695, + 0.5414620307648099, + 0.5420456131089348, + 0.5425065260896258, + 0.5428741613485486, + 0.5435273909554157, + 0.5440798903382158, + 0.5445789635092166, + 0.5447834940136214, + 0.5455517621083557, + 0.5455568612650257, + 0.5462314278769935, + 0.5468031898818537, + 0.548429156889144, + 0.5488898437745016, + 0.5493059467086585, + 0.549932237410441, + 0.5504839878814398, + 0.5507264098873919 + ], + "model_fit1": [ + 4.73778196828684, + 4.735521732614828, + 4.7330053965160985, + 4.730788912250429, + 4.728525435294195, + 4.726550570784113, + 4.725056923942933, + 4.722507072008819, + 4.720316125854996, + 4.7183126731808445, + 4.717397902721562, + 4.714373400873156, + 4.714197584063138, + 4.711504522278045, + 4.709184901171225, + 4.702495517776749, + 4.700556524882403, + 4.69861306303917, + 4.696010730582082, + 4.6936850389188365, + 4.692562119253269 + ], + "model_fit2": [ + 0.5396755224600758, + 0.5402679008661397, + 0.5408884116776695, + 0.5414620307648099, + 0.5420456131089348, + 0.5425065260896258, + 0.5428741613485486, + 0.5435273909554157, + 0.5440798903382158, + 0.5445789635092166, + 0.5447834940136214, + 0.5455517621083557, + 0.5455568612650257, + 0.5462314278769935, + 0.5468031898818537, + 0.548429156889144, + 0.5488898437745016, + 0.5493059467086585, + 0.549932237410441, + 0.5504839878814398, + 0.5507264098873919 + ], + "meas_eci": [ + [ + -0.6651403376230054, + 0.6131039197684128, + 0.42625334583010854 + ], + [ + -0.6666597964124698, + 0.6121494659158085, + 0.42525021719721046 + ], + [ + -0.6683315036211338, + 0.610923065694228, + 0.4243889832103093 + ], + [ + -0.6697948071375551, + 0.6095521783058309, + 0.424053131404759 + ], + [ + -0.6712985427876046, + 0.6084534187936955, + 0.42325252935984004 + ], + [ + -0.6725967633254724, + 0.6074952687408567, + 0.42256726378361675 + ], + [ + -0.6735794940210694, + 0.6067233106956108, + 0.4221107550073526 + ], + [ + -0.6752733035008325, + 0.6057194052358751, + 0.42084435091821104 + ], + [ + -0.6767109144101443, + 0.6045569835569059, + 0.42020613031078136 + ], + [ + -0.6780353559270645, + 0.6038403657657955, + 0.41910007013205 + ], + [ + -0.6786100121852274, + 0.6029190303333297, + 0.41949623862900476 + ], + [ + -0.6806225551669195, + 0.602272242271424, + 0.41715834354283843 + ], + [ + -0.6807019943374223, + 0.6014721244956676, + 0.4181818723470973 + ], + [ + -0.6824555864862213, + 0.6000788818635572, + 0.4173244637151418 + ], + [ + -0.6839643602217045, + 0.5989750663284349, + 0.4164392198909243 + ], + [ + -0.6882992447918803, + 0.5959956375357374, + 0.41356178457069387 + ], + [ + -0.6895268279492358, + 0.5945684648394516, + 0.41357114763571556 + ], + [ + -0.6907558791802513, + 0.5934317844296477, + 0.4131525536730498 + ], + [ + -0.6924328423375028, + 0.592393747301069, + 0.41183298436502674 + ], + [ + -0.6938856684688789, + 0.5904989435368124, + 0.4121088166678862 + ], + [ + -0.6946099128507072, + 0.5902511457747254, + 0.41124281620613995 + ] + ], + "meas_eci_los": [ + [ + -0.6651532201448609, + 0.6130899435642051, + 0.42625334583010854 + ], + [ + -0.6666744422176303, + 0.6121335155564811, + 0.42525021719721046 + ], + [ + -0.668349681390559, + 0.6109031791657809, + 0.4243889832103093 + ], + [ + -0.669814719865365, + 0.6095302968659562, + 0.424053131404759 + ], + [ + -0.6713201921678998, + 0.6084295324671147, + 0.42325252935984004 + ], + [ + -0.6726219198904363, + 0.6074674151436691, + 0.42256726378361675 + ], + [ + -0.6736063861027676, + 0.6066934539853648, + 0.4221107550073526 + ], + [ + -0.6753019174198893, + 0.6056875041052527, + 0.42084435091821104 + ], + [ + -0.6767412328073986, + 0.6045230449433344, + 0.42020613031078136 + ], + [ + -0.6780673992284806, + 0.6038043833219787, + 0.41910007013205 + ], + [ + -0.6786437629715607, + 0.602881040302245, + 0.41949623862900476 + ], + [ + -0.6806580260024921, + 0.6022321546139711, + 0.41715834354283843 + ], + [ + -0.6807391702046832, + 0.6014300489577449, + 0.4181818723470973 + ], + [ + -0.6824944243246932, + 0.6000347096215143, + 0.4173244637151418 + ], + [ + -0.6840048714786812, + 0.598928803723841, + 0.4164392198909243 + ], + [ + -0.6883447630503693, + 0.5959430656730842, + 0.41356178457069387 + ], + [ + -0.6895739692035289, + 0.5945137902859511, + 0.41357114763571556 + ], + [ + -0.6908063894172288, + 0.5933729853420778, + 0.4131525536730498 + ], + [ + -0.6924849898642252, + 0.5923327880522399, + 0.41183298436502674 + ], + [ + -0.693939367574386, + 0.5904358367808104, + 0.4121088166678862 + ], + [ + -0.6946653105480682, + 0.590185947342021, + 0.41124281620613995 + ] + ], + "model_eci": [ + [ + -3431648.7433502977, + -1734673.2501258308, + 5171746.249168466 + ], + [ + -3431502.168458378, + -1735198.4026242748, + 5171393.320457905 + ], + [ + -3431340.2471023854, + -1735778.206261865, + 5171003.513213945 + ], + [ + -3431198.8672727705, + -1736284.6365797631, + 5170663.117429066 + ], + [ + -3431055.612694895, + -1736797.753523634, + 5170318.214222535 + ], + [ + -3430931.3319215793, + -1737242.3632842037, + 5170019.113061614 + ], + [ + -3430837.965680444, + -1737576.5501712172, + 5169794.374518736 + ], + [ + -3430679.813194559, + -1738142.9361567015, + 5169413.623255734 + ], + [ + -3430544.9854288357, + -1738625.679263341, + 5169089.050559572 + ], + [ + -3430422.545034539, + -1739063.9846602748, + 5168794.316972965 + ], + [ + -3430366.765328334, + -1739263.305532724, + 5168660.1248029135 + ], + [ + -3430184.1694901455, + -1739917.1765316478, + 5168220.538002146 + ], + [ + -3430173.328761385, + -1739955.342499328, + 5168194.58388284 + ], + [ + -3430012.3423658106, + -1740531.6864444455, + 5167807.052084736 + ], + [ + -3429874.786286485, + -1741024.0084887384, + 5167475.953263732 + ], + [ + -3429483.8746772027, + -1742422.7843117185, + 5166535.0961491605 + ], + [ + -3429372.090910218, + -1742822.589670693, + 5166266.09232802 + ], + [ + -3429260.304079202, + -1743221.4092456542, + 5165997.30074232 + ], + [ + -3429112.4197078072, + -1743750.5190035156, + 5165641.380819524 + ], + [ + -3428981.246454044, + -1744219.694618384, + 5165325.712035401 + ], + [ + -3428918.0015465165, + -1744445.3384067302, + 5165173.638196536 + ] + ], + "meas_lat": [ + 0.9346450225936334, + 0.9345970660398534, + 0.9345465778299221, + 0.9345060097444198, + 0.9344609020518903, + 0.9344222008936756, + 0.9343935294446254, + 0.9343405561033522, + 0.9342993156178634, + 0.9342574203614202, + 0.93424620996589, + 0.9341778905737628, + 0.9341839571465814, + 0.9341340643559439, + 0.9340902381901774, + 0.9339630599390059, + 0.9339337530240859, + 0.9339012154522863, + 0.933851362550423, + 0.9338192346503082, + 0.9337956204733615 + ], + "meas_lon": [ + -0.04088550365419589, + -0.04075548711489885, + -0.04060781279524608, + -0.04046702814824051, + -0.04033520755003031, + -0.04022319461679362, + -0.040136396159376767, + -0.03999918021151192, + -0.03987197780910561, + -0.03976888838620386, + -0.03970217229821398, + -0.03956286802268136, + -0.03952989329711795, + -0.03937914973331329, + -0.039254062049537145, + -0.03890628729167373, + -0.0387879885582501, + -0.03868188951868613, + -0.03855458929929464, + -0.038409166927020906, + -0.03836284386104872 + ], + "meas_ht": [ + 80166.50920069027, + 79907.89847155106, + 79663.6709640296, + 79509.71227376115, + 79287.06050564267, + 79100.56625424455, + 78967.5446651115, + 78669.14774349949, + 78480.50679384157, + 78234.48391237574, + 78254.36663415054, + 77789.32418704791, + 77932.08594644774, + 77696.87267580134, + 77475.89917794727, + 76804.02455018043, + 76731.24392918951, + 76602.42998524354, + 76320.02753907315, + 76272.04281635238, + 76112.63523459999 + ], + "meas_range": [ + 153339.33047024682, + 152779.23373854544, + 152169.97299907313, + 151631.42779261048, + 151086.25079201206, + 150624.91058615095, + 150273.74436083622, + 149672.74252612054, + 149162.45846822963, + 148700.46758205735, + 148495.59961464474, + 147802.94181198507, + 147771.35775808938, + 147162.96362779627, + 146645.26441739433, + 145179.17570669635, + 144762.42956183938, + 144356.17304021947, + 143802.8283496073, + 143313.7343373524, + 143083.5198313615 + ], + "model_lat": [ + 0.9346446667677631, + 0.9345989704981107, + 0.9345485008239762, + 0.9345044075468767, + 0.9344597200105587, + 0.9344209834416608, + 0.9343918634192264, + 0.9343425020668983, + 0.9343004177428308, + 0.9342621973971849, + 0.9342448099303645, + 0.9341877726487242, + 0.9341844358943037, + 0.9341341428363436, + 0.9340911683984181, + 0.9339690073179128, + 0.9339340724672882, + 0.9338992059698612, + 0.9338529539754011, + 0.9338119290348905, + 0.9337921890458436 + ], + "model_lon": [ + -0.04088647065639903, + -0.0407503038625055, + -0.04060257766467196, + -0.040471381075913575, + -0.0403384201294265, + -0.04022650428958335, + -0.04014092463279356, + -0.03999387866505028, + -0.039868976417382226, + -0.03975584834373117, + -0.03970598017255938, + -0.03953580064279074, + -0.03952858935807945, + -0.0393789360113246, + -0.03925152725986469, + -0.03889002516887339, + -0.038787118004688446, + -0.038687359650227564, + -0.03855024718026426, + -0.03842899481740237, + -0.038372180120937446 + ], + "model_ht": [ + 80160.57129329837, + 79939.75564689818, + 79695.8571107965, + 79482.97408082303, + 79267.32029777888, + 79080.2240190833, + 78939.71062025194, + 78701.77767818491, + 78498.9785627082, + 78314.84618101093, + 78230.94224464067, + 77956.46774587498, + 77940.11468171058, + 77698.18875523485, + 77491.51556920519, + 76904.43232644908, + 76736.61042515974, + 76568.72510363306, + 76346.81825810629, + 76150.02295057768, + 76055.10224458351 + ], + "model_range": [ + 153339.33064750832, + 152779.23885477215, + 152169.9782410282, + 151631.43142680972, + 151086.25277939526, + 150624.9127025781, + 150273.74833255954, + 149672.74799874792, + 149162.46022825968, + 148700.50095141708, + 148495.6024593428, + 147803.08674493877, + 147771.3580936107, + 147162.9636368488, + 146645.26569589667, + 145179.22897841557, + 144762.42971472227, + 144356.17909111048, + 143802.8321819995, + 143313.81433522768, + 143083.53761941093 + ], + "rbeg_lat": 0.9346446667677631, + "rbeg_lon": -0.04088647065639903, + "rbeg_ele": 80160.57129329837, + "rbeg_jd": 2461151.500208891, + "rend_lat": 0.9337921890458436, + "rend_lon": -0.038372180120937446, + "rend_ele": 76055.10224458351, + "rend_jd": 2461151.5002204534, + "htmin_lat": 0.9337921890458436, + "htmin_lon": -0.038372180120937446, + "htmin_ele": 76055.10224458351, + "htmin_jd": 2461151.5002204534, + "absolute_magnitudes": [ + 2.671732184227931, + 4.099678289776929, + 2.528355107417953, + 2.3360538259627734, + 3.3838752499018385, + 4.210515958921451, + 4.835584403455401, + 2.5742863380034136, + 3.111702310827387, + 4.798437841987752, + 3.591432036646971, + 4.961582479831281, + 2.7020486763676725, + 3.3610073736334165, + 4.828659766224398, + 3.200477571944522, + 3.726720681188872, + 4.412823108180669, + 2.931162802418613, + 4.938559724900953, + 5.222051654133499 + ], + "ra_data_los": [ + 2.396902327617516, + 2.3988184762780778, + 2.401070632813468, + 2.403280977947147, + 2.4052979235842242, + 2.4070490095918355, + 2.408410468104811, + 2.4104853396142762, + 2.4124997004205735, + 2.414063070928515, + 2.415244960008752, + 2.417250685659096, + 2.4179712070195376, + 2.420400728941866, + 2.42241123226677, + 2.42801853924753, + 2.43008904516489, + 2.431921517202225, + 2.433987690810178, + 2.4366073956132723, + 2.4373322775181343 + ], + "dec_data_los": [ + 0.44034695311399874, + 0.43923832758739784, + 0.4382869800294501, + 0.43791610200467607, + 0.43703226417084373, + 0.4362760437170146, + 0.43577241433660796, + 0.4343759101176626, + 0.4336724668163451, + 0.43245391542777895, + 0.432890297119161, + 0.43031636173280596, + 0.43144285337433513, + 0.4304991532821195, + 0.4295252455056574, + 0.42636260506048085, + 0.42637288878712926, + 0.42591318176987575, + 0.4244646349323693, + 0.4247673491685236, + 0.4238170894671061 + ], + "x_eci": [ + -0.6651403376230054, + -0.6666597964124698, + -0.6683315036211338, + -0.6697948071375551, + -0.6712985427876046, + -0.6725967633254724, + -0.6735794940210694, + -0.6752733035008325, + -0.6767109144101443, + -0.6780353559270645, + -0.6786100121852274, + -0.6806225551669195, + -0.6807019943374223, + -0.6824555864862213, + -0.6839643602217045, + -0.6882992447918803, + -0.6895268279492358, + -0.6907558791802513, + -0.6924328423375028, + -0.6938856684688789, + -0.6946099128507072 + ], + "y_eci": [ + 0.6131039197684128, + 0.6121494659158085, + 0.610923065694228, + 0.6095521783058309, + 0.6084534187936955, + 0.6074952687408567, + 0.6067233106956108, + 0.6057194052358751, + 0.6045569835569059, + 0.6038403657657955, + 0.6029190303333297, + 0.602272242271424, + 0.6014721244956676, + 0.6000788818635572, + 0.5989750663284349, + 0.5959956375357374, + 0.5945684648394516, + 0.5934317844296477, + 0.592393747301069, + 0.5904989435368124, + 0.5902511457747254 + ], + "z_eci": [ + 0.42625334583010854, + 0.42525021719721046, + 0.4243889832103093, + 0.424053131404759, + 0.42325252935984004, + 0.42256726378361675, + 0.4221107550073526, + 0.42084435091821104, + 0.42020613031078136, + 0.41910007013205, + 0.41949623862900476, + 0.41715834354283843, + 0.4181818723470973, + 0.4173244637151418, + 0.4164392198909243, + 0.41356178457069387, + 0.41357114763571556, + 0.4131525536730498, + 0.41183298436502674, + 0.4121088166678862, + 0.41124281620613995 + ], + "x_eci_los": [ + -0.6651532201448609, + -0.6666744422176303, + -0.668349681390559, + -0.669814719865365, + -0.6713201921678998, + -0.6726219198904363, + -0.6736063861027676, + -0.6753019174198893, + -0.6767412328073986, + -0.6780673992284806, + -0.6786437629715607, + -0.6806580260024921, + -0.6807391702046832, + -0.6824944243246932, + -0.6840048714786812, + -0.6883447630503693, + -0.6895739692035289, + -0.6908063894172288, + -0.6924849898642252, + -0.693939367574386, + -0.6946653105480682 + ], + "y_eci_los": [ + 0.6130899435642051, + 0.6121335155564811, + 0.6109031791657809, + 0.6095302968659562, + 0.6084295324671147, + 0.6074674151436691, + 0.6066934539853648, + 0.6056875041052527, + 0.6045230449433344, + 0.6038043833219787, + 0.602881040302245, + 0.6022321546139711, + 0.6014300489577449, + 0.6000347096215143, + 0.598928803723841, + 0.5959430656730842, + 0.5945137902859511, + 0.5933729853420778, + 0.5923327880522399, + 0.5904358367808104, + 0.590185947342021 + ], + "z_eci_los": [ + 0.42625334583010854, + 0.42525021719721046, + 0.4243889832103093, + 0.424053131404759, + 0.42325252935984004, + 0.42256726378361675, + 0.4221107550073526, + 0.42084435091821104, + 0.42020613031078136, + 0.41910007013205, + 0.41949623862900476, + 0.41715834354283843, + 0.4181818723470973, + 0.4173244637151418, + 0.4164392198909243, + 0.41356178457069387, + 0.41357114763571556, + 0.4131525536730498, + 0.41183298436502674, + 0.4121088166678862, + 0.41124281620613995 + ], + "x_stat": -3329692.8394425996, + "y_stat": -1828618.1638918526, + "z_stat": 5106390.987831098, + "stat_eci": [ + -3329692.8394425996, + -1828618.1638918526, + 5106390.987831098 + ], + "stat_eci_los": [ + [ + -3329654.415397559, + -1828688.127695391, + 5106390.987831098 + ], + [ + -3329649.0878441236, + -1828697.8280102296, + 5106390.987831098 + ], + [ + -3329638.4272870813, + -1828717.2383623847, + 5106390.987831098 + ], + [ + -3329633.099663696, + -1828726.9386035914, + 5106390.987831098 + ], + [ + -3329627.771997192, + -1828736.6388563307, + 5106390.987831098 + ], + [ + -3329617.1112139, + -1828756.049084223, + 5106390.987831098 + ], + [ + -3329611.783477447, + -1828765.7492633294, + 5106390.987831098 + ], + [ + -3329606.450332315, + -1828775.4592229126, + 5106390.987831098 + ], + [ + -3329601.1278900444, + -1828785.1496290863, + 5106390.987831098 + ], + [ + -3329595.7946883356, + -1828794.859557595, + 5106390.987831098 + ], + [ + -3329590.466838816, + -1828804.5596746, + 5106390.987831098 + ], + [ + -3329585.1335805035, + -1828814.2695720182, + 5106390.987831098 + ], + [ + -3329579.805659577, + -1828823.9696850183, + 5106390.987831098 + ], + [ + -3329574.4777103914, + -1828833.6697824954, + 5106390.987831098 + ], + [ + -3329569.1497478094, + -1828843.369837391, + 5106390.987831098 + ], + [ + -3329553.1656459146, + -1828872.4699901317, + 5106390.987831098 + ], + [ + -3329547.837570296, + -1828882.1699829372, + 5106390.987831098 + ], + [ + -3329537.17593862, + -1828901.5797448508, + 5106390.987831098 + ], + [ + -3329531.847763332, + -1828911.2797181348, + 5106390.987831098 + ], + [ + -3329526.5249406365, + -1828920.9698801264, + 5106390.987831098 + ], + [ + -3329521.191342843, + -1828930.679591074, + 5106390.987831098 + ] + ], + "plane_N": [ + -0.010758941735242446, + 0.5626841701025131, + -0.8266019416192922 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 4.808344044228582e-05, + 0.00025879609182125133, + 0.0002624805684568251, + 0.0002189398121328159, + 0.00016219719514936625, + 0.00016763637746942077, + 0.0002299126917867094, + 0.0002704217170759464, + 0.0001536191301664089, + 0.000669935090213715, + 0.0001957385228524681, + 0.0014004153194447633, + 6.738763283318817e-05, + 1.1091740167342496e-05, + 0.00013204797495846036, + 0.0008566651982668407, + 4.595851975374786e-05, + 0.0002895389345591846, + 0.0002308693235996496, + 0.0010565990502203683, + 0.0004986368036220028 + ], + "ang_res_std": 0.0003960746824252553 + } + }, + { + "ObservedPoints.UK00AS": { + "meas1": [ + 4.2684960666937455, + 4.275705660567851, + 4.282316120282617, + 4.287776816608948, + 4.295439857062819, + 4.301929096125738, + 4.308588575262349, + 4.314596261320367, + 4.3215806860608525, + 4.327444662401953, + 4.331822069286927, + 4.335799177502339 + ], + "meas2": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.0734864670857704, + 1.0676655293254065, + 1.0625353098743915, + 1.0568295338648426, + 1.0530700757470863, + 1.047988416812209, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 0.8142981686851195, + 0.8543421686851195, + 0.8943901686851182, + 0.934414168685119, + 0.97446616868512, + 1.0145101686851197, + 1.054550168685119, + 1.09461416868512, + 1.1346821686851198, + 1.1747101686851193, + 1.2147341686851183, + 1.2547741686851195 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 12, + "JD_data": [ + 2461151.500214981, + 2461151.5002154447, + 2461151.500215908, + 2461151.5002163714, + 2461151.5002168347, + 2461151.5002172985, + 2461151.500217762, + 2461151.5002182256, + 2461151.5002186894, + 2461151.5002191523, + 2461151.5002196156, + 2461151.500220079 + ], + "lat": 0.9311541522498763, + "lon": -0.045808173347230974, + "ele": 57.0, + "station_id": "UK00AS", + "azim_data": [ + 0.8832534008354802, + 0.8974325902214364, + 0.9104409422636692, + 0.9232847050629855, + 0.9391342537659964, + 0.9528309723365957, + 0.9677775061336478, + 0.9774560061825399, + 0.9902964441331861, + 1.0053988251477577, + 1.01598915152872, + 1.0237734767338456 + ], + "elev_data": [ + 1.191408858155571, + 1.1885973861086219, + 1.1859092541919156, + 1.1836589415935101, + 1.1802988113469086, + 1.1773284999232583, + 1.1741538069154087, + 1.1711813003457707, + 1.1676161578412807, + 1.164451512475476, + 1.162008573079885, + 1.1597749824215797 + ], + "ra_data": [ + 4.2684960666937455, + 4.275705660567851, + 4.282316120282618, + 4.2877768166089485, + 4.295439857062819, + 4.301929096125738, + 4.308588575262349, + 4.314596261320368, + 4.3215806860608525, + 4.327444662401954, + 4.331822069286927, + 4.33579917750234 + ], + "dec_data": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.07348646708577, + 1.0676655293254065, + 1.0625353098743917, + 1.0568295338648421, + 1.0530700757470863, + 1.0479884168122084, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "magnitudes": [ + 2.07, + 2.55, + 2.28, + 2.28, + 2.54, + 2.07, + 2.44, + 2.91, + 1.92, + 2.53, + 4.48, + 3.45 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK00AS_20260421-000018.455603_8295", + "comment": "{\"ff_name\": \"FF_UK00AS_20260421_000008_555_0363264.fits\"}", + "incident_angle": 1.265480375315784, + "weight": 0.909642928791794, + "h_residuals": [ + 5.292367312510447, + -1.2325180959965676, + -8.34854165501151, + 18.532634348422874, + 21.298045268951483, + 26.66314727338721, + 45.7974867531929, + 1.1143548526059945, + -25.281726091833455, + 20.538142904874615, + 42.62245203257627, + 32.87553355400295 + ], + "h_res_rms": 25.272385757689367, + "v_residuals": [ + 1.948174626316228, + -0.453757025563914, + -3.0739954328321524, + 6.815581499143073, + 7.830632268485128, + 9.7997429292699, + 16.81714105751497, + 0.40987265024026254, + -9.307426018837527, + 7.546207273811804, + 15.64451435403657, + 12.069949587749356 + ], + "v_res_rms": 9.28371695850286, + "velocities": [ + 0.0, + 12969.297700466464, + 12063.810480597225, + 11535.301400186596, + 14810.349609688103, + 12948.52617841113, + 14077.237634374165, + 10248.745079827027, + 13145.40515860947, + 14305.426366502534, + 10388.296265044806, + 8232.259675064872 + ], + "velocities_prev_point": [ + 12177.028722194334, + 12177.028722194334, + 12177.028722194334, + 12177.028722194334, + 12635.927988514766, + 12811.691915261166, + 13010.049718796889, + 12862.45411623616, + 12812.459983391418, + 12868.413677161267, + 12786.266891221863, + 12563.748843977468 + ], + "length": [ + 0.0, + 519.3425571174787, + 1002.4740392444213, + 1464.162942485498, + 2057.347065052742, + 2575.8578473410325, + 3139.510442221366, + 3550.116165099566, + 4076.826258994727, + 4649.4438655930835, + 5065.225035305226, + 5394.844712694833 + ], + "state_vect_dist": [ + 9921.354086256408, + 10440.696643373885, + 10923.828125500828, + 11385.517028741906, + 11978.70115130915, + 12497.21193359744, + 13060.864528477774, + 13471.470251355973, + 13998.180345251134, + 14570.79795184949, + 14986.579121561634, + 15316.19879895124 + ], + "lag": [ + -393.8633893220631, + -393.8001635112796, + -429.99988356618087, + -487.3309572559665, + -413.52990774576574, + -414.29845676416335, + -369.87332231534674, + -478.8062851196937, + -471.68674778223067, + -418.1409889898896, + -521.3797962085864, + -710.987579250519 + ], + "lag_line": [ + 12179.051046767432, + -9899.353326496881 + ], + "v_init": 12179.051046767432, + "v_init_stddev": null, + "jacchia_fit": [ + 169.25151905025842, + 0.9545719833694175 + ], + "model_ra": [ + 4.268675306470282, + 4.275740112640083, + 4.282193626963838, + 4.288259757151979, + 4.295985212896367, + 4.302594155354585, + 4.3096754275247084, + 4.3147005165367975, + 4.321109894501091, + 4.327977157888565, + 4.332834122852471, + 4.336600836612544 + ], + "model_dec": [ + 1.0877269738198503, + 1.0827457055324, + 1.078088202406111, + 1.073615583501862, + 1.067812547679924, + 1.0627178988374508, + 1.0571404830083215, + 1.0530775705703026, + 1.0478192667242059, + 1.0420648001571293, + 1.0378881555178725, + 1.0345818657151975 + ], + "model_azim": [ + 0.8831700010505981, + 0.8974524277636791, + 0.9105774389431591, + 0.9229769805934369, + 0.9387740337613688, + 0.9523731955673052, + 0.9669789957103925, + 0.9774364487803786, + 0.9907468066797422, + 1.0050282483732862, + 1.0152133154617018, + 1.0231714421513614 + ], + "model_elev": [ + 1.1913491255580793, + 1.188611241077182, + 1.1860024765005819, + 1.1834532326767415, + 1.1800641024878358, + 1.1770365570460057, + 1.1736559939503042, + 1.171169280976094, + 1.1678878621525661, + 1.1642326384892834, + 1.161556949829732, + 1.1594281762863772 + ], + "model_fit1": [ + 0.8831700010505981, + 0.8974524277636791, + 0.9105774389431591, + 0.9229769805934369, + 0.9387740337613688, + 0.9523731955673052, + 0.9669789957103925, + 0.9774364487803786, + 0.9907468066797422, + 1.0050282483732862, + 1.0152133154617018, + 1.0231714421513614 + ], + "model_fit2": [ + 1.1913491255580793, + 1.188611241077182, + 1.1860024765005819, + 1.1834532326767415, + 1.1800641024878358, + 1.1770365570460057, + 1.1736559939503042, + 1.171169280976094, + 1.1678878621525661, + 1.1642326384892834, + 1.161556949829732, + 1.1594281762863772 + ], + "meas_eci": [ + [ + -0.19949752969882315, + -0.41951346273949935, + 0.885555865106421 + ], + [ + -0.19831370967000816, + -0.42489567883123996, + 0.8832527014775893 + ], + [ + -0.19719537224748762, + -0.42989189519454285, + 0.8810828244894001 + ], + [ + -0.19653440624741567, + -0.43469911459331356, + 0.8788691068257918 + ], + [ + -0.19526604393742353, + -0.44086253314794654, + 0.8760772790977996 + ], + [ + -0.19419198311969488, + -0.44623568023084337, + 0.8735921195735208 + ], + [ + -0.1931713762708322, + -0.45209532151521414, + 0.8708011481697114 + ], + [ + -0.19171870682741307, + -0.45626258010862597, + 0.8689467161138501 + ], + [ + -0.19020697856453836, + -0.4616673137779947, + 0.8664205657152649 + ], + [ + -0.18946461302037865, + -0.4676323484404185, + 0.8633789128216748 + ], + [ + -0.18879990840657096, + -0.4719169246748595, + 0.8611904613911451 + ], + [ + -0.18794699339414994, + -0.4752570683607163, + 0.8595386242905415 + ] + ], + "meas_eci_los": [ + [ + -0.19947261818984635, + -0.41952530837615254, + 0.885555865106421 + ], + [ + -0.19828723693595823, + -0.4249080335800251, + 0.8832527014775893 + ], + [ + -0.1971673332669236, + -0.42990475582643994, + 0.8810828244894001 + ], + [ + -0.19650478472685406, + -0.4347125057399289, + 0.8788691068257918 + ], + [ + -0.19523471543822826, + -0.44087640777926657, + 0.8760772790977996 + ], + [ + -0.19415896881448527, + -0.4462500458800414, + 0.8735921195735208 + ], + [ + -0.19313660866544666, + -0.4521101754434669, + 0.8708011481697114 + ], + [ + -0.19168228546062463, + -0.4562778824308138, + 0.8689467161138501 + ], + [ + -0.19016877668854015, + -0.4616830511058688, + 0.8664205657152649 + ], + [ + -0.18942455378532386, + -0.4676485767305336, + 0.8633789128216748 + ], + [ + -0.18875810448810454, + -0.4719336470299235, + 0.8611904613911451 + ], + [ + -0.18790350619325405, + -0.4752742636762536, + 0.8595386242905415 + ] + ], + "model_eci": [ + [ + -3430218.4459287277, + -1739794.238366333, + 5168303.099138084 + ], + [ + -3430101.130186622, + -1740214.0687953206, + 5168020.730054632 + ], + [ + -3429991.976360051, + -1740604.6083564344, + 5167758.024153548 + ], + [ + -3429887.652948811, + -1740977.7996632662, + 5167506.958303962 + ], + [ + -3429753.6593147092, + -1741457.3125123843, + 5167184.447316998 + ], + [ + -3429636.5043168305, + -1741876.432179816, + 5166902.494989177 + ], + [ + -3429509.159525174, + -1742332.044553453, + 5166596.010709642 + ], + [ + -3429416.332842804, + -1742663.8893094896, + 5166372.661447595 + ], + [ + -3429297.3084989507, + -1743089.6082505668, + 5166086.229687476 + ], + [ + -3429167.922097079, + -1743552.438223741, + 5165774.850941512 + ], + [ + -3429073.9084734865, + -1743888.437613092, + 5165548.665302157 + ], + [ + -3428999.3243624857, + -1744154.7560413252, + 5165369.277862051 + ] + ], + "meas_lat": [ + 0.9341977291845097, + 0.9341620422361886, + 0.9341289887201897, + 0.9340925154995868, + 0.9340502484747027, + 0.9340128678846222, + 0.9339702969792177, + 0.9339477745675356, + 0.9339143956069487, + 0.9338673009272427, + 0.9338347231487523, + 0.9338128429395532 + ], + "meas_lon": [ + -0.039569337714437845, + -0.0394604803690885, + -0.039359318234566246, + -0.03926606572328635, + -0.03914232800703642, + -0.039034834123200954, + -0.03891903030012694, + -0.03882974941824365, + -0.03871741853890708, + -0.038602319686930085, + -0.03851846883407553, + -0.03844994217817617 + ], + "meas_ht": [ + 78009.79883316693, + 77831.20354207544, + 77664.69644717604, + 77517.24209363233, + 77316.91530664213, + 77142.80788407566, + 76958.1916129066, + 76803.36867867864, + 76615.55707637589, + 76437.24094479789, + 76303.77256128346, + 76188.47275826824 + ], + "meas_range": [ + 83839.35913329039, + 83740.33676324955, + 83650.64655986936, + 83567.18288250422, + 83464.48024344677, + 83377.45673232341, + 83286.60977747764, + 83221.62612094023, + 83142.13428744262, + 83059.64575911379, + 83000.93009824904, + 82954.89554656077 + ], + "model_lat": [ + 0.9341984959671219, + 0.9341618633579815, + 0.9341277785486444, + 0.9340952007177374, + 0.934053334269257, + 0.9340167308727484, + 0.9339769318673277, + 0.9339479355618667, + 0.9339107317015435, + 0.9338702759470549, + 0.9338408971786679, + 0.933817604920053 + ], + "model_lon": [ + -0.039568822172874184, + -0.039460600665898946, + -0.03936013225744678, + -0.03926425828503005, + -0.03914025008272261, + -0.03903223179007913, + -0.03891455786744328, + -0.03882964093347483, + -0.03871988735385722, + -0.03860031305219093, + -0.03851430205713854, + -0.03844672807852617 + ], + "model_ht": [ + 78007.96659065987, + 77831.63104327311, + 77667.58915264205, + 77510.83050120615, + 77309.5485524352, + 77133.5883195504, + 76942.36899348421, + 76802.98414373519, + 76624.31618154436, + 76430.14134564155, + 76289.05225369317, + 76177.11611285266 + ], + "model_range": [ + 83839.35932285733, + 83740.33677357878, + 83650.64703317253, + 83567.18521487652, + 83464.4833274639, + 83377.46157053398, + 83286.62406500315, + 83221.62612936008, + 83142.13865346192, + 83059.64864007127, + 83000.94251389406, + 82954.90293702218 + ], + "rbeg_lat": 0.9341984959671219, + "rbeg_lon": -0.039568822172874184, + "rbeg_ele": 78007.96659065987, + "rbeg_jd": 2461151.500214981, + "rend_lat": 0.933817604920053, + "rend_lon": -0.03844672807852617, + "rend_ele": 76177.11611285266, + "rend_jd": 2461151.500220079, + "htmin_lat": 0.933817604920053, + "htmin_lon": -0.03844672807852617, + "htmin_ele": 76177.11611285266, + "htmin_jd": 2461151.500220079, + "absolute_magnitudes": [ + 2.4527602455285358, + 2.935326484323726, + 2.667653477222837, + 2.6698211294198684, + 2.9324914534964, + 2.4647566555887437, + 2.837123706123827, + 3.3088190122834766, + 2.3208940419532453, + 2.9330495519000843, + 4.884584879911243, + 3.855789702473728 + ], + "ra_data_los": [ + 4.268555447761289, + 4.2757679637412895, + 4.282381342636115, + 4.287844958142504, + 4.295510917776434, + 4.302003078945247, + 4.3086654772619175, + 4.314676085433959, + 4.321663432280336, + 4.327530324867534, + 4.331910650932565, + 4.3358906783280355 + ], + "dec_data_los": [ + 1.0876892604784882, + 1.082754431923786, + 1.0781467943009044, + 1.07348646708577, + 1.0676655293254065, + 1.0625353098743917, + 1.0568295338648421, + 1.0530700757470863, + 1.0479884168122084, + 1.0419285874153617, + 1.037607171800031, + 1.0343662231599273 + ], + "x_eci": [ + -0.19949752969882315, + -0.19831370967000816, + -0.19719537224748762, + -0.19653440624741567, + -0.19526604393742353, + -0.19419198311969488, + -0.1931713762708322, + -0.19171870682741307, + -0.19020697856453836, + -0.18946461302037865, + -0.18879990840657096, + -0.18794699339414994 + ], + "y_eci": [ + -0.41951346273949935, + -0.42489567883123996, + -0.42989189519454285, + -0.43469911459331356, + -0.44086253314794654, + -0.44623568023084337, + -0.45209532151521414, + -0.45626258010862597, + -0.4616673137779947, + -0.4676323484404185, + -0.4719169246748595, + -0.4752570683607163 + ], + "z_eci": [ + 0.885555865106421, + 0.8832527014775893, + 0.8810828244894001, + 0.8788691068257918, + 0.8760772790977996, + 0.8735921195735208, + 0.8708011481697114, + 0.8689467161138501, + 0.8664205657152649, + 0.8633789128216748, + 0.8611904613911451, + 0.8595386242905415 + ], + "x_eci_los": [ + -0.19947261818984635, + -0.19828723693595823, + -0.1971673332669236, + -0.19650478472685406, + -0.19523471543822826, + -0.19415896881448527, + -0.19313660866544666, + -0.19168228546062463, + -0.19016877668854015, + -0.18942455378532386, + -0.18875810448810454, + -0.18790350619325405 + ], + "y_eci_los": [ + -0.41952530837615254, + -0.4249080335800251, + -0.42990475582643994, + -0.4347125057399289, + -0.44087640777926657, + -0.4462500458800414, + -0.4521101754434669, + -0.4562778824308138, + -0.4616830511058688, + -0.4676485767305336, + -0.4719336470299235, + -0.4752742636762536 + ], + "z_eci_los": [ + 0.885555865106421, + 0.8832527014775893, + 0.8810828244894001, + 0.8788691068257918, + 0.8760772790977996, + 0.8735921195735208, + 0.8708011481697114, + 0.8689467161138501, + 0.8664205657152649, + 0.8633789128216748, + 0.8611904613911451, + 0.8595386242905415 + ], + "x_stat": -3413601.4235978713, + "y_stat": -1704419.3296659163, + "z_stat": 5094057.194026681, + "stat_eci": [ + -3413601.4235978713, + -1704419.3296659163, + 5094057.194026681 + ], + "stat_eci_los": [ + [ + -3413500.207340225, + -1704622.0299575112, + 5094057.194026681 + ], + [ + -3413495.2262395704, + -1704632.004559309, + 5094057.194026681 + ], + [ + -3413490.2500972687, + -1704641.969159245, + 5094057.194026681 + ], + [ + -3413485.273925883, + -1704651.9337446468, + 5094057.194026681 + ], + [ + -3413480.2977254046, + -1704661.89831553, + 5094057.194026681 + ], + [ + -3413475.316508251, + -1704671.8728591497, + 5094057.194026681 + ], + [ + -3413470.340249567, + -1704681.8374009654, + 5094057.194026681 + ], + [ + -3413465.358960294, + -1704691.8119432305, + 5094057.194026681 + ], + [ + -3413460.37765573, + -1704701.7864431958, + 5094057.194026681 + ], + [ + -3413455.4063112848, + -1704711.7409264494, + 5094057.194026681 + ], + [ + -3413450.4299362227, + -1704721.7054101357, + 5094057.194026681 + ], + [ + -3413445.4535320713, + -1704731.6698792966, + 5094057.194026681 + ] + ], + "plane_N": [ + -0.9633203118488703, + -0.08160610934519683, + -0.2556451049735453 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 6.724691477696396e-05, + 1.5706629728077543e-05, + 0.00010637754358927273, + 0.0002362632170843941, + 0.00027184572472634675, + 0.00034066933402676907, + 0.0005857413387279998, + 1.4225497938347832e-05, + 0.0003240762673397417, + 0.0002633833955488076, + 0.0005469632219021634, + 0.00042211398237902714 + ], + "ang_res_std": 0.0003236383693663661 + } + }, + { + "ObservedPoints.UK00CD": { + "meas1": [ + 3.2350197840765165, + 3.2477072959811264, + 3.2552292551574604, + 3.2758807935362744, + 3.293926737375255, + 3.3099222963095567, + 3.3239878762449746, + 3.3380897824049134, + 3.357389348527242, + 3.371805691892109, + 3.3873157784796977, + 3.4044770059058163, + 3.4212650205456225, + 3.4374557844393463 + ], + "meas2": [ + 1.2609999783504144, + 1.2612395779929737, + 1.2595198256953215, + 1.260472338963183, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814451, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153048, + 1.2547119136331997, + 1.2533115917530044, + 1.253010663890778 + ], + "jdt_ref": 2461151.500205556, + "time_data": [ + 6.187651883518428e-05, + 0.0399378765188352, + 0.0798018765188349, + 0.11962187651883464, + 0.15939787651883444, + 0.1992378765188352, + 0.2390978765188352, + 0.27892987651883483, + 0.3187098765188352, + 0.3585698765188352, + 0.3983738765188345, + 0.4381978765188348, + 0.4779978765188344, + 0.5178458765188345 + ], + "ignore_station": false, + "ignore_list": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "kmeas": 14, + "JD_data": [ + 2461151.500205559, + 2461151.5002060207, + 2461151.500206482, + 2461151.500206943, + 2461151.500207403, + 2461151.500207864, + 2461151.5002083257, + 2461151.5002087867, + 2461151.500209247, + 2461151.5002097087, + 2461151.5002101692, + 2461151.5002106302, + 2461151.500211091, + 2461151.500211552 + ], + "lat": 0.9302909996683025, + "lon": -0.039105358527994384, + "ele": 80.0, + "station_id": "UK00CD", + "azim_data": [ + 5.966419815139067, + 5.97542519016121, + 5.977778599822299, + 5.993954146606859, + 6.005275633071222, + 6.017243491584776, + 6.02792804365347, + 6.0375395619757395, + 6.051683132695393, + 6.063393040906969, + 6.074541567562709, + 6.0886289147110215, + 6.101951058174785, + 6.116263187282847 + ], + "elev_data": [ + 1.2028290600570986, + 1.2049672270465874, + 1.2077118383916157, + 1.2105467452617509, + 1.214565742676128, + 1.2171622436903107, + 1.2193966508469822, + 1.2223914732590124, + 1.226048237869094, + 1.2281100846973485, + 1.2315217824941145, + 1.2341667869259583, + 1.2373483805420769, + 1.2393083048928824 + ], + "ra_data": [ + 3.2350197840765156, + 3.2477072959811255, + 3.2552292551574595, + 3.275880793536273, + 3.293926737375254, + 3.3099222963095567, + 3.323987876244974, + 3.3380897824049134, + 3.3573893485272417, + 3.371805691892108, + 3.387315778479697, + 3.404477005905816, + 3.421265020545622, + 3.4374557844393454 + ], + "dec_data": [ + 1.2609999783504147, + 1.2612395779929737, + 1.2595198256953215, + 1.2604723389631833, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814453, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153052, + 1.2547119136331992, + 1.2533115917530042, + 1.2530106638907785 + ], + "magnitudes": [ + 4.86, + 4.32, + 5.89, + 4.98, + 2.84, + 2.54, + 2.93, + 2.47, + 2.12, + 2.31, + 2.72, + 2.1, + 2.7, + 2.14 + ], + "fov_beg": true, + "fov_end": true, + "obs_id": "UK00CD_20260421-000017.700500_9470", + "comment": "{\"ff_name\": \"FF_UK00CD_20260421_000013_479_0364800.fits\"}", + "incident_angle": 0.9673542667254906, + "weight": 0.6779664749397492, + "h_residuals": [ + 10.60965356642866, + -24.36076247965052, + 105.75947002249839, + -7.220710828385594, + 52.4106292421939, + 13.980469426264557, + -21.11566161888541, + 16.385999544642043, + 31.159896386556547, + -15.566908035957312, + 42.18031417134474, + 11.207776909509969, + 34.794840093942796, + -31.51281526197431 + ], + "h_res_rms": 38.65757618379624, + "v_residuals": [ + 3.2446947715231613, + -7.4621198683776795, + 32.17069118416699, + -2.20894625977602, + 15.980407194763492, + 4.270405033174591, + -6.460331591938528, + 5.002493265014071, + 9.503554253729916, + -4.7583795692783655, + 12.852019409716682, + 3.4197354351588864, + 10.601440013163899, + -9.632696675175211 + ], + "v_res_rms": 11.775952423901554, + "velocities": [ + 0.0, + 10471.680103624529, + 6537.224031186331, + 16633.282812297697, + 14826.52683173458, + 12873.383577289029, + 11286.270727626033, + 11554.888061040885, + 15564.781179446425, + 11390.115494189105, + 12773.925244016458, + 13582.991536140642, + 13615.88234822533, + 12542.882660933778 + ], + "velocities_prev_point": [ + 10743.85866816658, + 10743.85866816658, + 10743.85866816658, + 10743.85866816658, + 12007.055445941307, + 12492.235701727113, + 12536.279099510239, + 12472.845218100547, + 12649.080739643105, + 12670.109215040093, + 12686.294204401467, + 12730.115433503099, + 12785.779374174901, + 12813.830897857222 + ], + "length": [ + 0.0, + 417.56871581213187, + 678.1686145913418, + 1340.5059361770318, + 1930.2458674361035, + 2443.121469155308, + 2892.992220358482, + 3353.246521605858, + 3972.4135169242427, + 4426.4235205226205, + 4934.8768409354425, + 5475.8058958707115, + 6017.718013330074, + 6517.526801602965 + ], + "state_vect_dist": [ + 37.76837659225623, + 379.80033921987564, + 640.4002379990856, + 1302.7375595847757, + 1892.4774908438474, + 2405.3530925630516, + 2855.223843766226, + 3315.4781450136015, + 3934.6451403319866, + 4388.655143930364, + 4897.108464343187, + 5438.037519278456, + 5979.949636737819, + 6479.758425010708 + ], + "lag": [ + 281.3381680637624, + 106.26937614174375, + -150.0758670031762, + -4.113107714768603, + 69.82284087386734, + 66.06452591993548, + -0.9579939258560444, + -57.2338676012655, + 46.077274171508634, + -16.805993279080212, + -24.51975166286138, + -0.017129900050349534, + 25.779779637870888, + 8.85090948729794 + ], + "lag_line": [ + 12015.663931514431, + -83.75025379184109 + ], + "v_init": 12015.663931514431, + "v_init_stddev": null, + "jacchia_fit": [ + 0.0, + 0.0 + ], + "model_ra": [ + 3.2350317233883077, + 3.2476714150887465, + 3.255438140444483, + 3.2758702514549296, + 3.2941153488378436, + 3.3099924216832672, + 3.323910252516376, + 3.3381929059797426, + 3.3575930020386857, + 3.371733421626219, + 3.3876369233400556, + 3.404593711003503, + 3.4215849629232125, + 3.437217005959056 + ], + "model_dec": [ + 1.2611273499274973, + 1.2609464139140631, + 1.260793448991946, + 1.2603850642422985, + 1.259911335665489, + 1.2594122969591925, + 1.2589067455139373, + 1.2583285427696425, + 1.25746144668329, + 1.2567431069600323, + 1.2558671504145162, + 1.254848700705218, + 1.253736238689485, + 1.252625998752515 + ], + "model_azim": [ + 5.966643957880864, + 5.974911280194201, + 5.9800178789208704, + 5.993802064108692, + 6.006378939289549, + 6.017536643092251, + 6.0274870372718645, + 6.03788102220137, + 6.052329074285987, + 6.0630724131073555, + 6.075406678946376, + 6.088856757848179, + 6.102652853784429, + 6.115634949478984 + ], + "model_elev": [ + 1.202730398539852, + 1.2051959385233475, + 1.2067159430592809, + 1.210615868504016, + 1.2140581450105756, + 1.2170252460038622, + 1.2196057067116586, + 1.2222277443684135, + 1.225732770549091, + 1.2282693521279788, + 1.2310859268705145, + 1.234049575685928, + 1.2369803907988324, + 1.239645469881098 + ], + "model_fit1": [ + 5.966643957880864, + 5.974911280194201, + 5.9800178789208704, + 5.993802064108692, + 6.006378939289549, + 6.017536643092251, + 6.0274870372718645, + 6.03788102220137, + 6.052329074285987, + 6.0630724131073555, + 6.075406678946376, + 6.088856757848179, + 6.102652853784429, + 6.115634949478984 + ], + "model_fit2": [ + 1.202730398539852, + 1.2051959385233475, + 1.2067159430592809, + 1.210615868504016, + 1.2140581450105756, + 1.2170252460038622, + 1.2196057067116586, + 1.2222277443684135, + 1.225732770549091, + 1.2282693521279788, + 1.2310859268705145, + 1.234049575685928, + 1.2369803907988324, + 1.239645469881098 + ], + "meas_eci": [ + [ + -0.3035351281043153, + -0.02844121514996133, + 0.9523956758025997 + ], + [ + -0.3029229405820039, + -0.03226575808186291, + 0.9524686939341127 + ], + [ + -0.3042986687014469, + -0.034729084187786134, + 0.9519433864143417 + ], + [ + -0.3026179106160443, + -0.04088405047247195, + 0.9522346846189463 + ], + [ + -0.30295616797952424, + -0.046510880186298745, + 0.9518688451186214 + ], + [ + -0.30220529917399286, + -0.05135608475551451, + 0.9518584504587548 + ], + [ + -0.30152721261512533, + -0.05561523142857777, + 0.9518341694253778 + ], + [ + -0.30167832324719224, + -0.06005383655938584, + 0.9515165400544865 + ], + [ + -0.30143648571506654, + -0.0660779059164778, + 0.9511938579645285 + ], + [ + -0.3005904430157494, + -0.07044880838966484, + 0.9511479122429234 + ], + [ + -0.30091929891322594, + -0.07546789342433696, + 0.9506588097754449 + ], + [ + -0.3001680129514678, + -0.08077893212098153, + 0.9504598508754388 + ], + [ + -0.3000486040229262, + -0.08617384954180235, + 0.9500236328002793 + ], + [ + -0.2988875713280024, + -0.09110370995523792, + 0.9499296467307672 + ] + ], + "meas_eci_los": [ + [ + -0.30353512752019685, + -0.028441221383888878, + 0.9523956758025997 + ], + [ + -0.3029228461071481, + -0.03226664503462707, + 0.9524686939341127 + ], + [ + -0.30429846603725963, + -0.03473085989960501, + 0.9519433864143417 + ], + [ + -0.3026175532804857, + -0.040886695332201205, + 0.9522346846189463 + ], + [ + -0.30295562663884373, + -0.04651440615989257, + 0.9518688451186214 + ], + [ + -0.30220455226634335, + -0.05136047974617341, + 0.9518584504587548 + ], + [ + -0.301526242054227, + -0.05562049322827857, + 0.9518341694253778 + ], + [ + -0.30167710078673465, + -0.06005997721985251, + 0.9515165400544865 + ], + [ + -0.3014349488853584, + -0.06608491628985995, + 0.9511938579645285 + ], + [ + -0.30058859968688656, + -0.07045667302821534, + 0.9511479122429234 + ], + [ + -0.30091710526470994, + -0.07547663979962246, + 0.9506588097754449 + ], + [ + -0.30016543028477377, + -0.08078852849133539, + 0.9504598508754388 + ], + [ + -0.30004559881096554, + -0.08618431269743403, + 0.9500236328002793 + ], + [ + -0.2988841295616315, + -0.09111500073064646, + 0.9499296467307672 + ] + ], + "model_eci": [ + [ + -3432466.7540115756, + -1731741.8935464248, + 5173716.011629781 + ], + [ + -3432372.547769191, + -1732079.5683393998, + 5173489.145111228 + ], + [ + -3432313.743999133, + -1732290.3063399354, + 5173347.5431581 + ], + [ + -3432164.301102949, + -1732825.91596933, + 5172987.668751478 + ], + [ + -3432031.227056555, + -1733302.8168760038, + 5172667.221548597 + ], + [ + -3431915.4844218343, + -1733717.5583187286, + 5172388.519694615 + ], + [ + -3431813.9466015864, + -1734081.3479430394, + 5172144.033616263 + ], + [ + -3431710.0576202665, + -1734453.532469572, + 5171893.892688835 + ], + [ + -3431570.3102787603, + -1734954.2219146248, + 5171557.403865901 + ], + [ + -3431467.812772388, + -1735321.3520204683, + 5171310.628589792 + ], + [ + -3431353.024745032, + -1735732.5059074648, + 5171034.262150875 + ], + [ + -3431230.901985564, + -1736169.9185074358, + 5170740.239346225 + ], + [ + -3431108.549160772, + -1736608.122839488, + 5170445.669753482 + ], + [ + -3430995.6861842214, + -1737012.276212167, + 5170173.961306149 + ] + ], + "meas_lat": [ + 0.9348979840563066, + 0.9348736638383488, + 0.9348366981997408, + 0.9348063479155164, + 0.9347563423139719, + 0.9347257901580164, + 0.9346991840939688, + 0.9346614322637298, + 0.9346157521163507, + 0.9345905006711562, + 0.934546433761751, + 0.9345127904652669, + 0.9344712488826498, + 0.9344455500907388 + ], + "meas_lon": [ + -0.04164301451607974, + -0.04155291066409006, + -0.041513252115616614, + -0.04136276158768624, + -0.04124541609544114, + -0.041134549051738925, + -0.041037524018862015, + -0.040945674933504726, + -0.040817480073928235, + -0.040718389405427956, + -0.040618299375673295, + -0.04050220635063718, + -0.040391477328569035, + -0.04028058930887304 + ], + "meas_ht": [ + 81396.9675099336, + 81244.77789502541, + 81193.34068859647, + 80935.62631653334, + 80752.07851757792, + 80566.58131972801, + 80403.44378474863, + 80257.65942465847, + 80051.33834901436, + 79883.51877213769, + 79727.19354210321, + 79534.42486416693, + 79356.9731703905, + 79168.05194906335 + ], + "meas_range": [ + 87069.72999388228, + 86836.55913972364, + 86692.05673251014, + 86325.53975271186, + 86002.07075931023, + 85723.10812734772, + 85480.18898973276, + 85233.27293127988, + 84903.47168914345, + 84663.79807745767, + 84397.2537452506, + 84115.98487302613, + 83836.60845417317, + 83581.14830424251 + ], + "model_lat": [ + 0.934899505502217, + 0.9348701702014254, + 0.9348518594457105, + 0.9348053124719798, + 0.9347638562520294, + 0.9347277946223416, + 0.934696156319921, + 0.934663781456173, + 0.9346202190594269, + 0.9345882687471724, + 0.9345524799775028, + 0.9345143970336911, + 0.9344762361830826, + 0.9344410323178205 + ], + "model_lon": [ + -0.04164193466936269, + -0.04155538966397436, + -0.041502471430448676, + -0.04136349692543704, + -0.04124007338001663, + -0.04113312415058746, + -0.04103967585574465, + -0.04094400396225524, + -0.04081430089807224, + -0.040719977257233045, + -0.04061399294714445, + -0.0405010622126221, + -0.04038792316631062, + -0.0402838067317805 + ], + "model_ht": [ + 81393.91662581416, + 81251.79438980817, + 81163.08976953034, + 80937.70349190966, + 80737.05100419743, + 80562.56553501167, + 80409.51917280334, + 80252.95499555573, + 80042.40065984003, + 79887.99406011793, + 79715.1061188881, + 79531.20864436727, + 79347.00196171085, + 79177.11246162215 + ], + "model_range": [ + 87069.7307007453, + 86836.56287739017, + 86692.12721183259, + 86325.54008297449, + 86002.08821370563, + 85723.10937370706, + 85480.19184196735, + 85233.27465311192, + 84903.4779387753, + 84663.79964238926, + 84397.26526402768, + 84115.98568912731, + 83836.61634464488, + 83581.15480030599 + ], + "rbeg_lat": 0.934899505502217, + "rbeg_lon": -0.04164193466936269, + "rbeg_ele": 81393.91662581416, + "rbeg_jd": 2461151.500205559, + "rend_lat": 0.9344410323178205, + "rend_lon": -0.0402838067317805, + "rend_ele": 79177.11246162215, + "rend_jd": 2461151.500211552, + "htmin_lat": 0.9344410323178205, + "htmin_lon": -0.0402838067317805, + "htmin_ele": 79177.11246162215, + "htmin_jd": 2461151.500211552, + "absolute_magnitudes": [ + 5.160663993871118, + 4.6264868745522065, + 6.2001017019905555, + 5.2993034793540605, + 3.1674550176961667, + 2.8745104224398865, + 3.270672559130019, + 2.816954128451293, + 2.47537259597214, + 2.6715112232095644, + 3.0883581282452037, + 2.4756073077614653, + 3.082831290906207, + 2.529458163572881 + ], + "ra_data_los": [ + 3.235019804614262, + 3.247710223963076, + 3.2552350905836143, + 3.275889533472668, + 3.293938375945835, + 3.3099368393903763, + 3.324005326769998, + 3.3381101374401774, + 3.3574126051387827, + 3.371831855939726, + 3.387344844103591, + 3.404508976039951, + 3.4212998922560343, + 3.4374935606518706 + ], + "dec_data_los": [ + 1.2609999783504147, + 1.2612395779929737, + 1.2595198256953215, + 1.2604723389631833, + 1.2592765366352765, + 1.2592426249824373, + 1.2591634241814453, + 1.2581291561405523, + 1.2570818128175514, + 1.2569329596745575, + 1.2553525946153052, + 1.2547119136331992, + 1.2533115917530042, + 1.2530106638907785 + ], + "x_eci": [ + -0.3035351281043153, + -0.3029229405820039, + -0.3042986687014469, + -0.3026179106160443, + -0.30295616797952424, + -0.30220529917399286, + -0.30152721261512533, + -0.30167832324719224, + -0.30143648571506654, + -0.3005904430157494, + -0.30091929891322594, + -0.3001680129514678, + -0.3000486040229262, + -0.2988875713280024 + ], + "y_eci": [ + -0.02844121514996133, + -0.03226575808186291, + -0.034729084187786134, + -0.04088405047247195, + -0.046510880186298745, + -0.05135608475551451, + -0.05561523142857777, + -0.06005383655938584, + -0.0660779059164778, + -0.07044880838966484, + -0.07546789342433696, + -0.08077893212098153, + -0.08617384954180235, + -0.09110370995523792 + ], + "z_eci": [ + 0.9523956758025997, + 0.9524686939341127, + 0.9519433864143417, + 0.9522346846189463, + 0.9518688451186214, + 0.9518584504587548, + 0.9518341694253778, + 0.9515165400544865, + 0.9511938579645285, + 0.9511479122429234, + 0.9506588097754449, + 0.9504598508754388, + 0.9500236328002793, + 0.9499296467307672 + ], + "x_eci_los": [ + -0.30353512752019685, + -0.3029228461071481, + -0.30429846603725963, + -0.3026175532804857, + -0.30295562663884373, + -0.30220455226634335, + -0.301526242054227, + -0.30167710078673465, + -0.3014349488853584, + -0.30058859968688656, + -0.30091710526470994, + -0.30016543028477377, + -0.30004559881096554, + -0.2988841295616315 + ], + "y_eci_los": [ + -0.028441221383888878, + -0.03226664503462707, + -0.03473085989960501, + -0.040886695332201205, + -0.04651440615989257, + -0.05136047974617341, + -0.05562049322827857, + -0.06005997721985251, + -0.06608491628985995, + -0.07045667302821534, + -0.07547663979962246, + -0.08078852849133539, + -0.08618431269743403, + -0.09111500073064646 + ], + "z_eci_los": [ + 0.9523956758025997, + 0.9524686939341127, + 0.9519433864143417, + 0.9522346846189463, + 0.9518688451186214, + 0.9518584504587548, + 0.9518341694253778, + 0.9515165400544865, + 0.9511938579645285, + 0.9511479122429234, + 0.9506588097754449, + 0.9504598508754388, + 0.9500236328002793, + 0.9499296467307672 + ], + "x_stat": -3406048.6136386427, + "y_stat": -1729266.1246223755, + "z_stat": 5090787.796276712, + "stat_eci": [ + -3406048.6136386427, + -1729266.1246223755, + 5090787.796276712 + ], + "stat_eci_los": [ + [ + -3406048.5781234126, + -1729266.1945749386, + 5090787.796276712 + ], + [ + -3406043.5503640417, + -1729276.0974638269, + 5090787.796276712 + ], + [ + -3406038.5225758795, + -1729286.0003380955, + 5090787.796276712 + ], + [ + -3406033.4998326153, + -1729295.8932045705, + 5090787.796276712 + ], + [ + -3406028.4872221127, + -1729305.766042447, + 5090787.796276712 + ], + [ + -3406023.464421443, + -1729315.6588797665, + 5090787.796276712 + ], + [ + -3406018.4365182575, + -1729325.5616956379, + 5090787.796276712 + ], + [ + -3406013.413660088, + -1729335.454503771, + 5090787.796276712 + ], + [ + -3406008.39584702, + -1729345.3373042115, + 5090787.796276712 + ], + [ + -3406003.3678716016, + -1729355.24004859, + 5090787.796276712 + ], + [ + -3405998.350001125, + -1729365.1228198803, + 5090787.796276712 + ], + [ + -3405993.3270280496, + -1729375.0155696734, + 5090787.796276712 + ], + [ + -3405988.309100192, + -1729384.8983118308, + 5090787.796276712 + ], + [ + -3405983.286083734, + -1729394.791004777, + 5090787.796276712 + ] + ], + "plane_N": [ + -0.9508405323585105, + -0.0543339152478599, + -0.30487720098102244 + ], + "excluded_time": null, + "excluded_indx_range": [], + "ang_res": [ + 0.00012742337477004607, + 0.0002934025890114486, + 0.001275134678871608, + 8.747317337504222e-05, + 0.000637107786152886, + 0.00017052484520883212, + 0.00025833001748669024, + 0.00020100450053262754, + 0.0003836891026201303, + 0.00019227104764286688, + 0.0005224611490777694, + 0.00013929892102162514, + 0.00043386000256940454, + 0.00039426294165447807 + ], + "ang_res_std": 0.0004704327154024583 + } + } + ], + "los_mini_status": true, + "t_ref_station": 2, + "time_diffs_final": [ + 0.0, + -0.0019011817811903332, + -0.00014963188985710543 + ], + "intersection_list": [ + { + "PlaneIntersection.UK002Z_UK00AS": { + "conv_angle": 1.3977829287229966, + "radiant_eci": [ + -0.2119721838741445, + 0.805179044558886, + 0.5538542222163385 + ], + "radiant_eq": [ + 1.8282164010696587, + 0.5869862134998625 + ], + "w1": [ + -0.9772483431518503, + -0.17887979587700067, + -0.1139635662567909 + ], + "w2": [ + -0.16103171664643293, + -0.5877633547814152, + 0.792844893412324 + ], + "rcpa_stat1": [ + -73631.49434216082, + -13477.82963291005, + -8586.668622006024 + ], + "rcpa_stat2": [ + -13242.80545538603, + -48336.04163995439, + 65201.38329524699 + ], + "cpa_eci": [ + -3403324.3337847604, + -1842095.9935247626, + 5097804.319209092 + ], + "weight": 0.0024214538252984794 + } + }, + { + "PlaneIntersection.UK002Z_UK00CD": { + "conv_angle": 1.3408619526894816, + "radiant_eci": [ + -0.2224668519112711, + 0.8032092732613852, + 0.5525969264731806 + ], + "radiant_eq": [ + 1.8409953958825356, + 0.5854769115887319 + ], + "w1": [ + -0.9749128323753357, + -0.18752636752381052, + -0.11991176236393772 + ], + "w2": [ + -0.2150761118164841, + -0.5932692097626505, + 0.7757408786917855 + ], + "rcpa_stat1": [ + -72116.41498877588, + -13871.731802662609, + -8870.132928298488 + ], + "rcpa_stat2": [ + -15374.393477834303, + -42408.9602147798, + 55452.67396327532 + ], + "cpa_eci": [ + -3401809.2544313753, + -1842489.8956945152, + 5097520.854902799 + ], + "weight": 0.002371116245910391 + } + }, + { + "PlaneIntersection.UK00AS_UK00CD": { + "conv_angle": 0.0578872564894713, + "radiant_eci": [ + -0.17933529144848642, + 0.8982410335263312, + 0.4012504192279862 + ], + "radiant_eq": [ + 1.767856898383979, + 0.4128815721461222 + ], + "w1": [ + -0.19672893468820898, + -0.4323680153644721, + 0.8799747868809822 + ], + "w2": [ + -0.25216678471696985, + -0.43620698939268593, + 0.8637912798184956 + ], + "rcpa_stat1": [ + -16494.934626699258, + -36252.32942686249, + 73782.36763061871 + ], + "rcpa_stat2": [ + -19567.098351078548, + -33847.856181592375, + 67026.62662722637 + ], + "cpa_eci": [ + -3430096.3582245708, + -1740671.6590927788, + 5167839.561657299 + ], + "weight": 1.3053179448789485e-05 + } + } + ], + "rbeg_lat": 0.934899505502217, + "rbeg_lon": -0.04164193466936269, + "rbeg_ele": 81393.91662581416, + "rbeg_ele_wgs84": 81445.53494089004, + "rbeg_jd": 2461151.500205559, + "rend_lat": 0.9337921890458436, + "rend_lon": -0.038372180120937446, + "rend_ele": 76055.10224458351, + "rend_ele_wgs84": 76106.4147786973, + "rend_jd": 2461151.5002204534, + "htmin_lat": 0.9337921890458436, + "htmin_lon": -0.038372180120937446, + "htmin_ele": 76055.10224458351, + "htmin_ele_wgs84": 76106.4147786973, + "htmin_jd": 2461151.5002204534, + "state_vect": [ + -3432369.342207372, + -1731768.1585065187, + 5173694.939644102 + ], + "incident_angles": [ + 0.5027492433632643, + 1.2638605479641993, + 0.9659504756451831 + ], + "state_vect_mini": [ + -3432458.2336270753, + -1731772.435690051, + 5173695.492595176 + ], + "radiant_eci_mini": [ + -0.22559573032168215, + 0.8086697491907738, + 0.5432861154165067 + ], + "radiant_eq_mini": [ + 1.842850952886513, + 0.5743463248927236 + ], + "v_init": 12967.718791996132, + "v0z": -4413.997141557832, + "v_avg": 12312.85904311762, + "timing_minimization_successful": true, + "velocity_fit": [ + 12967.718791996132, + -244.37218876758848 + ], + "jacchia_fit": [ + 43.65312744412464, + 2.1924329546040098 + ], + "timing_res": 7.041064582452035e-05, + "timing_stddev": 0.008391105161092927, + "state_vect_avg": [ + -3430549.6763656787, + -1738607.335042225, + 5169100.684310519 + ], + "jd_avg": 2461151.5002130065, + "orbit": { + "Orbit": { + "ra": 1.842850952886513, + "dec": 0.5743463248927236, + "azimuth_apparent": 5.215879750489071, + "elevation_apparent": 0.3473192026194278, + "v_avg": 12312.85904311762, + "v_init": 12967.718791996132, + "v_init_stddev": 10.13578330407507, + "ra_norot": 1.837726586059179, + "dec_norot": 0.5860709340821146, + "azimuth_apparent_norot": 5.227200613194143, + "elevation_apparent_norot": 0.3538606337963674, + "v_avg_norot": 12082.953555046177, + "v_init_norot": 12737.81330392469, + "jd_ref": 2461151.500205559, + "jd_dyn": 2461151.5010063, + "lst_ref": 3.6088563643950953, + "lon_ref": -0.0416338437576483, + "lat_ref": 0.9348968523981452, + "ht_ref": 81381.06170645228, + "ht_ref_wgs84": 81432.6793071907, + "lat_geocentric": 0.9317248461260144, + "zc": 1.2250089673888134, + "zg": 1.6682930856125058, + "v_inf": 12967.718791996132, + "v_g": 6669.855942091, + "ra_g": 1.560157053440776, + "dec_g": 0.21033070922631758, + "L_g": 1.560182664795458, + "B_g": -0.19871170575289931, + "meteor_pos": [ + -129457249.42153406, + -76394902.82501395, + 10345.503483171698 + ], + "v_h": 35477.25261044726, + "v_h_x": 14.598935218756163, + "v_h_y": -32.30740600750524, + "v_h_z": 1.3183551608964306, + "L_h": 1.9952090815491528, + "B_h": -0.037169134413427715, + "la_sun": 0.5331351764388832, + "a": 1.749324945375272, + "e": 0.4367632022782812, + "i": 0.037382200258429005, + "peri": 5.925529697530141, + "node": 3.6728731864642414, + "pi": 3.3152175768147956, + "b": -0.0130840548179203, + "q": 0.9852841804078888, + "Q": 2.513365710342655, + "true_anomaly": 0.35949713851885046, + "eccentric_anomaly": 0.22656761811223144, + "mean_anomaly": 0.12845566895513721, + "last_perihelion": "2026-04-03 17:22:02.404589", + "n": 0.007434915611796666, + "Tj": 4.017379995926152, + "T": 2.313696397548099 + } + }, + "uncertainties": { + "MCUncertainties": { + "ci": 95, + "mc_traj_list": null, + "state_vect_mini": [ + 23.302927645551236, + 37.2070223921984, + 47.8951574781733 + ], + "state_vect_mini_ci": [ + [ + -3432495.617382003, + -3432423.837728241 + ], + [ + -1731807.3913284293, + -1731696.1065867343 + ], + [ + 5173640.759083595, + 5173792.756573728 + ] + ], + "x": 23.302927645551236, + "x_ci": [ + -3432495.617382003, + -3432423.837728241 + ], + "y": 37.2070223921984, + "y_ci": [ + -1731807.3913284293, + -1731696.1065867343 + ], + "z": 47.8951574781733, + "z_ci": [ + 5173640.759083595, + 5173792.756573728 + ], + "vx": 13.858206008974927, + "vx_ci": [ + -2938.2878636842593, + -2861.212929938162 + ], + "vy": 59.54333260289223, + "vy_ci": [ + 10483.746446609672, + 10492.11339262698 + ], + "vz": 57.58213998290366, + "vz_ci": [ + 6945.863075051403, + 7122.386599949237 + ], + "radiant_eci_mini": [ + 0.0016920905173864757, + 0.002356945317379389, + 0.002939081393705436 + ], + "radiant_eci_mini_ci": [ + [ + -0.22617399930035348, + -0.2214846318885415 + ], + [ + 0.8059754297844866, + 0.8128571058966497 + ], + [ + 0.5381818809865024, + 0.5473502583390504 + ] + ], + "rbeg_lon": 2.2088929690296427e-06, + "rbeg_lon_ci": [ + 6.2415409901294865, + 6.241546962406092 + ], + "rbeg_lon_m": 8.38540985943337, + "rbeg_lat": 2.638433710383294e-06, + "rbeg_lat_ci": [ + 0.9348949134986161, + 0.9349036888013773 + ], + "rbeg_lat_m": 16.864870452262917, + "rbeg_ele": 45.558677417463386, + "rbeg_ele_ci": [ + 81325.7017569644, + 81464.2043463111 + ], + "rbeg_ele_wgs84": 45.5588599685256, + "rbeg_ele_wgs84_ci": [ + 81377.31980654877, + 81515.822891308 + ], + "rend_lon": 3.407449543509572e-06, + "rend_lon_ci": [ + 6.244800260516159, + 6.244811360843754 + ], + "rend_lon_m": 12.954725654542509, + "rend_lat": 1.0453520900156793e-06, + "rend_lat_ci": [ + 0.933790446617204, + 0.9337939283069673 + ], + "rend_lat_m": 6.681867643440438, + "rend_ele": 32.15807998869947, + "rend_ele_ci": [ + 76036.61348161136, + 76133.00603886494 + ], + "rend_ele_wgs84": 32.15826496087123, + "rend_ele_wgs84_ci": [ + 76087.92684692098, + 76184.31958180107 + ], + "htmin_lon": 3.407449543509572e-06, + "htmin_lon_ci": [ + 6.244800260516159, + 6.244811360843754 + ], + "htmin_lon_m": 12.954725654542509, + "htmin_lat": 1.0453520900156793e-06, + "htmin_lat_ci": [ + 0.933790446617204, + 0.9337939283069673 + ], + "htmin_lat_m": 6.681867643440438, + "htmin_ele": 32.15807998869947, + "htmin_ele_ci": [ + 76036.61348161136, + 76133.00603886494 + ], + "htmin_ele_wgs84": 32.15826496087123, + "htmin_ele_wgs84_ci": [ + 76087.92684692098, + 76184.31958180107 + ], + "ra": 0.002611503709103583, + "ra_ci": [ + 1.8370325403202352, + 1.8440354813447923 + ], + "dec": 0.003499436816389006, + "dec_ci": [ + 0.5682786814957027, + 0.579194832887812 + ], + "azimuth_apparent": 0.0011488927073448102, + "azimuth_apparent_ci": [ + 5.2149278320957935, + 5.218617334102857 + ], + "elevation_apparent": 0.003985943562293819, + "elevation_apparent_ci": [ + 0.3401376381944182, + 0.35146749322636645 + ], + "v_avg": 25.363986414427266, + "v_avg_ci": [ + 12263.698119250277, + 12331.272064862434 + ], + "v_init": 35.83555406641995, + "v_init_ci": [ + 12906.156161188343, + 13012.85741906931 + ], + "lon_ref": 9.746851701222101e-06, + "lon_ref_ci": [ + 6.241530852167341, + 6.241562416968685 + ], + "lat_ref": 4.037368389248857e-06, + "lat_ref_ci": [ + 0.9348930977264116, + 0.9349048456367035 + ], + "ht_ref": 46.23031409446649, + "ht_ref_ci": [ + 81327.7139370313, + 81465.6113865276 + ], + "lat_geocentric": 4.058469875306498e-06, + "lat_geocentric_ci": [ + 0.9317210791689499, + 0.9317328894806066 + ], + "zc": 0.003984415255032236, + "zc_ci": [ + 1.2208682718192776, + 1.2321888843316788 + ], + "zg": 0.010437731100424239, + "zg_ci": [ + 1.6565960535519642, + 1.6867956381526732 + ], + "v_inf": 35.83555406641995, + "v_inf_ci": [ + 12906.156161188343, + 13012.85741906931 + ], + "v_g": 69.86456353616636, + "v_g_ci": [ + 6549.278440215807, + 6757.316401539255 + ], + "ra_g": 0.005218583891039907, + "ra_g_ci": [ + 1.550331619290657, + 1.5642537989451053 + ], + "dec_g": 0.009175375088677046, + "dec_g_ci": [ + 0.19423320526384985, + 0.22168953806563507 + ], + "L_g": 0.005261542789721588, + "L_g_ci": [ + 1.550262999253746, + 1.5642996777101477 + ], + "B_g": 0.009147790639302643, + "B_g_ci": [ + -0.2147525041002339, + -0.18736660983219178 + ], + "meteor_pos": [ + 0.023402287765977765, + 0.04500344767424416, + 0.04074976099956943 + ], + "meteor_pos_ci": [ + [ + -129457249.45909907, + -129457249.3870311 + ], + [ + -76394902.87402098, + -76394902.73235357 + ], + [ + 10345.446462113892, + 10345.578152204636 + ] + ], + "v_h": 85.48609930680463, + "v_h_ci": [ + 35328.09302105182, + 35577.45936832089 + ], + "v_h_x": 0.03321975068903373, + "v_h_x_ci": [ + 14.53657918185901, + 14.62520287469976 + ], + "v_h_y": 0.08104467976059955, + "v_h_y_ci": [ + -32.40782446857918, + -32.16672196346313 + ], + "v_h_z": 0.04568102257541548, + "v_h_z_ci": [ + 1.2602424833462527, + 1.3973518963978855 + ], + "L_h": 0.00021478363911639652, + "L_h_ci": [ + 1.9947075813557262, + 1.9953602674305384 + ], + "B_h": 0.0013798257879346354, + "B_h_ci": [ + -0.03956396089577628, + -0.035430681123816095 + ], + "la_sun": 0.0, + "la_sun_ci": [ + 0.5331351764388832, + 0.5331351764388832 + ], + "a": 0.020764918399534503, + "a_ci": [ + 1.7136415146239252, + 1.7742356355051931 + ], + "e": 0.00662985086508511, + "e_ci": [ + 0.4252246127134539, + 0.44463172993888717 + ], + "i": 0.0013875907380813083, + "i_ci": [ + 0.03563537240700222, + 0.03979064362004738 + ], + "peri": 0.0036974620554850617, + "peri_ci": [ + 5.918617880304961, + 5.928510238183485 + ], + "node": 6.736619195035191e-05, + "node_ci": [ + 3.6727828832139906, + 3.6729846226879324 + ], + "pi": 0.0036316227110077645, + "pi_ci": [ + 3.308403222800056, + 3.3181100489377644 + ], + "b": 0.0006164034149704704, + "b_ci": [ + -0.01417379179358153, + -0.01237568509476685 + ], + "q": 0.0001887248437724549, + "q_ci": [ + 0.9848735253457906, + 0.9853733247776466 + ], + "Q": 0.04135376621516677, + "Q_ci": [ + 2.4423263887544406, + 2.563117103921014 + ], + "true_anomaly": 0.003631669498448071, + "true_anomaly_ci": [ + 0.3566046072044634, + 0.3663115643921616 + ], + "eccentric_anomaly": 0.00415997564946805, + "eccentric_anomaly_ci": [ + 0.22255528058867302, + 0.23405984987107875 + ], + "mean_anomaly": 0.0038950653377452164, + "mean_anomaly_ci": [ + 0.12441501198245487, + 0.13542723129154152 + ], + "last_perihelion": 0.21162646186227413, + "last_perihelion_ci": [ + 2461133.838177647, + 2461134.4084006962 + ], + "n": 0.00013353213752772863, + "n_ci": [ + 0.00727888466701047, + 0.007668381602780202 + ], + "T": 0.04112839790029051, + "T_ci": [ + 2.243267062584157, + 2.3632930877217384 + ], + "Tj": 0.03300359258971792, + "Tj_ci": [ + 3.978562778709054, + 4.074873090067962 + ], + "ra_norot": 0.0026819047506464275, + "ra_norot_ci": [ + 1.831751910635052, + 1.8389425768418368 + ], + "dec_norot": 0.003535179504345732, + "dec_norot_ci": [ + 0.5799393752075872, + 0.590966744099576 + ], + "v_avg_norot": 25.73922847929732, + "v_avg_norot_ci": [ + 12033.412162374418, + 12102.036345823391 + ], + "v_init_norot": 36.25000387087065, + "v_init_norot_ci": [ + 12675.549576533595, + 12783.621700030268 + ], + "azimuth_apparent_norot": 0.0011626219913197264, + "azimuth_apparent_norot_ci": [ + 5.226249501183861, + 5.229971656717279 + ], + "elevation_apparent_norot": 0.004037622005774978, + "elevation_apparent_norot_ci": [ + 0.3465823432479639, + 0.35805015466321544 + ], + "ht_ref_wgs84": 46.23066689867216, + "ht_ref_wgs84_ci": [ + 81379.33149963012, + 81517.23000879795 + ] + } + }, + "orbit_cov": [ + [ + 4.839509762694922e-05, + 1.2839048270422571e-06, + 0.001529597070645031, + -2.8134845951441427e-05, + 0.0015235437188076953, + -0.0005796707377483675 + ], + [ + 1.2839048270422571e-06, + 3.92532878805147e-08, + 4.283334419962881e-05, + -7.366720739670027e-07, + 4.318110319692552e-05, + -1.5189142360366112e-05 + ], + [ + 0.001529597070645031, + 4.283334419962881e-05, + 0.04932644359606424, + -0.0008848174909047756, + 0.049355279485603636, + -0.018239993412701383 + ], + [ + -2.8134845951441427e-05, + -7.366720739670027e-07, + -0.0008848174909047754, + 1.640345974762046e-05, + -0.0008803958803679891, + 0.00033761695055935486 + ], + [ + 0.0015235437188076953, + 4.318110319692551e-05, + 0.04935527948560364, + -0.0008803958803679891, + 0.049434753433829214, + -0.018149168163731216 + ], + [ + -0.0005796707377483675, + -1.5189142360366112e-05, + -0.018239993412701383, + 0.00033761695055935486, + -0.018149168163731216, + 0.006957026666261073 + ] + ], + "state_vect_cov": [ + [ + 594.8872861480645, + -256.66487170381055, + -967.7624393923977, + 583.6215942851601, + 26.71726004970638, + -985.1200079805594 + ], + [ + -256.66487170381055, + 1522.6343263519718, + 718.6734107124669, + 359.1627911110367, + 46.990250811313, + -641.6507010433324 + ], + [ + -967.7624393923977, + 718.673410712467, + 2514.098708757615, + -931.2754297849413, + -11.982008124620437, + 2312.506952848055 + ], + [ + 583.6215942851601, + 359.1627911110367, + -931.2754297849413, + 923.3221697191502, + 62.170755663341666, + -1621.7192914483069 + ], + [ + 26.71726004970638, + 46.99025081131299, + -11.982008124620435, + 62.170755663341666, + 7.208948184864825, + -91.24287656294936 + ], + [ + -985.1200079805594, + -641.6507010433324, + 2312.5069528480544, + -1621.719291448307, + -91.24287656294938, + 3635.949624000401 + ] + ], + "phase_1_only": false, + "avg_radiant": [ + -0.21707459130810688, + 0.8045078859660982, + 0.5528523159278415 + ], + "radiant_eq": [ + 1.8343430182556113, + 0.5857833676735065 + ], + "best_conv_inter": { + "PlaneIntersection.UK002Z_UK00AS": { + "conv_angle": 1.3977829287229966, + "radiant_eci": [ + -0.2119721838741445, + 0.805179044558886, + 0.5538542222163385 + ], + "radiant_eq": [ + 1.8282164010696587, + 0.5869862134998625 + ], + "w1": [ + -0.9772483431518503, + -0.17887979587700067, + -0.1139635662567909 + ], + "w2": [ + -0.16103171664643293, + -0.5877633547814152, + 0.792844893412324 + ], + "rcpa_stat1": [ + -73631.49434216082, + -13477.82963291005, + -8586.668622006024 + ], + "rcpa_stat2": [ + -13242.80545538603, + -48336.04163995439, + 65201.38329524699 + ], + "cpa_eci": [ + -3403324.3337847604, + -1842095.9935247626, + 5097804.319209092 + ], + "weight": 0.0024214538252984794 + } + }, + "stations_time_dict": { + "OrderedDict": {} + }, + "stations_time_dict_copy": { + "OrderedDict": {} + }, + "v_init_stddev": 10.13578330407507, + "time_diffs": [ + 0.0, + 0.00026701713965955896, + 6.187651883518428e-05 + ], + "longname": "20260421_000017.759_UK", + "pre_mc_longname": "20260421_000017.759_UK", + "save_date": "2026-04-21 09:33:47.459946+00:00" +} \ No newline at end of file From f90ebaa9b887f0a934bed7ac2448db74089474b8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 22 Apr 2026 23:22:09 +0100 Subject: [PATCH 109/287] bugfix in api tests --- tests/test_apis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_apis.py b/tests/test_apis.py index 4394b5d1..3d98c763 100644 --- a/tests/test_apis.py +++ b/tests/test_apis.py @@ -56,7 +56,7 @@ def test_getMatchPickle(): for chunk in res.iter_content(chunk_size=4096): data = data + chunk jsd = json.loads(data) - assert len(jsd) == 62 + assert len(jsd) == 64 assert len(jsd['observations']) == 5 else: assert False From fc7b2e1887abd9a58f56be260e2ad36ceced1f0d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 12:30:22 +0100 Subject: [PATCH 110/287] bugfix - changes to dailydb not saved --- .../reports/reportOfLatestMatches.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index c8ffdc04..2d873858 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -81,10 +81,15 @@ def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): masterdb.closeTrajDatabase() # iterate over the delete list and update the daily db and new traj list accordingly + print('cleaning up deleted trajectories in dailydb') for testtr in deltrajs: if testtr in newtrajs: - sqlstr = f'update trajectories set status=0 where "traj_file_path={testtr[0]}";' + sqlstr = f'update trajectories set status=0 where traj_file_path="{testtr[0]}"' dailydb.dbhandle.execute(sqlstr) + dailydb.dbhandle.commit() + cur = dailydb.dbhandle.execute(f'select traj_id, status from trajectories where traj_file_path="{testtr[0]}"') + msg = cur.fetchone() + print(f'updated {msg}') newtrajs.pop(newtrajs.index(testtr)) dailydb.closeTrajDatabase() @@ -198,8 +203,12 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): daily_db_dir = sys.argv[2] offset = sys.argv[3] - # arguments dblocation, datadir, days ago, rundate eg 20220524 - findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) - # update the daily database of paired observations - daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') - updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) + flist = glob.glob(os.path.join(cand_db_dir, 'trajectories_*.db')) + if len(flist) == 0: + print('no container databases to process, aborting') + else: + # arguments dblocation, datadir, days ago, rundate eg 20220524 + findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) + # update the daily database of paired observations + daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') + updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) From d611e56c2925cf0b5a333fdde08017ba36d6a4d6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 12:31:06 +0100 Subject: [PATCH 111/287] improvements to process to cleanup deleted trajs --- archive/utils/cleanupDeletedTrajs.sh | 58 ++++++++-------------------- 1 file changed, 16 insertions(+), 42 deletions(-) diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh index ed5fa0d3..bd0836b9 100644 --- a/archive/utils/cleanupDeletedTrajs.sh +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -14,67 +14,41 @@ cd ${DATADIR}/distrib startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") -echo "checking main storage" +echo "checking main storage, website and csvfiles" sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + export moved=0 trajdir=$(dirname $traj) - echo $trajdir - moved=0 - aws s3 ls ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv ${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir/$fname ${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir/$fname --quiet - moved=1 + #echo $trajdir + srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir + trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir + aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet + export moved=1 done - [ $moved == 1 ] && echo moved $trajdir -done -echo "checking website" -sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do - trajdir=$(dirname $traj) yr=${trajdir:13:4} - trajpth=${trajdir:18} - echo $trajdir + trajpth=$(basename $trajdir) webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth - moved=0 aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do aws s3 mv $webloc/$fname $newloc/$fname --quiet - moved=1 + export moved=1 done - [ $moved == 1 ] && echo moved $trajdir -done -echo "checking fullcsv" -sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do - trajdir=$(dirname $traj) - yr=${trajdir:13:4} - trajpth=${trajdir:34:19} - csvname=$(echo $trajpth | sed 's/_/-/g') - echo $csvname + trajpth1=${trajdir:34:19} + csvname=$(echo $trajpth1 | sed 's/_/-/g') csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} - moved=0 aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do aws s3 mv $csvloc/$fname $newloc/$fname --quiet - moved=1 + export moved=1 done - [ $moved == 1 ] && echo moved $trajdir -done - -echo "checking historic fullcsv just in case" -sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do - trajdir=$(dirname $traj) - yr=${trajdir:13:4} - trajpth=$(basename $trajdir) - csvname=$(echo $trajpth | sed 's/_/-/g') - echo $csvname csvloc=$DATADIR/orbits/${yr}/fullcsv/processed - newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} - moved=0 ls -1 $csvloc/ | grep $csvname | while read fname; do aws s3 mv $csvloc/$fname $newloc/$fname --quiet - moved=1 + export moved=1 done - [ $moved == 1 ] && echo moved $trajdir - export trajpth + [ $moved == 1 ] && echo "moved $trajpth" done echo "checking consolidated matches" @@ -83,6 +57,7 @@ trajdir=$(dirname $lasttraj) yr=${trajdir:13:4} trajpth=$(basename $trajdir) matchcsv=$DATADIR/matched/matches-full-${yr}.csv +grep $trajpth $matchcsv if [ $? == 0 ] ; then python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$matchcsv')" python -m converters.toParquet $matchcsv @@ -99,4 +74,3 @@ export AWS_PROFILE=ukmonshared # needed for mariadb connection details echo "checking Mariadb Database" python -c "from maintenance.dataMaintenance import removeDelTrajFromDb;removeDelTrajFromDb()" unset AWS_PROFILE -done \ No newline at end of file From db558a3d9264de5bb568e7a49dc582535e6c6aa7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 13:35:23 +0100 Subject: [PATCH 112/287] after merging, check and remove any duplicate trajectories from the sqlite db and disk --- .../ukmon_pylib/traj/consolidateDistTraj.py | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index 848feeef..05db34c3 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -6,9 +6,26 @@ import datetime from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase +from wmpl.Trajectory.CorrelateRMS import RMSDataHandle +from wmpl.Utils.TrajConversions import jd2Date -def mergeDatabases(srcdir, dbdir, ignore_missing=False, purge_records=False): +def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=False, matchstart=3): + """ + merge container databases into the master database, looking for and cleaning deleted trajectories + + arguments: + srcdir location of container databases + dbdir location of master databases + basedir location of raw data and trajectories + + keyword args: + ignore_missing default false: if true, create master DBs if not present + purge_records default false: if true, purge records from master DBs + matchstart default 3: range of days to scan for duplicate and deleted trajectories + + """ + targdb = os.path.join(dbdir, 'observations.db') if os.path.isfile(targdb) or ignore_missing: obsdb = ObservationsDatabase(dbdir, purge_records=purge_records) @@ -31,8 +48,24 @@ def mergeDatabases(srcdir, dbdir, ignore_missing=False, purge_records=False): print(f'{tstamp} processing {fl}') if trajdb.mergeTrajDatabase(fl): os.remove(fl) + + # get the latest date in the database for use below + cur = trajdb.dbhandle.execute('select max(jdt_ref) from trajectories where status=1') + vals = cur.fetchall() + jdt_end = float(vals[0][0]) trajdb.closeTrajDatabase() + # find and remove duplicates using WMPL's built-in routine to clean up the trajectory DB + # NB: can only be run on the calcserver where trajectory folders are present + + # select range midday on the latest date to matchstart days earlier + dt_end = jd2Date(int(jdt_end) + 1, dt_obj=True, tzinfo=datetime.timezone.utc) + dt_beg = jd2Date(int(jdt_end) + 1 - matchstart, dt_obj=True, tzinfo=datetime.timezone.utc) + event_time_range = [dt_beg, dt_end] + + dh = RMSDataHandle(basedir, dt_range=event_time_range, db_dir=dbdir, output_dir=basedir,mcmode=1, archivemonths=0) + dh.updateTrajectoryDatabase(dt_range=event_time_range) + # should never need to run this part as the candidates are all made in one place targdb = os.path.join(dbdir, 'candidates.db') if os.path.isfile(targdb): @@ -51,8 +84,9 @@ def mergeDatabases(srcdir, dbdir, ignore_missing=False, purge_records=False): if __name__ == '__main__': if len(sys.argv) < 3: - print('usage: consolidateDistTraj folder_containing_srcdbs targ_dbdir') + print('usage: consolidateDistTraj folder_containing_srcdbs targ_dbdir outdir') exit(0) srcdir = sys.argv[1] dbdir = sys.argv[2] - mergeDatabases(srcdir, dbdir) + basedir = os.path.dirname(os.path.normpath(dbdir)) + mergeDatabases(srcdir, dbdir, basedir) From 784c73d0653914060fc581f970f0d05e78a17083 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 16:25:43 +0100 Subject: [PATCH 113/287] Adding version --- archive/README.md | 3 +++ bumpver.toml | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/README.md b/archive/README.md index 9fefc633..7111c2e8 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,5 +1,8 @@ Data Processing and Flows ========================== + +version: 2025.11.1 + This diagram shows the overall flow of data from Cameras to websites and out to the public. ```mermaid diff --git a/bumpver.toml b/bumpver.toml index 9cab7605..9c69ae69 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -13,6 +13,3 @@ push = true "bumpver.toml" = [ 'current_version = "{version}"', ] -".github/workflows/build_usermgmt.yml" = [ - 'tag_name: {version}' -] From dad925b6217463be46cac1eab92d70077e59538e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 16:25:55 +0100 Subject: [PATCH 114/287] bump version 2025.11.1 -> 2026.04.0 --- README.md | 2 +- bumpver.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 226593df..9172894e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2024.04.2 +version: 2026.04.0 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/bumpver.toml b/bumpver.toml index 9c69ae69..891ea996 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2024.04.2" +current_version = "2026.04.0" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true From 3ba5cfb121ebfe8af3398587d93898be0b1be1d9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 16:27:56 +0100 Subject: [PATCH 115/287] update files containing vesion --- bumpver.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bumpver.toml b/bumpver.toml index 891ea996..038c222e 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -13,3 +13,6 @@ push = true "bumpver.toml" = [ 'current_version = "{version}"', ] +"archive/README.md" = [ + "version: {version}" +] From 284de964807d24a1a228c95c917aeace96156fcc Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 16:28:15 +0100 Subject: [PATCH 116/287] bump version 2026.04.0 -> 2026.04.1 --- README.md | 2 +- archive/README.md | 2 +- bumpver.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9172894e..595de7c5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2026.04.0 +version: 2026.04.1 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index 7111c2e8..108c7fbf 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,7 +1,7 @@ Data Processing and Flows ========================== -version: 2025.11.1 +version: 2026.04.1 This diagram shows the overall flow of data from Cameras to websites and out to the public. diff --git a/bumpver.toml b/bumpver.toml index 038c222e..f8ef5f0f 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2026.04.0" +current_version = "2026.04.1" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true From a319dd23a2fda9e3307e97d42e2606f91e48a625 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 17:01:16 +0100 Subject: [PATCH 117/287] unmangling side effets of automerge --- .../ukmon_pylib/reports/reportOfLatestMatches.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index 4e1eabeb..2d873858 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -81,7 +81,6 @@ def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): masterdb.closeTrajDatabase() # iterate over the delete list and update the daily db and new traj list accordingly -<<<<<<< markmac99 print('cleaning up deleted trajectories in dailydb') for testtr in deltrajs: if testtr in newtrajs: @@ -91,12 +90,6 @@ def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): cur = dailydb.dbhandle.execute(f'select traj_id, status from trajectories where traj_file_path="{testtr[0]}"') msg = cur.fetchone() print(f'updated {msg}') -======= - for testtr in deltrajs: - if testtr in newtrajs: - sqlstr = f'update trajectories set status=0 where "traj_file_path={testtr[0]}";' - dailydb.dbhandle.execute(sqlstr) ->>>>>>> master newtrajs.pop(newtrajs.index(testtr)) dailydb.closeTrajDatabase() @@ -210,7 +203,6 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): daily_db_dir = sys.argv[2] offset = sys.argv[3] -<<<<<<< markmac99 flist = glob.glob(os.path.join(cand_db_dir, 'trajectories_*.db')) if len(flist) == 0: print('no container databases to process, aborting') @@ -220,10 +212,3 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): # update the daily database of paired observations daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) -======= - # arguments dblocation, datadir, days ago, rundate eg 20220524 - findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) - # update the daily database of paired observations - daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') - updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) ->>>>>>> master From 89f2aa15d6df053430715ff880605f4868115fad Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 17:03:11 +0100 Subject: [PATCH 118/287] correct usage message --- archive/ukmon_pylib/traj/consolidateDistTraj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index 05db34c3..c1fee16d 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -84,7 +84,7 @@ def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=F if __name__ == '__main__': if len(sys.argv) < 3: - print('usage: consolidateDistTraj folder_containing_srcdbs targ_dbdir outdir') + print('usage: consolidateDistTraj folder_containing_srcdbs targ_dbdir') exit(0) srcdir = sys.argv[1] dbdir = sys.argv[2] From bf376f8866d0856eee5b99329b0ec31cc45eaf72 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 17:45:28 +0100 Subject: [PATCH 119/287] decommission old calcserver and add new batchserver --- archive/terraform/ukmda/batchserver.tf | 47 ++++++++++++++++++++++++++ archive/terraform/ukmda/calcserver.tf | 41 ---------------------- archive/terraform/ukmda/cloudwatch.tf | 2 +- archive/terraform/ukmda/variables.tf | 3 +- 4 files changed, 49 insertions(+), 44 deletions(-) create mode 100644 archive/terraform/ukmda/batchserver.tf diff --git a/archive/terraform/ukmda/batchserver.tf b/archive/terraform/ukmda/batchserver.tf new file mode 100644 index 00000000..7ff5993f --- /dev/null +++ b/archive/terraform/ukmda/batchserver.tf @@ -0,0 +1,47 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +resource "aws_instance" "batchserver" { + ami = "ami-06f0cf249257c89b3" + instance_type = "t4g.micro" + iam_instance_profile = aws_iam_instance_profile.calcserverrole.name + key_name = aws_key_pair.marks_key.key_name + force_destroy = false + + root_block_device { + tags = { + "Name" = "batchservervol" + "billingtag" = "ukmda" + } + volume_size = 50 + volume_type = "gp3" + throughput = 125 + iops = 3000 + encrypted = true + kms_key_id = aws_kms_key.container_key.arn + } + + tags = { + "Name" = "batchserver" + "billingtag" = "ukmda" + "Route53FQDN" = "batchserver.ukmeteors.co.uk" + "DNSRecordType" = "A" + } + primary_network_interface { + network_interface_id = aws_network_interface.batchserver_if.id + } + metadata_options { + http_tokens = "required" + } +} + +resource "aws_network_interface" "batchserver_if" { + subnet_id = aws_subnet.ec2_subnet.id + description = "Primary network interface" + private_ips = [var.batchserverip] + security_groups = [aws_security_group.ec2_secgrp.id] + ipv6_address_list_enabled = false + tags = { + "Name" = "batchserver" + "billingtag" = "ukmda" + } +} diff --git a/archive/terraform/ukmda/calcserver.tf b/archive/terraform/ukmda/calcserver.tf index fe124845..a1aa2a46 100644 --- a/archive/terraform/ukmda/calcserver.tf +++ b/archive/terraform/ukmda/calcserver.tf @@ -1,46 +1,5 @@ # Copyright (C) 2018-2023 Mark McIntyre -resource "aws_instance" "calc_server" { - ami = "ami-0df2d8f6def0bd716" - instance_type = "c8g.2xlarge" - iam_instance_profile = aws_iam_instance_profile.calcserverrole.name - key_name = aws_key_pair.marks_key.key_name - force_destroy = false - tags = { - "Name" = "Calcengine" - "billingtag" = "ukmda" - "Route53FQDN" = "calcengine.ukmeteors.co.uk" - "DNSRecordType" = "A" - } - root_block_device { - tags = { - "Name" = "calcengine" - "billingtag" = "ukmda" - } - volume_size = 120 - } - primary_network_interface { - network_interface_id = aws_network_interface.calcserver_if.id - } - - metadata_options { - http_tokens = "required" - } -} - -# elastic network interface attached to the calc server - -resource "aws_network_interface" "calcserver_if" { - subnet_id = aws_subnet.ec2_subnet.id - description = "Primary network interface" - private_ips = [var.calcserverip] - security_groups = [aws_security_group.ec2_secgrp.id] - ipv6_address_list_enabled = false - tags = { - "Name" = "calcengine" - "billingtag" = "ukmda" - } -} ################################################ # Ubuntu calc server ################################################ diff --git a/archive/terraform/ukmda/cloudwatch.tf b/archive/terraform/ukmda/cloudwatch.tf index 61ab690a..ee336b1d 100644 --- a/archive/terraform/ukmda/cloudwatch.tf +++ b/archive/terraform/ukmda/cloudwatch.tf @@ -21,7 +21,7 @@ resource "aws_cloudwatch_metric_alarm" "calcServerIdle" { "arn:aws:automate:${var.region}:ec2:stop", ] dimensions = { - "InstanceId" = aws_instance.calc_server.id + "InstanceId" = aws_instance.ubuntu_calc_server.id } tags = { "billingtag" = "ukmda" diff --git a/archive/terraform/ukmda/variables.tf b/archive/terraform/ukmda/variables.tf index e7377464..3e42e2a3 100644 --- a/archive/terraform/ukmda/variables.tf +++ b/archive/terraform/ukmda/variables.tf @@ -18,14 +18,13 @@ data "aws_canonical_user_id" "current" {} variable "remote_profile" { default = "default"} variable "remote_account_id" { default = "317976261112" } variable "remote_region" {default = "eu-west-2"} -#variable "eeaccountid" {default = "822069317839" } variable "main_cidr" {default = "172.32.0.0/16" } variable mgmt_cidr { default = "172.32.36.0/22" } variable lambda_cidr { default = "172.32.32.0/22" } variable ec2_cidr { default = "172.32.16.0/20" } -variable calcserverip { default = "172.32.16.136" } variable ubuntu_calcserverip { default = "172.32.16.137" } +variable batchserverip { default = "172.32.16.138" } variable archalias { default = "archive.ukmeteors.co.uk"} From 21eb2be9b6c09f62fc0f9a395ab2ac1eb69e147c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 18:04:25 +0100 Subject: [PATCH 120/287] add calcserver IP to ssm variables and use it in scripts --- archive/analysis/runDistrib.sh | 41 ++++++++------------ archive/analysis/updatePlotsAndDetStatus.sh | 18 +++------ archive/terraform/mjmm/dev_ssm_parameters.tf | 10 +++++ archive/terraform/mjmm/ssm_parameters.tf | 9 ++++- archive/terraform/ukmda/ssm_variables.tf | 10 +++++ archive/utils/makeConfig.sh | 4 +- 6 files changed, 52 insertions(+), 40 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 9350e5db..eb690991 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -66,38 +66,30 @@ execMatchingsh=/tmp/$execdist python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE chmod +x $execMatchingsh -log2cw $NJLOGGRP $NJLOGSTREAM "get server details" runDistrib -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -while [ "$privip" == "" ] ; do - sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" runDistrib - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -done - -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" runDistrib +log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $CALCSERVERIP and run it" runDistrib -scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist +scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib - scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$privip:data/distrib/$execdist + scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist done # push the python code and ECS templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$privip:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$$CALCSERVERIP:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj # now run the script log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execdist" +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execdist" -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib aws ec2 stop-instances --instance-ids $SERVERINSTANCEID @@ -123,7 +115,6 @@ while [ "$stat" -ne 16 ]; do log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) execcons=execconsol.sh execConsolsh=/tmp/$execcons @@ -131,14 +122,14 @@ python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExe chmod +x $execConsolsh log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib -scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$privip:data/distrib/$execcons -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execcons" +scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execcons +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execcons" log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation, copying databases" runDistrib -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$privip:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib # remote temporary files -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib aws ec2 stop-instances --instance-ids $SERVERINSTANCEID diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index beed3337..937f7bb8 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -62,28 +62,20 @@ execrerunsh=/tmp/$execrerun python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh', '$TESTMODE')" chmod +x $execrerunsh -log2cw $NJLOGGRP $NJLOGSTREAM "get server details" updatePlotsAndDetStatus -privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -while [ "$privip" == "" ] ; do - sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "getting IP address" updatePlotsAndDetStatus - privip=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) -done - -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $privip and run it" updatePlotsAndDetStatus +log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $CALCSERVERIP and run it" updatePlotsAndDetStatus -scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun +scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" updatePlotsAndDetStatus - scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$privip:data/distrib/$execrerun + scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun done # push the python and templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$privip:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj # now run the script -ssh -i $SERVERSSHKEY $SERVERUSERID@$privip "data/distrib/$execrerun" +ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execrerun" log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" updatePlotsAndDetStatus aws ec2 stop-instances --instance-ids $SERVERINSTANCEID diff --git a/archive/terraform/mjmm/dev_ssm_parameters.tf b/archive/terraform/mjmm/dev_ssm_parameters.tf index a9216604..819844f6 100644 --- a/archive/terraform/mjmm/dev_ssm_parameters.tf +++ b/archive/terraform/mjmm/dev_ssm_parameters.tf @@ -65,6 +65,15 @@ resource "aws_ssm_parameter" "dev_calcuser" { } } +resource "aws_ssm_parameter" "dev_calcip" { + name = "dev_calcserverip" + type = "String" + value = "172.32.16.137" + tags = { + "billingtag" = "ukmon" + } +} + resource "aws_ssm_parameter" "dev_backupinstance" { name = "dev_backupinstance" type = "String" @@ -127,3 +136,4 @@ resource "aws_ssm_parameter" "dev_batchloggroup" { "billingtag" = "ukmon" } } + diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 73dff36b..1d29cdb2 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -67,6 +67,14 @@ resource "aws_ssm_parameter" "prod_calcuser" { } } +resource "aws_ssm_parameter" "prod_calcip" { + name = "prod_calcserverip" + type = "String" + value = "172.32.16.137" + tags = { + "billingtag" = "ukmon" + } +} resource "aws_ssm_parameter" "prod_backupinstance" { name = "prod_backupinstance" type = "String" @@ -129,4 +137,3 @@ resource "aws_ssm_parameter" "prod_batchloggroup" { "billingtag" = "ukmon" } } - diff --git a/archive/terraform/ukmda/ssm_variables.tf b/archive/terraform/ukmda/ssm_variables.tf index 29fba5ac..46b596a2 100644 --- a/archive/terraform/ukmda/ssm_variables.tf +++ b/archive/terraform/ukmda/ssm_variables.tf @@ -41,3 +41,13 @@ resource "aws_ssm_parameter" "prod_dbuser" { "billingtag" = "ukmon" } } + +resource "aws_ssm_parameter" "prod_calcserverip" { + provider = aws.eu-west-1-prov + name = "prod_calcserverip" + type = "String" + value = var.ubuntu_calcserverip + tags = { + "billingtag" = "ukmon" + } +} diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index a0aa9d56..05ed8295 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -23,6 +23,7 @@ BKPINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_bac SERVERSSHKEY=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_sshkey --query Parameters[0].Value | tr -d '"') NJLOGGRP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_batchloggroup --query Parameters[0].Value | tr -d '"') SERVERUSERID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcuser --query Parameters[0].Value | tr -d '"') +CALCSERVERIP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcserverip --query Parameters[0].Value | tr -d '"') unset AWS_PROFILE # hardcoded @@ -64,6 +65,7 @@ echo "RCODEDIR=${RCODEDIR}" >> ${CFGFILE} echo "DATADIR=${DATADIR}" >> ${CFGFILE} echo "SERVERINSTANCEID=${SERVERINSTANCEID}" >> ${CFGFILE} echo "SERVERUSERID=${SERVERUSERID}" >> ${CFGFILE} +echo "CALCSERVERIP=${CALCSERVERIP}" >> ${CFGFILE} echo "BKPINSTANCEID=${BKPINSTANCEID}" >> ${CFGFILE} echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> ${CFGFILE} echo "RMS_ENV=${RMS_ENV}" >> ${CFGFILE} @@ -85,7 +87,7 @@ echo "export PYLIB TEMPLATES RCODEDIR DATADIR BKPINSTANCEID AWS_DEFAULT_REGION" echo "export RMS_ENV RMS_LOC WMPL_ENV WMPL_LOC" >> ${CFGFILE} echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFILE} echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} -echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID SERVERUSERID NJLOGGRP" >> ${CFGFILE} +echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID SERVERUSERID CALCSERVERIP NJLOGGRP" >> ${CFGFILE} echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib" >> ${CFGFILE} echo "export TESTMODE TESTSUFF" >> ${CFGFILE} From 9159b418df141cd6033d1474e9548f93b5b8caf1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 20:53:44 +0100 Subject: [PATCH 121/287] Update my branch with the latest from master (#436) * Update version (#433) * remove disused code * update livestream to accept normal RMS filenames * search dialog - fix bug and set default date range * small change to explain default dates * improvement in live image search * remove redundant file * update deployment scripts * improvements in routine to convert GMN report to pandas * more analysis of gmn data * Update the terraform a little * remove hardcoded reference to ec2-user * add SERVERUSERID env /ssm variable for calcserver user id * update container for new distrib-processing method * use calcserver SERVERUSERID variable * support for test ECS containers * fix paths in deployment scripts * remove unused file * missed some templates * update dev/test env settings * remove disused file * make sure paths are expanded * fixing various config issues * updates to makeConfig * updates to distributed processing to align with newest wmpl * add a comment to explain what the code does * remove json db * update consolidation to handle sqlite dbs * remove unused function * create data for daily report * remove unused file * update to call new version of reportOfLatestMatches * add test variables * mildly improve doco * put test logs in the right CW log group * handle missing log groups better * extra value in TrajQualityParams * bugfixes revealed in testing * fix issues in reportOfLatestMatches * put container dbs in a sensible place * catch logging errors * tidy up file locations * simplify calcserver start up * sorting out errors in container logging * update container logging info * avoid unnecessary error * fix bugs in log capture process * add MAX_STNS so i can experiement with limits * add logging. Add max_stns to constraints and set to 9999 push logs to s3 copy calcserver logs to ukmonhelper * don't pull or push the databases from S3 during distribution but do back them up after consolidation * catch error if trajpickle can't be found * set rundate more safely move some processes into nightlyJob to make testing easier * tidy up backup file names and housekeep backup folder * move some processes from findAllMatches to nightlyJob to make testing easier * support for testing of createExecReplot * pass testmode to createExecReplot * ensure logs are unique and capture container logs * realigning createFireballPage with prod. Updating gitignore * remove verbosity flag from one tar operation * fixup to handle new style log with datestamps * get privip after restarting calcserver * temporarily don't run rerunFailedLambdas in test * update to work in a test env too * add 'period' to the match summary api * add period to match summary api * re-enabling rerunFailedLambdas * quieten an S3 transfer * include the foldername when reporting on container progress * put daily trajdb in a safe place * be more specific when finding the daily log * only tar the current container logs * save daily obsdb in the right place * rename ec2.tf to calcserver.tf * quieten some s3 transfers * rename ec2.tf in my acct to ukmonhelper_ec2.tf to make it clearer what it is * update SSM params to reflect new serverid * bugfix in distributeCandidates - using wrong path * add newline to getCostMetrics * weird bug in statsToMqtt * bugfix in update_wmpl.sh for traj container * Script to create an AMI for the calcengine * exclude test data when syncing from s3 * writing databases and logs to the wrong target * bad test on istest * typo in serveruserid * adding to server migration notes * purge old rawcsvs and fullcsvs older than 180 days * improved housekeeping of old data * make sure we scoop up data for 31/12 * stop creation of an unused set of files * change default locations for container data * put container logs in the right place * update dockerfile syntax * change locations of test container data and output * remove incorrect comments * Add script to clean up deleted/duplicate trajs * simplify the stats gathering and move to python fix bug in getMatchStats * comment out unused files * update reportOfLatestMatches to exclude deleted trajectories * add scp to requirements * add SQL database maintenance to dataMaintenance * incorrect default for datadir * update call to pymysql.connect * ensure consolidateOutput rerunnable for prior years * upgrading WMPL * update gitignore a bit * bugfix to catch data from 31/12 each year * add paramiko to dependencies for matchPickle to support wmpl upgrade * update getpickle to newest WMPL * bugfix in api tests * bugfix - changes to dailydb not saved * improvements to process to cleanup deleted trajs * after merging, check and remove any duplicate trajectories from the sqlite db and disk * Adding version * bump version 2025.11.1 -> 2026.04.0 * update files containing vesion * bump version 2026.04.0 -> 2026.04.1 * unmangling side effets of automerge * correct usage message * adding install script --- install_or_update.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 install_or_update.sh diff --git a/install_or_update.sh b/install_or_update.sh new file mode 100755 index 00000000..8efa03f0 --- /dev/null +++ b/install_or_update.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre + +if [[ "$1" != "PROD" && "$1" != "DEV" ]] ; then + echo "must provide runtime env of PROD or DEV" + exit -1 +fi +RUNTIME_ENV=$1 +envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +mkdir -p ~/${envname} + +cd $here/archive + +for loc in analysis ukmon_pylib website cronjobs utils share static_content +do + rsync -avz $loc/ ~/${envname}/$loc + chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 +done +DATADIR=~/$envname/data +mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} +mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} +mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} +mkdir -p ~/$envname/logs + From 8f8b4f272e5df944034178eac8203a46d572390a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 20:56:48 +0100 Subject: [PATCH 122/287] changes to ansible scripts --- archive/deployment/dev-vars.yml | 2 +- archive/deployment/prod-vars.yml | 2 +- archive/deployment/pylib.yml | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/archive/deployment/dev-vars.yml b/archive/deployment/dev-vars.yml index 86dfcf57..ab664a7a 100644 --- a/archive/deployment/dev-vars.yml +++ b/archive/deployment/dev-vars.yml @@ -1,3 +1,3 @@ -destdir: /home/ec2-user/dev +destdir: dev env: DEV s3profile: default diff --git a/archive/deployment/prod-vars.yml b/archive/deployment/prod-vars.yml index 20171b91..cf3ee721 100644 --- a/archive/deployment/prod-vars.yml +++ b/archive/deployment/prod-vars.yml @@ -1,3 +1,3 @@ -destdir: /home/ec2-user/prod +destdir: prod env: PROD s3profile: ukmda_admin \ No newline at end of file diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml index 31317339..1d91b924 100644 --- a/archive/deployment/pylib.yml +++ b/archive/deployment/pylib.yml @@ -14,6 +14,12 @@ - name: Ensures {{destdir}}/ukmon_pylib exists file: path={{destdir}}/ukmon_pylib state=directory tags: [dev,prod] + - name: Ensures {{destdir}}/ukmon_pylib/share exists + file: path={{destdir}}/ukmon_pylib/share state=directory + tags: [dev,prod] + - name: Ensures {{destdir}}/ukmon_pylib/share/maps exists + file: path={{destdir}}/ukmon_pylib/share/maps state=directory + tags: [dev,prod] - name: clean local filesystem ansible.builtin.command: 'pyclean {{srcdir}}/ukmon_pylib -v --debris all' delegate_to: localhost From 26cc0963e96b9af69b15847c479ae2eb7629995d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 21:02:58 +0100 Subject: [PATCH 123/287] remove irrelevant file --- using_vault.sh | 49 ------------------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 using_vault.sh diff --git a/using_vault.sh b/using_vault.sh deleted file mode 100644 index b87bd80d..00000000 --- a/using_vault.sh +++ /dev/null @@ -1,49 +0,0 @@ -# testing vault - -# set the default to JSON exports -export VAULT_FORMAT="json" - -# create an approle thats bound to your cidr blocks - vault write auth/approle/role/usermaint \ - secret_id_ttl=10m \ - token_num_uses=10 \ - token_ttl=20m \ - token_max_ttl=30m \ - secret_id_num_uses=40 \ - secret_id_bound_cidrs="192.168.1.0/24" - -# a policy must exist to allow the role to access the secret -echo 'path "secret/test" { capabilities = ["list"]}' > usermaint.hcl -echo 'path "secret/test" { capabilities = ["read"] }' >> usermaint.hcl -vault policy write usermaint usermaint.hcl -vault write auth/approle/role/usermaint policies=usermaint - -# a token must exist for the server to initially authenticate. -# This needs a policy which grants access the approle -# in a corporate environment this step would be done with LDAP or AD authentication - -echo 'path "auth/approle/role" { capabilities = ["list"]}' > server1.hcl -echo 'path "auth/approle/role/usermaint/*" { capabilities = ["read", "update"] }' >> server1.hcl -vault policy write server1 server1.hcl -servertoken=$(vault token create -policy=server1 -format=json | jq -r .auth.client_token) - - -# now on the target server we can use this unprivileged token to retrieve a more privileged one -export VAULT_TOKEN=$servertoken -# get the role ID - this could be statically stored, its not considered Secret -vault_role_id=$(vault read -format=json auth/approle/role/usermaint/role-id | jq -r .data.role_id) - -# then either get a wrapped token and unwrap it to get a secret -# this is more secure because the wrapped token is single-use and expires after 120s -wraptoken=$(vault write -format=json -wrap-ttl=120s -f auth/approle/role/usermaint/secret-id | jq -r .wrap_info.token) -vault_secret_id=$(vault unwrap -format=json $wraptoken | jq -r .data.secret_id) - -# or just straight up get the secret. -vault_secret_id=$(vault write -format=json -force auth/approle/role/usermaint/secret-id | jq -r .data.secret_id) - -# then login to get a token with permission to read the actual secret -vault_token=$(vault write -format=json auth/approle/login role_id=$vault_role_id secret_id=$vault_secret_id | jq -r .auth.client_token) - -# now you can use vault_token in subsequent calls eg -VAULT_TOKEN=$vault_token vault kv get -format=json -mount=secret test - From 6ca87957328cd61a83d75543856e340027e3652b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 23 Apr 2026 22:03:52 +0100 Subject: [PATCH 124/287] Adding SSM variables for the new batchserver in the ukmda account --- archive/terraform/ukmda/dev_ssm_variables.tf | 170 +++++++++++++++++++ archive/terraform/ukmda/ssm_variables.tf | 131 +++++++++++++- archive/terraform/ukmda/variables.tf | 4 + archive/utils/makeConfig.sh | 10 +- 4 files changed, 300 insertions(+), 15 deletions(-) create mode 100644 archive/terraform/ukmda/dev_ssm_variables.tf diff --git a/archive/terraform/ukmda/dev_ssm_variables.tf b/archive/terraform/ukmda/dev_ssm_variables.tf new file mode 100644 index 00000000..fcfaed25 --- /dev/null +++ b/archive/terraform/ukmda/dev_ssm_variables.tf @@ -0,0 +1,170 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +# SSM parameters for use in Lambdas and the batch + +resource "aws_ssm_parameter" "dev_dbhost" { + provider = aws.eu-west-1-prov + name = "dev_dbhost" + type = "String" + value = "3.11.55.160" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_dbname" { + provider = aws.eu-west-1-prov + name = "dev_dbname" + type = "String" + value = "ukmon" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_dbpw" { + provider = aws.eu-west-1-prov + name = "dev_dbpw" + type = "SecureString" + value = "Batch33mdl" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_dbuser" { + provider = aws.eu-west-1-prov + name = "dev_dbuser" + type = "String" + value = "batch" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_websitebucket" { + name = "dev_websitebucket" + type = "String" + value = var.dev_websitebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_sharedbucket" { + name = "dev_sharedbucket" + type = "String" + value = var.dev_sharedbucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_livebucket" { + name = "dev_livebucket" + type = "String" + value = var.dev_livebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_siteurl" { + name = "dev_siteurl" + type = "String" + value = "https://www.ukmeteors.co.uk/dummy/" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_envname" { + name = "dev_envname" + type = "String" + value = "DEV" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_calcinstance" { + name = "dev_calcinstance" + type = "String" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_calcuser" { + name = "dev_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmda" + } +} + + +resource "aws_ssm_parameter" "dev_calcserverip" { + name = "dev_calcserverip" + type = "String" + value = var.ubuntu_calcserverip + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_wmplhome" { + name = "dev_wmplhome" + type = "String" + value = "$HOME/src/wmpldev" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_rmshome" { + name = "dev_rmshome" + type = "String" + value = "$HOME/src/RMS" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_srcdir" { + name = "dev_srcdir" + type = "String" + value = "$HOME/dev" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_caminfo" { + name = "dev_caminfo" + type = "String" + value = "$HOME/dev/data/consolidated/camera-details.csv" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_sshkey" { + name = "dev_sshkey" + type = "String" + value = "$HOME/.ssh/markskey.pem" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "dev_batchloggroup" { + name = "dev_batchloggroup" + type = "String" + value = "/ukmondev/nightlyjob" + tags = { + "billingtag" = "ukmda" + } +} diff --git a/archive/terraform/ukmda/ssm_variables.tf b/archive/terraform/ukmda/ssm_variables.tf index 46b596a2..11b3fdaa 100644 --- a/archive/terraform/ukmda/ssm_variables.tf +++ b/archive/terraform/ukmda/ssm_variables.tf @@ -1,6 +1,6 @@ # Copyright (C) 2018-2023 Mark McIntyre -# SSM parameters for use in Lambdas +# SSM parameters for use in Lambdas and the batch resource "aws_ssm_parameter" "prod_dbhost" { provider = aws.eu-west-1-prov @@ -8,7 +8,7 @@ resource "aws_ssm_parameter" "prod_dbhost" { type = "String" value = "3.11.55.160" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } @@ -18,7 +18,7 @@ resource "aws_ssm_parameter" "prod_dbname" { type = "String" value = "ukmon" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } @@ -28,7 +28,7 @@ resource "aws_ssm_parameter" "prod_dbpw" { type = "SecureString" value = "Batch33mdl" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } @@ -38,16 +38,133 @@ resource "aws_ssm_parameter" "prod_dbuser" { type = "String" value = "batch" tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" } } +resource "aws_ssm_parameter" "prod_websitebucket" { + name = "prod_websitebucket" + type = "String" + value = var.websitebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_sharedbucket" { + name = "prod_sharedbucket" + type = "String" + value = var.sharedbucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_livebucket" { + name = "prod_livebucket" + type = "String" + value = var.livebucket + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_siteurl" { + name = "prod_siteurl" + type = "String" + value = "https://archive.ukmeteors.co.uk" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_envname" { + name = "prod_envname" + type = "String" + value = "PROD" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_calcinstance" { + name = "prod_calcinstance" + type = "String" + value = "i-0ab47af23705beb31" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_calcuser" { + name = "prod_calcuser" + type = "String" + value = "ubuntu" + tags = { + "billingtag" = "ukmda" + } +} + + resource "aws_ssm_parameter" "prod_calcserverip" { - provider = aws.eu-west-1-prov name = "prod_calcserverip" type = "String" value = var.ubuntu_calcserverip tags = { - "billingtag" = "ukmon" + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_wmplhome" { + name = "prod_wmplhome" + type = "String" + value = "$HOME/src/WesternMeteorPyLib" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_rmshome" { + name = "prod_rmshome" + type = "String" + value = "$HOME/src/RMS" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_srcdir" { + name = "prod_srcdir" + type = "String" + value = "$HOME/prod" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_caminfo" { + name = "prod_caminfo" + type = "String" + value = "$HOME/prod/data/consolidated/camera-details.csv" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_sshkey" { + name = "prod_sshkey" + type = "String" + value = "$HOME/.ssh/markskey.pem" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_batchloggroup" { + name = "prod_batchloggroup" + type = "String" + value = "/ukmonbatch/nightlyjob" + tags = { + "billingtag" = "ukmda" } } diff --git a/archive/terraform/ukmda/variables.tf b/archive/terraform/ukmda/variables.tf index 3e42e2a3..01d6aef6 100644 --- a/archive/terraform/ukmda/variables.tf +++ b/archive/terraform/ukmda/variables.tf @@ -8,6 +8,10 @@ variable "sharedbucket" { default = "ukmda-shared" } variable "livebucket" { default = "ukmda-live" } variable "adminbucket" { default = "ukmda-admin" } +variable "dev_websitebucket" { default = "ukmda-website" } +variable "dev_sharedbucket" { default = "ukmda-shared" } +variable "dev_livebucket" { default = "ukmda-live" } + variable "region" { default = "eu-west-2"} variable "liveregion" { default = "eu-west-1"} diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index 05ed8295..c8cc9e14 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -8,7 +8,7 @@ fi RUNTIME_ENV=$1 envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') -export AWS_PROFILE=default +#export AWS_PROFILE=default # read from AWS SSM Parameterset SRC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_srcdir --query Parameters[0].Value | tr -d '"') @@ -19,7 +19,6 @@ UKMONLIVEBUCKET=s3://$(aws ssm get-parameters --region eu-west-2 --names ${envna RMS_LOC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_rmshome --query Parameters[0].Value | tr -d '"') WMPL_LOC=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_wmplhome --query Parameters[0].Value | tr -d '"') SERVERINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcinstance --query Parameters[0].Value | tr -d '"') -BKPINSTANCEID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_backupinstance --query Parameters[0].Value | tr -d '"') SERVERSSHKEY=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_sshkey --query Parameters[0].Value | tr -d '"') NJLOGGRP=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_batchloggroup --query Parameters[0].Value | tr -d '"') SERVERUSERID=$(aws ssm get-parameters --region eu-west-2 --names ${envname}_calcuser --query Parameters[0].Value | tr -d '"') @@ -29,7 +28,6 @@ unset AWS_PROFILE # hardcoded PYLIB=$SRC/ukmon_pylib TEMPLATES=$SRC/website/templates -RCODEDIR=$SRC/R DATADIR=$SRC/data AWS_DEFAULT_REGION=eu-west-2 MATCHSTART=2 @@ -61,12 +59,10 @@ echo "UKMONSHAREDBUCKET=${UKMONSHAREDBUCKET}" >> ${CFGFILE} echo "UKMONLIVEBUCKET=${UKMONLIVEBUCKET}" >> ${CFGFILE} echo "PYLIB=${PYLIB}" >> ${CFGFILE} echo "TEMPLATES=${TEMPLATES}" >> ${CFGFILE} -echo "RCODEDIR=${RCODEDIR}" >> ${CFGFILE} echo "DATADIR=${DATADIR}" >> ${CFGFILE} echo "SERVERINSTANCEID=${SERVERINSTANCEID}" >> ${CFGFILE} echo "SERVERUSERID=${SERVERUSERID}" >> ${CFGFILE} echo "CALCSERVERIP=${CALCSERVERIP}" >> ${CFGFILE} -echo "BKPINSTANCEID=${BKPINSTANCEID}" >> ${CFGFILE} echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> ${CFGFILE} echo "RMS_ENV=${RMS_ENV}" >> ${CFGFILE} echo "WMPL_ENV=${WMPL_ENV}" >> ${CFGFILE} @@ -83,7 +79,7 @@ echo "TESTSUFF=${TESTSUFF}" >> ${CFGFILE} echo "" >> ${CFGFILE} echo "export RUNTIME_ENV SRC SITEURL" >> ${CFGFILE} echo "export WEBSITEBUCKET UKMONSHAREDBUCKET" >> ${CFGFILE} -echo "export PYLIB TEMPLATES RCODEDIR DATADIR BKPINSTANCEID AWS_DEFAULT_REGION" >> ${CFGFILE} +echo "export PYLIB TEMPLATES DATADIR AWS_DEFAULT_REGION" >> ${CFGFILE} echo "export RMS_ENV RMS_LOC WMPL_ENV WMPL_LOC" >> ${CFGFILE} echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFILE} echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} @@ -98,5 +94,3 @@ echo 'function log2cw() { ' >> ${CFGFILE} echo 'aws logs put-log-events --log-group-name $1 --log-stream-name $2 --log-events "[{\"timestamp\": $(date +%s%3N), \"message\": \"$3\"}]" --profile ukmonshared > /dev/null ' >> ${CFGFILE} echo 'logger -s -t $4 RUNTIME $SECONDS $3' >> ${CFGFILE} echo '}' >> ${CFGFILE} - - From 40d76bce9c8cd9e7f58bcf846e2aa64b101793b3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Apr 2026 12:22:06 +0100 Subject: [PATCH 125/287] attempt to add logging to consolidateDistTraj --- archive/ukmon_pylib/traj/consolidateDistTraj.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/archive/ukmon_pylib/traj/consolidateDistTraj.py b/archive/ukmon_pylib/traj/consolidateDistTraj.py index c1fee16d..0202a06f 100644 --- a/archive/ukmon_pylib/traj/consolidateDistTraj.py +++ b/archive/ukmon_pylib/traj/consolidateDistTraj.py @@ -4,11 +4,14 @@ import sys import glob import datetime +import logging from wmpl.Trajectory.CorrelateDB import ObservationsDatabase, TrajectoryDatabase, CandidateDatabase from wmpl.Trajectory.CorrelateRMS import RMSDataHandle from wmpl.Utils.TrajConversions import jd2Date +log = logging.getLogger("consol_disttraj") + def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=False, matchstart=3): """ @@ -63,6 +66,11 @@ def mergeDatabases(srcdir, dbdir, basedir, ignore_missing=False, purge_records=F dt_beg = jd2Date(int(jdt_end) + 1 - matchstart, dt_obj=True, tzinfo=datetime.timezone.utc) event_time_range = [dt_beg, dt_end] + # capture RMSDataHandle log messages to console + log.setLevel(logging.DEBUG) + console_handler = logging.StreamHandler() + log.addHandler(console_handler) + dh = RMSDataHandle(basedir, dt_range=event_time_range, db_dir=dbdir, output_dir=basedir,mcmode=1, archivemonths=0) dh.updateTrajectoryDatabase(dt_range=event_time_range) From f4e9ff0d52176188758d99771b022c8e4df408f6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Apr 2026 13:07:39 +0100 Subject: [PATCH 126/287] remove reference to ee account --- archive/terraform/mjmm/aws.tf | 6 +++--- archive/terraform/mjmm/crossacct-role.tf | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/archive/terraform/mjmm/aws.tf b/archive/terraform/mjmm/aws.tf index e18c7d50..a9871b9e 100644 --- a/archive/terraform/mjmm/aws.tf +++ b/archive/terraform/mjmm/aws.tf @@ -6,8 +6,8 @@ provider "aws" { } provider "aws" { - alias = "eeacct" - region = var.remote_region - profile = "ukmonshared" + profile = var.profile + region = "eu-west-1" + alias = "eu-west-1-prov" } diff --git a/archive/terraform/mjmm/crossacct-role.tf b/archive/terraform/mjmm/crossacct-role.tf index fabcd4ec..858365ad 100644 --- a/archive/terraform/mjmm/crossacct-role.tf +++ b/archive/terraform/mjmm/crossacct-role.tf @@ -1,7 +1,5 @@ # Copyright (C) 2018-2023 Mark McIntyre -data "aws_caller_identity" "eeacct" {provider = aws.eeacct } - # RESTRICT THESE PERMISSIONS!! data "aws_iam_policy" "terraformpol" { name = "AdministratorAccess" From b9b135321b85a446e8a1a6e6bd7496eb7957c4c1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Apr 2026 13:08:03 +0100 Subject: [PATCH 127/287] temporarily add mariadb deets to ssm in mjmm account --- archive/terraform/mjmm/ssm_parameters.tf | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 1d29cdb2..5eb11b0b 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -1,6 +1,45 @@ # Copyright (C) 2018-2023 Mark McIntyre # SSM parameters for use in the environment config +resource "aws_ssm_parameter" "prod_dbhost" { + provider = aws.eu-west-1-prov + name = "prod_dbhost" + type = "String" + value = "3.11.55.160" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbname" { + provider = aws.eu-west-1-prov + name = "prod_dbname" + type = "String" + value = "ukmon" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbpw" { + provider = aws.eu-west-1-prov + name = "prod_dbpw" + type = "SecureString" + value = "Batch33mdl" + tags = { + "billingtag" = "ukmda" + } +} + +resource "aws_ssm_parameter" "prod_dbuser" { + provider = aws.eu-west-1-prov + name = "prod_dbuser" + type = "String" + value = "batch" + tags = { + "billingtag" = "ukmda" + } +} resource "aws_ssm_parameter" "prod_websitebucket" { name = "prod_websitebucket" From a582121d49e6a10b97e2932f5b6961a25e41ca93 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Apr 2026 13:08:44 +0100 Subject: [PATCH 128/287] bugfixes in dataMaintenance and cleanupDeletedTrajs to avoid deleting disk files when trajs share same folder --- .../maintenance/dataMaintenance.py | 49 ++++++++----- archive/utils/cleanupDeletedTrajs.sh | 72 ++++++++++--------- 2 files changed, 67 insertions(+), 54 deletions(-) diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index 911abcb3..a249870b 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -125,18 +125,23 @@ def removeDeletedTraj(csvfile): masterdb = TrajectoryDatabase(db_path=masterdb_path) cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') deltrajs = cur.fetchall() - masterdb.closeTrajDatabase() i=0 for traj in deltrajs: - fldr = os.path.basename(os.path.dirname(traj[0])) - match = [tr for tr in csvdata if fldr in tr] - if len(match) > 0: - for thismtch in match: - print(f'removing {fldr}') - idx = csvdata.index(thismtch) - _ = csvdata.pop(idx) - i += 1 + cur = masterdb.dbhandle.execute(f'select count(traj_id) from trajectories where traj_file_path="{traj[0]}"') + numtraj = int(cur.fetchone()[0]) + if numtraj == 1: + fldr = os.path.basename(os.path.dirname(traj[0])) + match = [tr for tr in csvdata if fldr in tr] + if len(match) > 0: + for thismtch in match: + print(f'removing {fldr}') + idx = csvdata.index(thismtch) + _ = csvdata.pop(idx) + i += 1 + else: + print(f'skipping {traj[0]} as there are {numtraj} with the same path') + masterdb.closeTrajDatabase() print(f'removed {i} trajectories') open(csvfile, 'w').writelines(csvdata) @@ -156,9 +161,10 @@ def getSqlLoginDetails(): password = res['Parameter']['Value'] res = ssm.get_parameter(Name='prod_dbhost') host = res['Parameter']['Value'] - # should really do these too but they won't change often if at all - user = 'batch' - db = 'ukmon' + res = ssm.get_parameter(Name='prod_user') + user = res['Parameter']['Value'] + res = ssm.get_parameter(Name='prod_dbname') + db = res['Parameter']['Value'] return host, user, password, db @@ -173,7 +179,6 @@ def removeDelTrajFromDb(): masterdb = TrajectoryDatabase(db_path=masterdb_path) cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') deltrajs = cur.fetchall() - masterdb.closeTrajDatabase() # get connection to the SQL database host, user, passwd, db = getSqlLoginDetails() @@ -181,14 +186,20 @@ def removeDelTrajFromDb(): count = 0 for traj in deltrajs: - fldr = os.path.basename(os.path.dirname(traj[0])) - sqlstr = f"delete from matches where orbname like '{fldr}%'" - with connection.cursor() as cursor: - cursor.execute(sqlstr) - result = cursor.fetchall() - count += len(result) + cur = masterdb.dbhandle.execute(f'select count(traj_id) from trajectories where traj_file_path="{traj[0]}"') + numtraj = int(cur.fetchone()[0]) + if numtraj == 1: + fldr = os.path.basename(os.path.dirname(traj[0])) + sqlstr = f"delete from matches where orbname like '{fldr}%'" + with connection.cursor() as cursor: + cursor.execute(sqlstr) + result = cursor.fetchall() + count += len(result) + else: + print(f'skipping {traj[0]} as there are {numtraj} with the same path') connection.commit() connection.close() + masterdb.closeTrajDatabase() print(f'cleaned up {count} trajectories') return diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh index bd0836b9..cff47f49 100644 --- a/archive/utils/cleanupDeletedTrajs.sh +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -16,39 +16,43 @@ jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import d echo "checking main storage, website and csvfiles" sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do - export moved=0 - trajdir=$(dirname $traj) - #echo $trajdir - srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir - trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir - aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet - export moved=1 - done + # check if there are two trajectories with the same folder - we don't want to delete the active one + cnt=$(sqlite3 $DATADIR/distrib/trajectories.db "select count(traj_id) from trajectories where traj_file_path='$traj';") + moved=0 + if [ $cnt == 1 ] ; then + trajdir=$(dirname $traj) + srcloc=${UKMONSHAREDBUCKET}/matches/RMSCorrelate/$trajdir + trgloc=${UKMONSHAREDBUCKET}/matches/duplicates/$trajdir + aws s3 ls $srcloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv ${srcloc}/$fname ${trgloc}/$fname --quiet + moved=1 + done + yr=${trajdir:13:4} + trajpth=$(basename $trajdir) + webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth + aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do + aws s3 mv $webloc/$fname $newloc/$fname --quiet + moved=1 + done - yr=${trajdir:13:4} - trajpth=$(basename $trajdir) - webloc=$WEBSITEBUCKET/reports/${yr}/orbits/$trajpth - newloc=${UKMONSHAREDBUCKET}/matches/duplicates/reports/${yr}/orbits/$trajpth - aws s3 ls $webloc/ | awk -F " " '{print $4}' | while read fname; do - aws s3 mv $webloc/$fname $newloc/$fname --quiet - export moved=1 - done - - trajpth1=${trajdir:34:19} - csvname=$(echo $trajpth1 | sed 's/_/-/g') - csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv - newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} - aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do - aws s3 mv $csvloc/$fname $newloc/$fname --quiet - export moved=1 - done - csvloc=$DATADIR/orbits/${yr}/fullcsv/processed - ls -1 $csvloc/ | grep $csvname | while read fname; do - aws s3 mv $csvloc/$fname $newloc/$fname --quiet - export moved=1 - done - [ $moved == 1 ] && echo "moved $trajpth" + trajpth1=${trajdir:34:19} + csvname=$(echo $trajpth1 | sed 's/_/-/g') + csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv + newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} + aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do + aws s3 mv $csvloc/$fname $newloc/$fname --quiet + moved=1 + done + csvloc=$DATADIR/orbits/${yr}/fullcsv/processed + ls -1 $csvloc/ | grep $csvname | while read fname; do + aws s3 mv $csvloc/$fname $newloc/$fname --quiet + moved=1 + done + [ $moved == 1 ] && echo "moved $trajpth" + else + echo skipping $traj + fi done echo "checking consolidated matches" @@ -57,7 +61,7 @@ trajdir=$(dirname $lasttraj) yr=${trajdir:13:4} trajpth=$(basename $trajdir) matchcsv=$DATADIR/matched/matches-full-${yr}.csv -grep $trajpth $matchcsv +grep $trajpth $matchcsv > /dev/null if [ $? == 0 ] ; then python -c "from maintenance.dataMaintenance import removeDeletedTraj;removeDeletedTraj('$matchcsv')" python -m converters.toParquet $matchcsv @@ -70,7 +74,5 @@ if [ $? == 0 ] ; then aws s3 sync $DATADIR/searchidx/ $WEBSITEBUCKET/search/indexes/ --exclude "*" --include "*allevents.csv" --quiet fi -export AWS_PROFILE=ukmonshared # needed for mariadb connection details echo "checking Mariadb Database" python -c "from maintenance.dataMaintenance import removeDelTrajFromDb;removeDelTrajFromDb()" -unset AWS_PROFILE From da410f6a831109260bbb5664488c46c7355c7cd2 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Apr 2026 16:53:47 +0100 Subject: [PATCH 129/287] updating migration instructions --- archive/server_setup/migratingBatchServer.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index 584a4d90..53aa3987 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -66,3 +66,10 @@ diff passwd.orig passwd.mig diff shadow.orig shadow.mig diff group.orig group.mig ``` + +# Other stuff +To install PyQt5 - needs at least 4GB memory so add 3GB swap +``` bash +sudo apt install qtbase5-dev qt5-qmake +pip install pyqt5 +``` \ No newline at end of file From 2010c05bc50d6b35840e5402746d752c7a170ab4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Apr 2026 17:50:35 +0100 Subject: [PATCH 130/287] more work on deduplication for ukmon --- .../maintenance/dataMaintenance.py | 137 ++++++++++-------- archive/utils/cleanupDeletedTrajs.sh | 36 ++--- 2 files changed, 97 insertions(+), 76 deletions(-) diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index a249870b..d67cff95 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -11,8 +11,8 @@ from time import sleep import pandas as pd import datetime - -import pymysql.cursors +import json +import operator from wmpl.Utils.TrajConversions import datetime2JD from wmpl.Trajectory.CorrelateDB import TrajectoryDatabase @@ -106,8 +106,7 @@ def deleteFromCalcServerByMonth(outfname): def removeDeletedTraj(csvfile): """ - Remove deleted trajectories from the consolidated match CSV and Parquet files - and from the search index. + Remove deleted trajectories from the consolidated match CSV, Parquet and search files """ csvdata = open(csvfile, 'r').readlines() @@ -117,8 +116,7 @@ def removeDeletedTraj(csvfile): jdt_end = datetime2JD(dt_end) else: jdt_end = float(csvdata[-1].split(',')[3]) + 2400000.5 - - jdt_beg =jdt_end - 21 + jdt_beg = jdt_end - 21 datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) masterdb_path = os.path.join(datadir, 'distrib') @@ -127,23 +125,34 @@ def removeDeletedTraj(csvfile): deltrajs = cur.fetchall() i=0 + # loop over the deleted trajectories, removing the corresponding one from the CSV file. + # Note that there's no need to update the MariaDB database, as it is populated + # from the CSV file *after* it has been purged for traj in deltrajs: - cur = masterdb.dbhandle.execute(f'select count(traj_id) from trajectories where traj_file_path="{traj[0]}"') - numtraj = int(cur.fetchone()[0]) - if numtraj == 1: + cur = masterdb.dbhandle.execute(f'select jdt_ref, participating_stations from trajectories where traj_file_path="{traj[0]}" and status=0') + thistraj = cur.fetchall() + if len(thistraj) == 1: fldr = os.path.basename(os.path.dirname(traj[0])) + + # find the rows in the CSV file that correspond to the deleted trajectory + # then go through them and compare obs_ids. The one with the same obs_ids + # is the one we want to remove match = [tr for tr in csvdata if fldr in tr] - if len(match) > 0: - for thismtch in match: + obs_ids = thistraj[0][1] + obs_ids_str = ';'.join(json.loads(obs_ids)) + ';' + for thismtch in match: + if thismtch.split(',')[131] == obs_ids_str: print(f'removing {fldr}') idx = csvdata.index(thismtch) _ = csvdata.pop(idx) i += 1 + break else: - print(f'skipping {traj[0]} as there are {numtraj} with the same path') + print(f'skipping {traj[0]} as there are {len(thistraj)} active trajs with the same path') masterdb.closeTrajDatabase() - print(f'removed {i} trajectories') + print(f'removed {i} trajectories from the text file') + # save the CSV file again open(csvfile, 'w').writelines(csvdata) if 'search' not in csvfile: @@ -154,54 +163,64 @@ def removeDeletedTraj(csvfile): return -def getSqlLoginDetails(): - # retrieve password and host from SSM. This allows me to manage them from Terraform - ssm = boto3.client('ssm', region_name='eu-west-1') - res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) - password = res['Parameter']['Value'] - res = ssm.get_parameter(Name='prod_dbhost') - host = res['Parameter']['Value'] - res = ssm.get_parameter(Name='prod_user') - user = res['Parameter']['Value'] - res = ssm.get_parameter(Name='prod_dbname') - db = res['Parameter']['Value'] - return host, user, password, db - - -def removeDelTrajFromDb(): - dt_end = datetime.datetime.now(tz=datetime.timezone.utc) - jdt_end = datetime2JD(dt_end) - jdt_beg =jdt_end - 7 - - # get list of deleted trajectories - datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - masterdb_path = os.path.join(datadir, 'distrib') - masterdb = TrajectoryDatabase(db_path=masterdb_path) - cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') - deltrajs = cur.fetchall() +def removeDeletedTrajCsv(csvloc, targloc): + """ + Remove csv files corresponding to deleted trajectories before they are + consolidated into the annual CSV and Parquet files + """ + srcbucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] - # get connection to the SQL database - host, user, passwd, db = getSqlLoginDetails() - connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + csvfiles = [] + s3 = boto3.client('s3') + print('checking in ', csvloc, 'sending to ', targloc) + res = s3.list_objects(Bucket=srcbucket, Prefix=csvloc) + if 'Contents' in res: + for k in res['Contents']: + csvname = k["Key"] + csv_dt = datetime.datetime.strptime(os.path.basename(k["Key"])[:22], "%Y%m%d-%H%M%S.%f") + csvfiles.append({"name":csvname, "dt":csv_dt}) + csvfiles.sort(key=operator.itemgetter('name')) + jdt_beg = datetime2JD(csvfiles[0]['dt']) + jdt_end = datetime2JD(csvfiles[-1]['dt']) + + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + masterdb_path = os.path.join(datadir, 'distrib') + masterdb = TrajectoryDatabase(db_path=masterdb_path) + cur = masterdb.dbhandle.execute(f'select traj_file_path from trajectories where status=0 and jdt_ref >= {jdt_beg} and jdt_ref <={jdt_end}') + deltrajs = cur.fetchall() + + i=0 + # loop over the deleted trajectories, removing the corresponding CSV file. + for del_traj in deltrajs: + cur = masterdb.dbhandle.execute(f'select jdt_ref, participating_stations from trajectories where traj_file_path="{del_traj[0]}" and status=0') + thistraj = cur.fetchall() + if len(thistraj) == 1: + obs_ids = thistraj[0][1] + obs_ids_str = ';'.join(json.loads(obs_ids)) + ';' + fldr = os.path.basename(os.path.dirname(del_traj[0]))[:19].replace('_', '-') + + matched_csvs = [x for x in csvfiles if fldr in x['name']] + for csv in matched_csvs: + csvdata = s3.get_object(Bucket=srcbucket, Key=csv['name'])['Body'].read() + csv_obs_ids = csvdata.decode('utf-8').split(',')[131].strip() + if csv_obs_ids == obs_ids_str: + # we have a match to within 0.001s and with the same observations + print(f'removing {fldr}') + bare_csv_name = os.path.basename(matched_csvs[0]['name']) + yr = bare_csv_name[:4] + destkey = f"{targloc}/{yr}/{bare_csv_name}" + s3.copy({"Bucket":srcbucket,"Key":csv['name']}, srcbucket, destkey) + s3.delete_object(Bucket=srcbucket, Key=matched_csvs[0]['name']) + i += 1 + break + else: + print(f'skipping {del_traj[0]} as there are {len(thistraj)} inactive trajs with the same path') + masterdb.closeTrajDatabase() + print(f'removed {i} trajectories from the text file') + else: + print('no fullcsv files to process') - count = 0 - for traj in deltrajs: - cur = masterdb.dbhandle.execute(f'select count(traj_id) from trajectories where traj_file_path="{traj[0]}"') - numtraj = int(cur.fetchone()[0]) - if numtraj == 1: - fldr = os.path.basename(os.path.dirname(traj[0])) - sqlstr = f"delete from matches where orbname like '{fldr}%'" - with connection.cursor() as cursor: - cursor.execute(sqlstr) - result = cursor.fetchall() - count += len(result) - else: - print(f'skipping {traj[0]} as there are {numtraj} with the same path') - connection.commit() - connection.close() - masterdb.closeTrajDatabase() - print(f'cleaned up {count} trajectories') - return + return if __name__ == '__main__': diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh index cff47f49..6ffe0a6d 100644 --- a/archive/utils/cleanupDeletedTrajs.sh +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -14,8 +14,9 @@ cd ${DATADIR}/distrib startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") -echo "checking main storage, website and csvfiles" +echo "checking main storage and website" sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do + # check if there are two trajectories with the same folder - we don't want to delete the active one cnt=$(sqlite3 $DATADIR/distrib/trajectories.db "select count(traj_id) from trajectories where traj_file_path='$traj';") moved=0 @@ -35,26 +36,27 @@ sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectorie aws s3 mv $webloc/$fname $newloc/$fname --quiet moved=1 done - - trajpth1=${trajdir:34:19} - csvname=$(echo $trajpth1 | sed 's/_/-/g') - csvloc=${UKMONSHAREDBUCKET}/matches/${yr}/fullcsv - newloc=${UKMONSHAREDBUCKET}/matches/duplicates/csvs/${yr} - aws s3 ls $csvloc/ | awk -F " " '{print $4}' | grep $csvname | while read fname; do - aws s3 mv $csvloc/$fname $newloc/$fname --quiet - moved=1 - done - csvloc=$DATADIR/orbits/${yr}/fullcsv/processed - ls -1 $csvloc/ | grep $csvname | while read fname; do - aws s3 mv $csvloc/$fname $newloc/$fname --quiet - moved=1 - done [ $moved == 1 ] && echo "moved $trajpth" else echo skipping $traj fi done +echo "checking raw fullcsv data files" +yr=${startdt:0:4} +csvloc=matches/${yr}/fullcsv +newloc=matches/duplicates/csvs +python -c "from maintenance.dataMaintenance import removeDeletedTrajCsv;removeDeletedTrajCsv('$csvloc', '$newloc')" + +# cater for year-end +newyr=$(date +%Y) +if [ $newyr != $yr ] ; then + csvloc=matches/${newyr}/fullcsv + newloc=matches/duplicates/csvs + python -c "from maintenance.dataMaintenance import removeDeletedTrajCsv;removeDeletedTrajCsv('$csvloc', '$newloc')" + +fi + echo "checking consolidated matches" lasttraj=$(sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | tail -1) trajdir=$(dirname $lasttraj) @@ -74,5 +76,5 @@ if [ $? == 0 ] ; then aws s3 sync $DATADIR/searchidx/ $WEBSITEBUCKET/search/indexes/ --exclude "*" --include "*allevents.csv" --quiet fi -echo "checking Mariadb Database" -python -c "from maintenance.dataMaintenance import removeDelTrajFromDb;removeDelTrajFromDb()" +# no need to check the SQL database in mariadb, as it is populated from the CSV file that +# we cleaned up above \ No newline at end of file From 6424937c43f6e1ea1ea5f763fd6a50595faf6327 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Apr 2026 21:03:10 +0100 Subject: [PATCH 131/287] remove username-specific stuff --- archive/server_setup/.bashrc | 10 ++++++---- archive/server_setup/.condaon | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index 79e18c87..df588c17 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -42,15 +42,17 @@ export PATH=$PATH:$(dirname $(which node)) # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! -__conda_setup="$('/home/ec2-user/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +__conda_setup="$('~/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" else - if [ -f "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" ]; then - . "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" + if [ -f "~/miniconda3/etc/profile.d/conda.sh" ]; then + . "~/miniconda3/etc/profile.d/conda.sh" else - export PATH="/home/ec2-user/miniconda3/bin:$PATH" + export PATH="~/miniconda3/bin:$PATH" fi fi unset __conda_setup # <<< conda initialize <<< + +export rundate=$(date +%Y%m%d) \ No newline at end of file diff --git a/archive/server_setup/.condaon b/archive/server_setup/.condaon index b69b186c..4242bd99 100644 --- a/archive/server_setup/.condaon +++ b/archive/server_setup/.condaon @@ -1,12 +1,12 @@ # >>> conda initialize >>> -__conda_setup="$('/home/ec2-user/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +__conda_setup="$('~/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" else - if [ -f "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" ]; then - . "/home/ec2-user/miniconda3/etc/profile.d/conda.sh" + if [ -f "~/miniconda3/etc/profile.d/conda.sh" ]; then + . "~/miniconda3/etc/profile.d/conda.sh" else - export PATH="/home/ec2-user/miniconda3/bin:$PATH" + export PATH="~/miniconda3/bin:$PATH" fi fi unset __conda_setup From db3448e72fd8f845198fcc46a5aaf998abd4e22b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Apr 2026 21:03:41 +0100 Subject: [PATCH 132/287] remove the old deployment scripts --- archive/deployment/analysis.yml | 37 -------- archive/deployment/config.yml | 29 ------ archive/deployment/cronjobs.yml | 27 ------ archive/deployment/database.yml | 28 ------ archive/deployment/dev-vars.yml | 3 - archive/deployment/full_deployment.yml | 9 -- archive/deployment/prod-vars.yml | 3 - archive/deployment/pylib.yml | 118 ------------------------- archive/deployment/server_setup.yml | 36 -------- archive/deployment/shwrinfo.yml | 41 --------- archive/deployment/utils.yml | 43 --------- archive/deployment/website.yml | 41 --------- archive/deployment/website_static.yml | 42 --------- 13 files changed, 457 deletions(-) delete mode 100644 archive/deployment/analysis.yml delete mode 100644 archive/deployment/config.yml delete mode 100644 archive/deployment/cronjobs.yml delete mode 100644 archive/deployment/database.yml delete mode 100644 archive/deployment/dev-vars.yml delete mode 100644 archive/deployment/full_deployment.yml delete mode 100644 archive/deployment/prod-vars.yml delete mode 100644 archive/deployment/pylib.yml delete mode 100644 archive/deployment/server_setup.yml delete mode 100644 archive/deployment/shwrinfo.yml delete mode 100644 archive/deployment/utils.yml delete mode 100644 archive/deployment/website.yml delete mode 100644 archive/deployment/website_static.yml diff --git a/archive/deployment/analysis.yml b/archive/deployment/analysis.yml deleted file mode 100644 index f6506869..00000000 --- a/archive/deployment/analysis.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/analysis exists - file: path={{destdir}}/analysis state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/analysis/updatePlotsAndDetStatus.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/consolidateOutput.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/createSearchable.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/findAllMatches.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getBadStations.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getLogData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/getRMSSingleData.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/reportActiveShowers.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/runDistrib.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/showerReport.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/stationReports.sh', dest: '{{destdir}}/analysis', mode: '754', backup: no } - - {src: '{{srcdir}}/analysis/templates/extracsv.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/match_hdr_full.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/UA_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/UO_header.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/templates/ukmon-single.txt', dest: '{{destdir}}/analysis/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/analysis/README.md', dest: '{{destdir}}/analysis', mode: '644', backup: no } diff --git a/archive/deployment/config.yml b/archive/deployment/config.yml deleted file mode 100644 index 88ad1408..00000000 --- a/archive/deployment/config.yml +++ /dev/null @@ -1,29 +0,0 @@ -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: dev-vars.yml - tags: dev - - name: import prod variables - include_vars: prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/utils exists - file: path={{destdir}}/utils state=directory - tags: [dev,prod] - - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/utils/makeConfig.sh', dest: '{{destdir}}/utils/', mode: '754', backup: yes } - - {src: '{{srcdir}}/utils/stopstart-calcengine.sh', dest: '{{destdir}}/utils/', mode: '754', backup: yes } - - {src: '{{srcdir}}/server_setup/.bashrc', dest: '/home/ec2-user', mode: '644', backup: yes } - - {src: '{{srcdir}}/server_setup/.bash_aliases', dest: '/home/ec2-user', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/.condaon', dest: '/home/ec2-user', mode: '644', backup: no } - - - name: create Config file - shell: '{{destdir}}/utils/makeConfig.sh {{env}}' - tags: [dev,prod] diff --git a/archive/deployment/cronjobs.yml b/archive/deployment/cronjobs.yml deleted file mode 100644 index 6a80c1b5..00000000 --- a/archive/deployment/cronjobs.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/Dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/cronjobs exists - file: path={{destdir}}/cronjobs state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/cronjobs/nightlyJob.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/captureBright.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/getImoWSfile.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/mergeNewOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/deleteDuplicateOrbit.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/gatherMonthlyVideos.sh', dest: '{{destdir}}/cronjobs', mode: '754', backup: no } - - {src: '{{srcdir}}/cronjobs/README.md', dest: '{{destdir}}/cronjobs', mode: '644', backup: no } diff --git a/archive/deployment/database.yml b/archive/deployment/database.yml deleted file mode 100644 index 3abf9d78..00000000 --- a/archive/deployment/database.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/database exists - file: path={{destdir}}/database state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - - {src: '{{srcdir}}/database/ddl/create_brightness_table.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/ddl/create_dbs_and_users.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/ddl/create_tables.sql', dest: '{{destdir}}/database/ddl', mode: '644', backup: no } - - {src: '{{srcdir}}/database/dml/load_data.sql', dest: '{{destdir}}/database/dml', mode: '644', backup: no } - - {src: '{{srcdir}}/database/Setup.md', dest: '{{destdir}}/database', mode: '644', backup: no } - - {src: '{{srcdir}}/database/testDB.py', dest: '{{destdir}}/database', mode: '644', backup: no } - - {src: '{{srcdir}}/database/README.md', dest: '{{destdir}}/database', mode: '644', backup: no } diff --git a/archive/deployment/dev-vars.yml b/archive/deployment/dev-vars.yml deleted file mode 100644 index ab664a7a..00000000 --- a/archive/deployment/dev-vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -destdir: dev -env: DEV -s3profile: default diff --git a/archive/deployment/full_deployment.yml b/archive/deployment/full_deployment.yml deleted file mode 100644 index b682b18d..00000000 --- a/archive/deployment/full_deployment.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- import_playbook: cronjobs.yml -- import_playbook: analysis.yml -- import_playbook: website.yml -- import_playbook: database.yml -- import_playbook: pylib.yml -- import_playbook: server_setup.yml -- import_playbook: shwrinfo.yml -- import_playbook: utils.yml diff --git a/archive/deployment/prod-vars.yml b/archive/deployment/prod-vars.yml deleted file mode 100644 index cf3ee721..00000000 --- a/archive/deployment/prod-vars.yml +++ /dev/null @@ -1,3 +0,0 @@ -destdir: prod -env: PROD -s3profile: ukmda_admin \ No newline at end of file diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml deleted file mode 100644 index 1d91b924..00000000 --- a/archive/deployment/pylib.yml +++ /dev/null @@ -1,118 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/ukmon_pylib exists - file: path={{destdir}}/ukmon_pylib state=directory - tags: [dev,prod] - - name: Ensures {{destdir}}/ukmon_pylib/share exists - file: path={{destdir}}/ukmon_pylib/share state=directory - tags: [dev,prod] - - name: Ensures {{destdir}}/ukmon_pylib/share/maps exists - file: path={{destdir}}/ukmon_pylib/share/maps state=directory - tags: [dev,prod] - - name: clean local filesystem - ansible.builtin.command: 'pyclean {{srcdir}}/ukmon_pylib -v --debris all' - delegate_to: localhost - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - - {src: '{{srcdir}}/ukmon_pylib/analysis/compareBrightnessData.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/gatherDetectionData.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/meteorFlux.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/showerAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/stationAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/summaryAnalysis.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/__init__.py', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/analysis/README.md', dest: '{{destdir}}/ukmon_pylib/analysis', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/converters/gmnTxtToPandas.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/toParquet.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/__init__.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/converters/README.md', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/maintenance/compressOldArchiveData.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/cleanDdbTables.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/dataMaintenance.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/deDuplicate.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/getNextBatchStart.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/getUserAndKeyInfo.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/hcVaultActions.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/manageTraj.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/plotStationsOnMap.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/purgeOldImages.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/purgeOldOrbits.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/recreateOrbitPages.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/rerunFailedLambdas.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/sortToDateDirs.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/maintenance/README.md', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/metrics/camMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/costMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/getMatchStats.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/statsToMqtt.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/timingMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/volumeMetrics.py', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/metrics/README.md', dest: '{{destdir}}/ukmon_pylib/metrics', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/reports/CameraDetails.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/cameraStatusReport.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createAnnualBarChart.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createExchangeFiles.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createSearchableFormat.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/createSummaryTable.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/dailyReport.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/extractors.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/filterBylocDirBri.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findBestMp4s.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findFailedMatches.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/findFireballs.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/getLivestreamData.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/getSolutionStati.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/makeCoverageMap.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportActiveShowers.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportBadCameras.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/reportOfLatestMatches.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/trackThumbnails.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/__init__.py', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/reports/README.md', dest: '{{destdir}}/ukmon_pylib/reports', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/requirements.txt', dest: '{{destdir}}/ukmon_pylib', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/README.md', dest: '{{destdir}}/ukmon_pylib', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BM.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMNG_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMUK_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BMWE_hirez.png', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/BM_medres.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/images.json', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/maps/population.jpg', dest: '{{destdir}}/ukmon_pylib/share/maps', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/share/README.md', dest: '{{destdir}}/ukmon_pylib/share', mode: '644', backup: no } - - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails-mm.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetailstest-ukmda.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/clusdetails.txt', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/consolidateDistTraj.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/createDistribMatchingSh.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/distributeCandidates.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/gatherMissedPlatepars.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/manualBulkReruns.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/pickleAnalyser.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/plotOSMGroundTrack.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/ShowerAssociation.py', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/taskrunner_test.json', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } - - {src: '{{srcdir}}/ukmon_pylib/traj/README.md', dest: '{{destdir}}/ukmon_pylib/traj', mode: '644', backup: no } diff --git a/archive/deployment/server_setup.yml b/archive/deployment/server_setup.yml deleted file mode 100644 index f53ea3a7..00000000 --- a/archive/deployment/server_setup.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/server_setup exists - file: path={{destdir}}/server_setup state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/server_setup/copyTestData.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-cartopy.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-geos.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-proj4.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install-sqlite3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/install_opencv.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libgeos-3.7.2.so', dest: '{{destdir}}/server_setu/libsp', mode: '644', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libgeos.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libgeos_c.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - #- {src: '{{srcdir}}/server_setup/libs/libproj.so', dest: '{{destdir}}/server_setup/libs', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/migratingBatchServer.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } - - {src: '{{srcdir}}/server_setup/newserver.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/remount-s3.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/ssh-setup.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/yum-installs.sh', dest: '{{destdir}}/server_setup', mode: '754', backup: no } - - {src: '{{srcdir}}/server_setup/README.md', dest: '{{destdir}}/server_setup', mode: '644', backup: no } diff --git a/archive/deployment/shwrinfo.yml b/archive/deployment/shwrinfo.yml deleted file mode 100644 index c7bbfd11..00000000 --- a/archive/deployment/shwrinfo.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/data/shwrinfo exists - file: path={{destdir}}/data/shwrinfo state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/shwrinfo/AUR.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/CAP.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/COM.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/costnotes.html', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/DAD.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/ETA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/GEM.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/HYD.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/KCG.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/LEO.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/LYR.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/NOO.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/NTA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/ORI.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/PER.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/QUA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/SDA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/STA.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/template.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/shwrinfo/URS.txt', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } - - {src: '{{srcdir}}/README.md', dest: '{{destdir}}/data/shwrinfo', mode: '644', backup: no } diff --git a/archive/deployment/utils.yml b/archive/deployment/utils.yml deleted file mode 100644 index 73dd5f02..00000000 --- a/archive/deployment/utils.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/utils exists - file: path={{destdir}}/utils state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/utils/calcserver/clearSpace.sh', dest: '{{destdir}}/utils/calcserver', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/cftpd_templ.json', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/checkAndRollKeys.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/clearCaches.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/clearSpace.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/createTestDataSet.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/deleteOrbit.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/getCostMetrics.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/liveevent.template', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/loadMatchCsvMDB.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/loadSingleCsvMDB.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/makeConfig.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/monthlyCleardown.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/rerunFTPtoUKMONlambda.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/rerunGetExtraFiles.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/ses-msg-templ.json', dest: '{{destdir}}/utils', mode: '644', backup: no } - - {src: '{{srcdir}}/utils/statsToMqtt.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/stopstart-calcengine.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateDb.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateFireballFlag.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/updateFireballImage.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/userAudit.sh', dest: '{{destdir}}/utils', mode: '754', backup: no } - - {src: '{{srcdir}}/utils/README.md', dest: '{{destdir}}/utils', mode: '644', backup: no } diff --git a/archive/deployment/website.yml b/archive/deployment/website.yml deleted file mode 100644 index 78956dda..00000000 --- a/archive/deployment/website.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/website exists - file: path={{destdir}}/website state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/website/cameraStatusReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/costReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createFireballPage.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createMthlyExtracts.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createOrbitIndex.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createReportIndex.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createShwrExtracts.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/createSummaryTable.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/publishDailyReport.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/pushStatic.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - {src: '{{srcdir}}/website/README.md', dest: '{{destdir}}/website', mode: '644', backup: no } - - {src: '{{srcdir}}/website/updateIndexPages.sh', dest: '{{destdir}}/website', mode: '754', backup: no } - - - {src: '{{srcdir}}/website/templates/coverage-maps.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/fbreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/footer.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/frontpage.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/header.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/reportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/shwrcsvindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } - - {src: '{{srcdir}}/website/templates/statreportindex.html', dest: '{{destdir}}/website/templates', mode: '644', backup: no } diff --git a/archive/deployment/website_static.yml b/archive/deployment/website_static.yml deleted file mode 100644 index c15044f4..00000000 --- a/archive/deployment/website_static.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- hosts: "{{host | default ('ukmonhelper2')}}" - vars_files: - - /mnt/c/Users/{{ lookup('env','USER' )}}/apikeys/mqvariables.enc - vars: - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/archive" - tasks: - - name: import dev variables - include_vars: ./dev-vars.yml - tags: dev - - name: import prod variables - include_vars: ./prod-vars.yml - tags: prod - - name: Ensures {{destdir}}/static_content exists - file: path={{destdir}}/static_content state=directory - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - with_items: - - {src: '{{srcdir}}/static_content/browse/', dest: '{{destdir}}/static_content/browse', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/css/', dest: '{{destdir}}/static_content/css', mode: '754', backup: no, directory_mode: yes } - #- {src: '{{srcdir}}/static_content/data/', dest: '{{destdir}}/static_content/data', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/docs/', dest: '{{destdir}}/static_content/docs', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/fonts/', dest: '{{destdir}}/static_content/fonts', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/img/', dest: '{{destdir}}/static_content/img', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/js/', dest: '{{destdir}}/static_content/js', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/latest/', dest: '{{destdir}}/static_content/latest', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/live/', dest: '{{destdir}}/static_content/live', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/search/', dest: '{{destdir}}/static_content/search', mode: '754', backup: no, directory_mode: yes } - - {src: '{{srcdir}}/static_content/templates/', dest: '{{destdir}}/static_content/templates', mode: '754', backup: no, directory_mode: yes } - - - {src: '{{srcdir}}/static_content/favicon.ico', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/fundraising.html', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/privacy_data.html', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/README.md', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - {src: '{{srcdir}}/static_content/robots.txt', dest: '{{destdir}}/static_content', mode: '644', backup: no } - - - #- name: Update Static Content on the S3 website bucket - # shell: '{{destdir}}/website/pushStatic.sh {{env}}' - # tags: [dev,prod] From a926e33e974a07e9941bce3c1380873f7e204317 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 24 Apr 2026 22:48:32 +0100 Subject: [PATCH 133/287] update install script --- install_or_update.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/install_or_update.sh b/install_or_update.sh index 8efa03f0..c9a35fb1 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -13,15 +13,18 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" mkdir -p ~/${envname} cd $here/archive +git pull +[ -d ~/$envname/data ] && msg="upgrade" || msg="install" for loc in analysis ukmon_pylib website cronjobs utils share static_content do - rsync -avz $loc/ ~/${envname}/$loc + rsync -avz --delete $loc/ ~/${envname}/$loc chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 done DATADIR=~/$envname/data -mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} +mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls,manualuploads} mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs +echo "$msg complete" \ No newline at end of file From 59292b42cf859a0a16756f9681b56347a6ee6b2e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 25 Apr 2026 13:40:50 +0100 Subject: [PATCH 134/287] updates to improve server migration process --- archive/server_setup/.bash_aliases | 15 +++++++++------ archive/server_setup/.bashrc | 12 +++++++----- archive/server_setup/.condaon | 1 + archive/server_setup/.vimrc | 1 + 4 files changed, 18 insertions(+), 11 deletions(-) create mode 100644 archive/server_setup/.vimrc diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index 00d56993..57c10ccb 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -4,6 +4,7 @@ alias h='history' alias df='df -h' alias du='du -h' +alias ls='ls --color=auto' alias data='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/data && pwd ; fi' alias logs='if [ "$SRC" == "" ] ; then echo select env first; else cd $SRC/logs && pwd ; fi' @@ -20,15 +21,17 @@ alias startcalc='$SRC/utils/stopstart-calcengine.sh start' alias stopcalc='$SRC/utils/stopstart-calcengine.sh stop' function dev { - source ~/dev/config.ini >/dev/null - conda activate $HOME/miniconda3/envs/${WMPL_ENV} - PS1="(wmpl) (dev) [\W]\$ " + source ~/dev/config.ini + #conda activate $HOME/miniconda3/envs/${WMPL_ENV} + conda activate ${WMPL_ENV} + PS1="(dev) [\W]\$ " cd ~/dev } function prd { - source ~/prod/config.ini >/dev/null - conda activate $HOME/miniconda3/envs/${WMPL_ENV} - PS1="(wmpl) (prd) [\W]\$ " + source ~/prod/config.ini #>/dev/null + #conda activate $HOME/miniconda3/envs/${WMPL_ENV} + conda activate ${WMPL_ENV} + PS1="(prd) [\W]\$ " cd ~/prod } diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index df588c17..a456684e 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -42,17 +42,19 @@ export PATH=$PATH:$(dirname $(which node)) # >>> conda initialize >>> # !! Contents within this block are managed by 'conda init' !! -__conda_setup="$('~/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" +__conda_setup="$('/home/ubuntu/miniconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" else - if [ -f "~/miniconda3/etc/profile.d/conda.sh" ]; then - . "~/miniconda3/etc/profile.d/conda.sh" + if [ -f "/home/ubuntu/miniconda3/etc/profile.d/conda.sh" ]; then + . "/home/ubuntu/miniconda3/etc/profile.d/conda.sh" else - export PATH="~/miniconda3/bin:$PATH" + export PATH="/home/ubuntu/miniconda3/bin:$PATH" fi fi unset __conda_setup # <<< conda initialize <<< -export rundate=$(date +%Y%m%d) \ No newline at end of file +export rundate=$(date +%Y%m%d) +conda activate wmpl + diff --git a/archive/server_setup/.condaon b/archive/server_setup/.condaon index 4242bd99..a8766953 100644 --- a/archive/server_setup/.condaon +++ b/archive/server_setup/.condaon @@ -10,4 +10,5 @@ else fi fi unset __conda_setup +. ~/miniconda3/etc/profile.d/conda.sh # <<< conda initialize <<< diff --git a/archive/server_setup/.vimrc b/archive/server_setup/.vimrc new file mode 100644 index 00000000..2fd7a2b6 --- /dev/null +++ b/archive/server_setup/.vimrc @@ -0,0 +1 @@ +colo slate From 1a43eed7a5e536161c34da43d3f5c211937f14ba Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 25 Apr 2026 13:47:24 +0100 Subject: [PATCH 135/287] add bashrc to the installer --- install_or_update.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/install_or_update.sh b/install_or_update.sh index c9a35fb1..4242c74d 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -21,10 +21,22 @@ do rsync -avz --delete $loc/ ~/${envname}/$loc chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 done + DATADIR=~/$envname/data mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls,manualuploads} mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs +read -n 1 -p "Update bashrc and aliases? (Y/n) " yesno +if [[ "$yesno" == "n" || "$yesno" == "N" ]] +then + echo skipping bashrc +else + for fil in .bashrc .bash_aliases .vimrc .condaon + do + echo rsync server_setup/$fil ~ + done +fi +yesno= echo "$msg complete" \ No newline at end of file From efc9fbbc143b8910e8b2d1d4f5a44b2aff70e80e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 25 Apr 2026 13:48:52 +0100 Subject: [PATCH 136/287] slight tweak to installer --- install_or_update.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index 4242c74d..2d4df324 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -28,6 +28,8 @@ mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajd mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs +~/$envname/utils/makeConfig.sh $RUNTIME_ENV + read -n 1 -p "Update bashrc and aliases? (Y/n) " yesno if [[ "$yesno" == "n" || "$yesno" == "N" ]] then @@ -38,5 +40,4 @@ else echo rsync server_setup/$fil ~ done fi -yesno= echo "$msg complete" \ No newline at end of file From 66801ae181f72c71309f523cf571e23879681bae Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 25 Apr 2026 13:50:51 +0100 Subject: [PATCH 137/287] add a few installer messages --- install_or_update.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install_or_update.sh b/install_or_update.sh index 2d4df324..f7e4f5c5 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -28,6 +28,7 @@ mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajd mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs +echo "Updating config for $envname" ~/$envname/utils/makeConfig.sh $RUNTIME_ENV read -n 1 -p "Update bashrc and aliases? (Y/n) " yesno From 44b1ae69df1a268e8ecc2ff04579f92b41ba5ad6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 25 Apr 2026 13:53:08 +0100 Subject: [PATCH 138/287] remove an echo --- install_or_update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index f7e4f5c5..e5329ca1 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -38,7 +38,7 @@ then else for fil in .bashrc .bash_aliases .vimrc .condaon do - echo rsync server_setup/$fil ~ + rsync server_setup/$fil ~ done fi echo "$msg complete" \ No newline at end of file From 74923b5a26e0da63fbf6cfcf798d752dfc24af50 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 25 Apr 2026 13:58:24 +0100 Subject: [PATCH 139/287] be less verbose in the rsyncs --- install_or_update.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index e5329ca1..5626cc32 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -18,7 +18,7 @@ git pull [ -d ~/$envname/data ] && msg="upgrade" || msg="install" for loc in analysis ukmon_pylib website cronjobs utils share static_content do - rsync -avz --delete $loc/ ~/${envname}/$loc + rsync -a --delete $loc/ ~/${envname}/$loc chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 done @@ -41,4 +41,5 @@ else rsync server_setup/$fil ~ done fi +echo "" echo "$msg complete" \ No newline at end of file From 8f2ab42cec46d5aca76be2074ea795a045198fc6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 25 Apr 2026 13:59:34 +0100 Subject: [PATCH 140/287] oops need -a for one of the rsyncs --- install_or_update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index 5626cc32..21e403e6 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -38,7 +38,7 @@ then else for fil in .bashrc .bash_aliases .vimrc .condaon do - rsync server_setup/$fil ~ + rsync -a server_setup/$fil ~ done fi echo "" From 25ad5857a10a348ce464ecb31e22fa8458943c4e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 10:45:21 +0100 Subject: [PATCH 141/287] rename requirements file for ukmda --- ...ements.txt => additional_requirements.txt} | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) rename archive/ukmon_pylib/{requirements.txt => additional_requirements.txt} (68%) diff --git a/archive/ukmon_pylib/requirements.txt b/archive/ukmon_pylib/additional_requirements.txt similarity index 68% rename from archive/ukmon_pylib/requirements.txt rename to archive/ukmon_pylib/additional_requirements.txt index 8b9f56c6..4752c6cf 100644 --- a/archive/ukmon_pylib/requirements.txt +++ b/archive/ukmon_pylib/additional_requirements.txt @@ -1,32 +1,19 @@ # Copyright (C) 2018-2023 Mark McIntyre #cartopy INSTALL THIS WITH CONDA -# skip this -#https://github.com/matplotlib/basemap/archive/master.zip -basemap basemap-data-hires boto3 -Cython gmplot google-auth google-auth-oauthlib google-api-python-client googleapis-common-protos -jplephem -Keras -matplotlib==3.3.2 -numpy oauthlib -pandas Pillow pycparser pyephem -pyproj==2.6.1.post1 -PyQt5 +pyproj pyswarms -pytz -PyYAML -scipy Shapely xmltodict tweepy @@ -35,18 +22,17 @@ cryptography python-crontab simplekml astropy -gitpython pymysql mysql-connector-python pre-commit s3fs -# pyarrow +dynamodb_json +requests +paho-mqtt +scp +# testing requirements pytest behave pdoc -dynamodb_json pytest pytest-cov -requests -paho-mqtt -scp \ No newline at end of file From b712fb2d1a096464c354b5339593dd93cbafb6a9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 14:46:25 +0100 Subject: [PATCH 142/287] remove dependencies on MeteorTools --- archive/ukmon_pylib/reports/createAnnualBarChart.py | 2 +- archive/ukmon_pylib/reports/findFireballs.py | 2 +- archive/ukmon_pylib/reports/meteoriteTools.py | 2 +- archive/ukmon_pylib/reports/trackThumbnails.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 9426f5d6..d628fc9a 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt import datetime -from meteortools.utils import jd2Date +from wmpl.Utils.TrajConversions import jd2Date def createBarChart(datadir=None, yr=None): diff --git a/archive/ukmon_pylib/reports/findFireballs.py b/archive/ukmon_pylib/reports/findFireballs.py index d30949fa..178bd124 100644 --- a/archive/ukmon_pylib/reports/findFireballs.py +++ b/archive/ukmon_pylib/reports/findFireballs.py @@ -12,7 +12,7 @@ from shutil import rmtree from traj.pickleAnalyser import getBestView -from meteortools.utils import jd2Date +from wmpl.Utils.TrajConversions import jd2Date # diff --git a/archive/ukmon_pylib/reports/meteoriteTools.py b/archive/ukmon_pylib/reports/meteoriteTools.py index 4f7c72e7..e699bb07 100644 --- a/archive/ukmon_pylib/reports/meteoriteTools.py +++ b/archive/ukmon_pylib/reports/meteoriteTools.py @@ -8,7 +8,7 @@ import numpy as np import requests -from meteortools.utils.Math import greatCircleDistance +from wmpl.Utils.Earth import greatCircleDistance def stationsNearPoint(lat, lon, dist=75, email_only=True): diff --git a/archive/ukmon_pylib/reports/trackThumbnails.py b/archive/ukmon_pylib/reports/trackThumbnails.py index 0e012e1d..65837803 100644 --- a/archive/ukmon_pylib/reports/trackThumbnails.py +++ b/archive/ukmon_pylib/reports/trackThumbnails.py @@ -7,7 +7,7 @@ import argparse import requests from tempfile import mkdtemp -from meteortools.utils import greatCircleDistance +from wmpl.Utils.Earth import greatCircleDistance RAD2DEG=57.2958 From 6e89edc906f8d4e1632a341cb0f86467ba03c634 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 14:51:09 +0100 Subject: [PATCH 143/287] add utils to ukmon_pylib --- archive/server_setup/migratingBatchServer.md | 102 +++++++++++++++---- archive/ukmon_pylib/utils/__init__.py | 1 + archive/ukmon_pylib/utils/getRiseSet.py | 32 ++++++ 3 files changed, 118 insertions(+), 17 deletions(-) create mode 100644 archive/ukmon_pylib/utils/__init__.py create mode 100644 archive/ukmon_pylib/utils/getRiseSet.py diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index 53aa3987..e4b5ed8a 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -1,26 +1,100 @@ # Replacing the UKMON helper server The ukmon helper server provides two services. -* camera authentication and key management * batch processing +* camera authentication and key management + +## how to move batch processing +* Build a new Ubuntu server from the Terraform by cloning the existing batch server details in `terraform/ukmda/batchserver.tf` and making any necessary changes then deploying it with terraform in the normal way. +* I do not recommend using Amazon Linux as this doesn't contain many of the base requirements like C++, development libraries, GEOS and PROJ, so you would need to install and/or build these from source which is tedious. +* If you want to set the hostname to eg `batchserver1`, then do the following + * edit `/etc/cloud/cloud.cfg` and set `preserve_hostname: true` + * run `sudo hostnamctl hostname batchserver1` + * now the new hostname should be preserved through boot + +### Prerequisites +* install some prerequisites via apt +``` bash +sudo apt-get install unzip net-tools dos2unix mariadb-server +``` -# how to move batch processing -The best approach is to reinstall all the code using the ansible deployment scripts. -Both dev and prod envs should be created, though arguably the dev env should be on a separate server. -Additionally, WMPL will need to be reinstalled. -After doing this, the additional UKMON python requirements can be added. +### Miniconda +* Install miniconda with the default settings -The contents of ~/prod/data should be replicated to the new server. -The contents of ~/keymgmt should be replicated to the new server. +### WMPL +* Install WMPL + * create a 'wmpl' conda environment with at least python 3.13 + * install the python requirements + * See note below for how to get PyQt5 working. + * Alternatively you can delete subfolders of wmpl that rely on QT (`CAM0` `MetSIM` and `Utils/DynamicMassFit.py`). +``` bash +conda create -n wmpl python=3.13 +mkdir -p ~/src +cd ~/src +git clone --recursive git@github.com:markmac99/WesternMeteorPyLib.git +cd WesternMeteorPyLib +conda activate wmpl +pip install -r requirements.txt +python setup.py +``` -Finally, double check that all required SSM variables exist in the account holding the server. These -are used to build the config file, but originally were only in Mark McIntyre's account. +### UKMDA Dataprocessing +* Clone the ukmda git repo and then install all the code using the deployment script. Both dev and prod envs should be created, though arguably the dev env should be on a separate server. +``` bash +cd ~/src +git clone git@github.com:ukmda/ukmda-dataprocessing.git +cd ukmda-dataprocessing +conda activate wmpl +pip install -r additional_requirements.txt +./install_or_upgrade.sh PROD +``` +* double check that all required SSM variables exist in the account holding the server. These +are used to build the config file and are deployed via terraform so should be present! For example: +``` bash +aws ssm get-parameters --region eu-west-2 --names prod_siteurl --query Parameters[0].Value +"https://archive.ukmeteors.co.uk" +``` + +## data +Replicate `~/prod/data` to the new server. (once for prod and once for dev). This will have to be repeated every day till golive. +``` bash +cd $DATADIR +rsync -avz ukmonhelper2:prod/data/ . +``` +Replicate the contents of `~/keymgmt` to the new server. Repeat if any new cameras added before golive. +``` bash +rsync -avz ukmonhelper2:keymgmt/ ./keymgmt +``` + +## Mariadb database +TBC + +### To install PyQt5 +* needs at least 3GB memory so add 2GB swap if the server has less. +``` bash +sudo fallocate -l 2G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab + +sudo apt install qtbase5-dev qt5-qmake +pip install pyqt5 +``` + +### To get conda working in batch scripts +This is done by sourcing a file `.condaon` which is automatically installed by the ukmda deployment script and sourced from the environments' config.ini files. Make sure any scripts activate conda like this: + +``` bash +source $HOME/prod/config.ini >/dev/null 2>&1 +conda activate ${WMPL_ENV} +``` ## how to move accounts to a new server The basic process is to extract the user accounts from the current server along with group and password info, then import it back in on the new server. Its important to avoid accidentally overwriting system or otherwise-existing accounts on the new server. -On most systems, accoutns below 500 are system accounts. Check though, as some AWS servers create +On most systems, accounts below 500 are system accounts. Check though, as some AWS servers create new accounts starting at 1000 and working both upwards and downwards! Steps in brief. NB must all be done as root, of course. @@ -67,9 +141,3 @@ diff shadow.orig shadow.mig diff group.orig group.mig ``` -# Other stuff -To install PyQt5 - needs at least 4GB memory so add 3GB swap -``` bash -sudo apt install qtbase5-dev qt5-qmake -pip install pyqt5 -``` \ No newline at end of file diff --git a/archive/ukmon_pylib/utils/__init__.py b/archive/ukmon_pylib/utils/__init__.py new file mode 100644 index 00000000..1d590cf3 --- /dev/null +++ b/archive/ukmon_pylib/utils/__init__.py @@ -0,0 +1 @@ +# Copyright (C) 2018- Mark McIntyre diff --git a/archive/ukmon_pylib/utils/getRiseSet.py b/archive/ukmon_pylib/utils/getRiseSet.py new file mode 100644 index 00000000..081eeed9 --- /dev/null +++ b/archive/ukmon_pylib/utils/getRiseSet.py @@ -0,0 +1,32 @@ +# Copyright (C) 2018- Mark McIntyre +# +import ephem + + +def getNextRiseSet(lati, longi, elev, fordate=None): + """ Calculate the next rise and set times for a given lat, long, elev + + Paramters: + lati: [float] latitude in degrees + longi: [float] longitude in degrees (+E) + elev: [float] latitude in metres + fordate:[datetime] date to calculate for, today if none + + Returns: + rise, set: [date tuple] next rise and set as datetimes + + Note that set may be earlier than rise, if you're invoking the function during daytime. + + """ + obs = ephem.Observer() + obs.lat = float(lati) / 57.3 # convert to radians, close enough for this + obs.lon = float(longi) / 57.3 + obs.elev = float(elev) + obs.horizon = -6.0 / 57.3 # degrees below horizon for darkness + if fordate is not None: + obs.date = fordate + + sun = ephem.Sun() + rise = obs.next_rising(sun).datetime() + set = obs.next_setting(sun).datetime() + return rise, set From 2ea77b582a3b1822ace0551fe1e84fc075903256 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 15:05:13 +0100 Subject: [PATCH 144/287] more removal of dependencies on MeteorTools --- .../maintenance/getNextBatchStart.py | 2 +- .../reports/reportActiveShowers.py | 2 +- archive/ukmon_pylib/traj/manualBulkReruns.py | 5 +- archive/ukmon_pylib/traj/pickleAnalyser.py | 8 +- archive/ukmon_pylib/utils/convertSolLon.py | 41 ++ archive/ukmon_pylib/utils/ftpDetectInfo.py | 506 ++++++++++++++++++ archive/ukmon_pylib/utils/getActiveShowers.py | 48 ++ .../ukmon_pylib/utils/imoWorkingShowerList.py | 230 ++++++++ 8 files changed, 833 insertions(+), 9 deletions(-) create mode 100644 archive/ukmon_pylib/utils/convertSolLon.py create mode 100644 archive/ukmon_pylib/utils/ftpDetectInfo.py create mode 100644 archive/ukmon_pylib/utils/getActiveShowers.py create mode 100644 archive/ukmon_pylib/utils/imoWorkingShowerList.py diff --git a/archive/ukmon_pylib/maintenance/getNextBatchStart.py b/archive/ukmon_pylib/maintenance/getNextBatchStart.py index 2dc8f39a..d89b4030 100644 --- a/archive/ukmon_pylib/maintenance/getNextBatchStart.py +++ b/archive/ukmon_pylib/maintenance/getNextBatchStart.py @@ -6,7 +6,7 @@ import datetime import sys from crontab import CronTab -from meteortools.utils import getNextRiseSet +from utils.getRiseSet import getNextRiseSet offset = 90 diff --git a/archive/ukmon_pylib/reports/reportActiveShowers.py b/archive/ukmon_pylib/reports/reportActiveShowers.py index 7a58d91f..596b7b73 100644 --- a/archive/ukmon_pylib/reports/reportActiveShowers.py +++ b/archive/ukmon_pylib/reports/reportActiveShowers.py @@ -9,7 +9,7 @@ import shutil import argparse -from meteortools.utils import getActiveShowers +from utils.getActiveShowers import getActiveShowers from analysis.showerAnalysis import showerAnalysis from reports.findFireballs import findFireballs diff --git a/archive/ukmon_pylib/traj/manualBulkReruns.py b/archive/ukmon_pylib/traj/manualBulkReruns.py index 497310da..4d391af5 100644 --- a/archive/ukmon_pylib/traj/manualBulkReruns.py +++ b/archive/ukmon_pylib/traj/manualBulkReruns.py @@ -24,9 +24,10 @@ from wmpl.Utils.TrajConversions import geo2Cartesian from wmpl.Utils.TrajConversions import raDec2ECI from wmpl.Utils.Math import vectNorm, angleBetweenVectors, vectorFromPointDirectionAndAngle +from wmpl.Utils.TrajConversions import raDec2AltAz, altAz2RADec, datetime2JD +from wmpl.Utils.Earth import greatCircleDistance -from meteortools.utils import raDec2AltAz, altAz2RADec, datetime2JD, greatCircleDistance -from meteortools.fileformats import loadFTPDetectInfo, writeNewFTPFile +from utils.ftpDetectInfo import loadFTPDetectInfo, writeNewFTPFile # Dummy platepar structure needed by loadPlatePar diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index 5afb6944..028f60fe 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -14,12 +14,10 @@ import datetime import json +from utils.convertSolLon import sollon2jd +from traj.ShowerAssociation import associateShower + from wmpl.Utils.TrajConversions import jd2Date -try: - from meteortools.utils import sollon2jd - from traj.ShowerAssociation import associateShower -except: - pass from wmpl.Utils.Math import mergeClosePoints, angleBetweenSphericalCoords from wmpl.Utils.Physics import calcMass from wmpl.Utils.Pickling import loadPickle diff --git a/archive/ukmon_pylib/utils/convertSolLon.py b/archive/ukmon_pylib/utils/convertSolLon.py new file mode 100644 index 00000000..31e4f762 --- /dev/null +++ b/archive/ukmon_pylib/utils/convertSolLon.py @@ -0,0 +1,41 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +import numpy as np +from wmpl.Utils.TrajConversions import date2JD + + +def sollon2jd(Year, Month, Long): + """ + Calculate the julian date corresponding to a solar longitude. + Because Solar Longitude is relative to the Spring equinox, the exact date + of a given LS varies from year to year. + + Parameters: + Year: [int] year you wish to calculate in. + Month: [int] month you wish to calculate in. + Long: [float] The solar longitude to convert. + + Returns: + [float] julian date + + Notes: + The function is only stable for date ranges from 1900-2100. + """ + + Long = np.radians(Long) + N = Year - 2000 + if abs(N) > 100: + print("Algorithm is not stable for years below 1900 or above 2100") + + JDM0 = 2451182.24736 + 365.25963575 * N + ApproxJD = date2JD(Year, Month, 15, 12, 0, 0) + DiffJD = ApproxJD-2451545 + + Dt = 1.94330 * np.sin(Long - 1.798135) + 0.01305272 * np.sin(2*Long + 2.634232) + 78.195268 + 58.13165 * Long - 0.0000089408 * DiffJD + + if abs(ApproxJD - (JDM0 + Dt))>50: + Dt = Dt + 365.2596 + + JD1 = JDM0 + Dt + + return JD1 diff --git a/archive/ukmon_pylib/utils/ftpDetectInfo.py b/archive/ukmon_pylib/utils/ftpDetectInfo.py new file mode 100644 index 00000000..0e78d130 --- /dev/null +++ b/archive/ukmon_pylib/utils/ftpDetectInfo.py @@ -0,0 +1,506 @@ +# +# Load an FTPdetectInfo file - copied from RMS +# +# Copyright (C) 2018-2023 Mark McIntyre + +import os +import numpy as np +import configparser as crp +import json +import datetime + +from wmpl.Utils.TrajConversions import date2JD +from wmpl.Utils.Math import angleBetweenSphericalCoords + + +def filterFTPforSpecificTime(ftpfile, dtstr): + """ filter FTPdetect file for a specific event, by time, and copy it into a new file + + Arguments: + ftpfile - [string] full path to the ftpdetect file to filter + dtstr - [string] date/time of required event in yyyymmdd_hhmmss format + + Returns: + tuple containing + full name of the new file containing just matching events + the number of matching events + """ + meteor_list = loadFTPDetectInfo(ftpfile, time_offsets=None, join_broken_meteors=True, locdata=None) + refdt = datetime.datetime.strptime(dtstr, '%Y%m%d_%H%M%S') + #print(refdt) + new_met_list = [] + for met in meteor_list: + #print(met.ff_name) + dtpart = datetime.datetime.strptime(met.ff_name[10:25], '%Y%m%d_%H%M%S') + tdiff = (refdt - dtpart).seconds + #print(tdiff) + if abs(tdiff) < 21: + print('adding one entry') + new_met_list.append(met) + newname = writeNewFTPFile(ftpfile, new_met_list) + return newname, len(new_met_list) + + +def writeNewFTPFile(srcname, metlist): + """ creates a FTPDetect file from a list of MeteorObservation objects + + Arguments: + srcname - [string] full path to the original FTP file + metlist - list of MeteorObservation objects + + Returns: + the full path to the created file + + """ + outdir, fname = os.path.split(srcname) + newname = os.path.join(outdir, f'{fname}.old') + try: + os.rename(srcname, newname) + except: + pass + if os.path.isfile(srcname): + srcname = srcname[:-4] + '_new.txt' + with open(srcname, 'w') as ftpf: + _writeFTPHeader(ftpf, len(metlist), outdir, False) + metno = 1 + ffname = '' + for met in metlist: + if ffname == met.ff_name: + metno = metno + 1 + else: + metno = 1 + ffname = met.ff_name + _writeOneMeteor(ftpf, metno, met.station_id, met.time_data, len(met.frames), met.fps, met.frames, + np.degrees(met.ra_data), np.degrees(met.dec_data), + np.degrees(met.azim_data), np.degrees(met.elev_data), + None, met.mag_data, False, met.x_data, met.y_data, met.ff_name) + return srcname + + +def loadFTPDetectInfo(ftpdetectinfo_file_name, time_offsets=None, + join_broken_meteors=True, locdata=None): + """ Loads an FTPDEtect file into a list of MeteorObservation objects + + Arguments: + ftpdetectinfo_file_name: [str] Path to the FTPdetectinfo file. + stations: [dict] A dictionary where the keys are stations IDs, and values are lists of: + - latitude +N in radians + - longitude +E in radians + - height in meters + + + Keyword arguments: + time_offsets: [dict] (key, value) pairs of (stations_id, time_offset) for every station. None by + default. + join_broken_meteors: [bool] Join meteors broken across 2 FF files. + + + Return: + meteor_list: [list] A list of MeteorObservation objects filled with data from the FTPdetectinfo file. + + """ + stations={} + if locdata is None: + dirname, fname = os.path.split(ftpdetectinfo_file_name) + cfgfile = os.path.join(dirname, '.config') + cfg = crp.ConfigParser() + cfg.read(cfgfile) + try: + lat = float(cfg['System']['latitude'].split()[0]) + lon = float(cfg['System']['longitude'].split()[0]) + height = float(cfg['System']['elevation'].split()[0]) + except: + # try reading from platepars file + ppf = os.path.join(dirname, 'platepars_all_recalibrated.json') + if not os.path.isfile(ppf): + return [] + js = json.load(open(ppf, 'r')) + if len(js) < 10: + return [] + lat = js[list(js.keys())[0]]['lat'] + lon = js[list(js.keys())[0]]['lon'] + height = js[list(js.keys())[0]]['elev'] + statid= fname.split('_')[1] + else: + statid = locdata['station_code'] + lat = float(locdata['lat']) + lon = float(locdata['lon']) + height = float(locdata['elev']) + + stations[statid] = [np.radians(lat), np.radians(lon), height*1000] + meteor_list = [] + + with open(ftpdetectinfo_file_name) as f: + # Skip the header + for i in range(11): + next(f) + + current_meteor = None + + bin_name = False + cal_name = False + meteor_header = False + + for line in f: + # Skip comments + if line.startswith("#"): + continue + + line = line.replace('\n', '').replace('\r', '') + + # Skip the line if it is empty + if not line: + continue + + if '-----' in line: + # Mark that the next line is the bin name + bin_name = True + + # If the separator is read in, save the current meteor + if current_meteor is not None: + current_meteor._finish() + meteor_list.append(current_meteor) + continue + + if bin_name: + bin_name = False + + # Mark that the next line is the calibration file name + cal_name = True + + # Save the name of the FF file + ff_name = line + + # Extract the reference time from the FF bin file name + line = line.split('_') + + # Count the number of string segments, and determine if it the old or new CAMS format + if len(line) == 6: + sc = 1 + else: + sc = 0 + + ff_date = line[1 + sc] + ff_time = line[2 + sc] + milliseconds = line[3 + sc] + + year = ff_date[:4] + month = ff_date[4:6] + day = ff_date[6:8] + + hour = ff_time[:2] + minute = ff_time[2:4] + seconds = ff_time[4:6] + + year, month, day, hour, minute, seconds, milliseconds = map(int, [year, month, day, hour, + minute, seconds, milliseconds]) + + # Calculate the reference JD time + jdt_ref = date2JD(year, month, day, hour, minute, seconds, milliseconds) + continue + + if cal_name: + cal_name = False + # Mark that the next line is the meteor header + meteor_header = True + continue + + if meteor_header: + meteor_header = False + line = line.split() + + # Get the station ID and the FPS from the meteor header + station_id = line[0].strip() + fps = float(line[3]) + + # Try converting station ID to integer + try: + station_id = int(station_id) + except: + pass + + # If the time offsets were given, apply the correction to the JD + if time_offsets is not None: + if station_id in time_offsets: + print('Applying time offset for station {:s} of {:.2f} s'.format(str(station_id), + time_offsets[station_id])) + + jdt_ref += time_offsets[station_id]/86400.0 + else: + print('Time offset for given station not found!') + + # Get the station data + if station_id in stations: + lat, lon, height = stations[station_id] + else: + print('ERROR! No info for station ', station_id, ' found in CameraSites.txt file!') + print('Exiting...') + break + # Init a new meteor observation + current_meteor = MeteorObservation(jdt_ref, station_id, lat, lon, height, fps, + ff_name=ff_name) + continue + + # Read in the meteor observation point + if (current_meteor is not None) and (not bin_name) and (not cal_name) and (not meteor_header): + + line = line.replace('\n', '').split() + + # Read in the meteor frame, RA and Dec + frame_n = float(line[0]) + x = float(line[1]) + y = float(line[2]) + ra = float(line[3]) + dec = float(line[4]) + azim = float(line[5]) + elev = float(line[6]) + + # Read the visual magnitude, if present + if len(line) > 8: + mag = line[8] + if mag == 'inf': + mag = None + else: + mag = float(mag) + else: + mag = None + + # Add the measurement point to the current meteor + current_meteor._addPoint(frame_n, x, y, azim, elev, ra, dec, mag) + + # Add the last meteor the the meteor list + if current_meteor is not None: + current_meteor._finish() + meteor_list.append(current_meteor) + + # Concatenate observations across different FF files ### + if join_broken_meteors: + + # Go through all meteors and compare the next observation + merged_indices = [] + for i in range(len(meteor_list)): + + # If the next observation was merged, skip it + if (i + 1) in merged_indices: + continue + + # Get the current meteor observation + met1 = meteor_list[i] + + if i >= (len(meteor_list) - 1): + break + + # Get the next meteor observation + met2 = meteor_list[i + 1] + + # Compare only same station observations + if met1.station_id != met2.station_id: + continue + + # Extract frame number + met1_frame_no = int(met1.ff_name.split("_")[-1].split('.')[0]) + met2_frame_no = int(met2.ff_name.split("_")[-1].split('.')[0]) + + # Skip if the next FF is not exactly 256 frames later + if met2_frame_no != (met1_frame_no + 256): + continue + + + # Check for frame continouty + if (met1.frames[-1] < 254) or (met2.frames[0] > 2): + continue + + # Check if the next frame is close to the predicted position + + # Compute angular distance between the last 2 points on the first FF + ang_dist = angleBetweenSphericalCoords(met1.dec_data[-2], met1.ra_data[-2], met1.dec_data[-1], + met1.ra_data[-1]) + + # Compute frame difference between the last frame on the 1st FF and the first frame on the 2nd FF + df = met2.frames[0] + (256 - met1.frames[-1]) + + # Skip the pair if the angular distance between the last and first frames is 2x larger than the + # frame difference times the expected separation + ang_dist_between = angleBetweenSphericalCoords(met1.dec_data[-1], met1.ra_data[-1], + met2.dec_data[0], met2.ra_data[0]) + + if ang_dist_between > 2*df*ang_dist: + continue + + # If all checks have passed, merge observations ### + # Recompute the frames + frames = 256.0 + met2.frames + + # Recompute the time data + time_data = frames/met1.fps + + # Add the observations to first meteor object + met1.frames = np.append(met1.frames, frames) + met1.time_data = np.append(met1.time_data, time_data) + met1.x_data = np.append(met1.x_data, met2.x_data) + met1.y_data = np.append(met1.y_data, met2.y_data) + met1.azim_data = np.append(met1.azim_data, met2.azim_data) + met1.elev_data = np.append(met1.elev_data, met2.elev_data) + met1.ra_data = np.append(met1.ra_data, met2.ra_data) + met1.dec_data = np.append(met1.dec_data, met2.dec_data) + met1.mag_data = np.append(met1.mag_data, met2.mag_data) + + # Merge the FF file name and create a list + if (met1.ff_name is not None) and (met2.ff_name is not None): + met1.ff_name = met1.ff_name + ',' + met2.ff_name + + # Sort all observations by time + met1._finish() + + # Indicate that the next observation is to be skipped + merged_indices.append(i + 1) + + # Removed merged meteors from the list + meteor_list = [element for i, element in enumerate(meteor_list) if i not in merged_indices] + + return meteor_list + + +class MeteorObservation(object): + """ Container for meteor observations. + """ + def __init__(self, jdt_ref, station_id, latitude, longitude, height, fps, ff_name=None): + """ Construct the MeteorObservation object. + Arguments: + jdt_ref: [float] Reference Julian date when the relative time is t = 0s. + station_id: [str] Station ID. + latitude: [float] Latitude +N in radians. + longitude: [float] Longitude +E in radians. + height: [float] Elevation above sea level (MSL) in meters. + fps: [float] Frames per second. + + Keyword arguments: + ff_name: [str] Name of the originating FF file. + """ + self.jdt_ref = jdt_ref + self.station_id = station_id + self.latitude = latitude + self.longitude = longitude + self.height = height + self.fps = fps + self.ff_name = ff_name + self.frames = [] + self.time_data = [] + self.x_data = [] + self.y_data = [] + self.azim_data = [] + self.elev_data = [] + self.ra_data = [] + self.dec_data = [] + self.mag_data = [] + self.abs_mag_data = [] + + def _addPoint(self, frame_n, x, y, azim, elev, ra, dec, mag): + """ Adds the measurement point to the meteor. + + Arguments: + frame_n: [flaot] Frame number from the reference time. + x: [float] X image coordinate. + y: [float] X image coordinate. + azim: [float] Azimuth, J2000 in degrees. + elev: [float] Elevation angle, J2000 in degrees. + ra: [float] Right ascension, J2000 in degrees. + dec: [float] Declination, J2000 in degrees. + mag: [float] Visual magnitude. + """ + self.frames.append(frame_n) + + # Calculate the time in seconds w.r.t. to the reference JD + point_time = float(frame_n)/self.fps + self.time_data.append(point_time) + self.x_data.append(x) + self.y_data.append(y) + + # Angular coordinates converted to radians + self.azim_data.append(np.radians(azim)) + self.elev_data.append(np.radians(elev)) + self.ra_data.append(np.radians(ra)) + self.dec_data.append(np.radians(dec)) + self.mag_data.append(mag) + + def _finish(self): + """ When the initialization is done, convert data lists to numpy arrays. """ + self.frames = np.array(self.frames) + self.time_data = np.array(self.time_data) + self.x_data = np.array(self.x_data) + self.y_data = np.array(self.y_data) + self.azim_data = np.array(self.azim_data) + self.elev_data = np.array(self.elev_data) + self.ra_data = np.array(self.ra_data) + self.dec_data = np.array(self.dec_data) + self.mag_data = np.array(self.mag_data) + # Sort by frame + temp_arr = np.c_[self.frames, self.time_data, self.x_data, self.y_data, self.azim_data, + self.elev_data, self.ra_data, self.dec_data, self.mag_data] + temp_arr = temp_arr[np.argsort(temp_arr[:, 0])] + self.frames, self.time_data, self.x_data, self.y_data, self.azim_data, self.elev_data, self.ra_data, \ + self.dec_data, self.mag_data = temp_arr.T + + +def _writeFTPHeader(ftpf, metcount, fldr, ufo=True): + """ + Internal function to create the header of the FTPDetect file + """ + l1 = 'Meteor Count = {:06d}\n'.format(metcount) + ftpf.write(l1) + ftpf.write('-----------------------------------------------------\n') + if ufo is True: + ftpf.write('Processed with UFOAnalyser\n') + else: + ftpf.write('Processed with RMS 1.0\n') + ftpf.write('-----------------------------------------------------\n') + l1 = 'FF folder = {:s}\n'.format(fldr) + ftpf.write(l1) + l1 = 'CAL folder = {:s}\n'.format(fldr) + ftpf.write(l1) + ftpf.write('-----------------------------------------------------\n') + ftpf.write('FF file processed\n') + ftpf.write('CAL file processed\n') + ftpf.write('Cam# Meteor# #Segments fps hnr mle bin Pix/fm Rho Phi\n') + ftpf.write('Per segment: Frame# Col Row RA Dec Azim Elev Inten Mag\n') + + +def _writeOneMeteor(ftpf, metno, sta, evttime, fcount, fps, fno, ra, dec, az, alt, b, mag, + ufo=True, x=None, y=None, ffname = None): + """ Internal function to write one meteor event into the file in FTPDetectInfo style + """ + ftpf.write('-------------------------------------------------------\n') + if ffname is None: + ms = '{:03d}'.format(int(evttime.microsecond / 1000)) + + fname = 'FF_' + sta + '_' + evttime.strftime('%Y%m%d_%H%M%S_') + ms + '_0000000.fits\n' + else: + fname = ffname + '\n' + ftpf.write(fname) + + if ufo is True: + ftpf.write('UFO UKMON DATA Recalibrated on: ') + else: + ftpf.write('RMS data reprocessed on: ') + ftpf.write(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f UTC\n')) + li = f'{sta} {metno:04d} {fcount:04d} {fps:04.2f} 000.0 000.0 00.0 000.0 0000.0 0000.0\n' + ftpf.write(li) + + for i in range(len(fno)): + # 204.4909 0422.57 0353.46 262.3574 +16.6355 267.7148 +23.0996 000120 3.41 + bri = 0 + if b is not None: + bri = int(b[i]) + if ufo is True: + # UFO is timestamped as at the first detection + thisfn = fno[i] - fno[0] + thisx = 0 + thisy = 0 + else: + thisfn = fno[i] + thisx = x[i] + thisy = y[i] + li = f'{thisfn:.4f} {thisx:07.2f} {thisy:07.2f} ' + li = li + f'{ra[i]:8.4f} {dec[i]:+7.4f} {az[i]:8.4f} ' + li = li + f'{alt[i]:+7.4f} {bri:06d} {mag[i]:.02f}\n' + ftpf.write(li) diff --git a/archive/ukmon_pylib/utils/getActiveShowers.py b/archive/ukmon_pylib/utils/getActiveShowers.py new file mode 100644 index 00000000..3e18d632 --- /dev/null +++ b/archive/ukmon_pylib/utils/getActiveShowers.py @@ -0,0 +1,48 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +# simple script to get the active shower list from the IMO working list + +import imoWorkingShowerList as iwsl +import datetime + + +def getActiveShowers(targdate, retlist=False, inclMinor=False): + """ + Return a list of showers active at the specified date + + Arguments: + targdate: [str] Date in YYYYMMDD format + + Keyword Arguments: + retlist: [bool] return a list, or print to console. Default False=print + inclMinor: [bool] include minor showers or only return major showers + + Returns: + If retlist is true, returns a python list of shower short-codes eg ['PER','LYR'] + + """ + sl = iwsl.IMOshowerList() + testdate = datetime.datetime.strptime(targdate, '%Y%m%d') + listofshowers=sl.getActiveShowers(testdate, True, inclMinor=inclMinor) + if retlist is False: + for shwr in listofshowers: + print(shwr) + else: + return listofshowers + + +def getActiveShowersStr(targdatestr): + """ + Prints a comma-separated list of showers active at the specified date + + Arguments: + targdate: [str] Date in YYYYMMDD format + + Returns: + nothing + + """ + shwrs = getActiveShowers(targdatestr, retlist=True) + shwrs.append('spo') + for s in shwrs: + print(s) diff --git a/archive/ukmon_pylib/utils/imoWorkingShowerList.py b/archive/ukmon_pylib/utils/imoWorkingShowerList.py new file mode 100644 index 00000000..9c9e109a --- /dev/null +++ b/archive/ukmon_pylib/utils/imoWorkingShowerList.py @@ -0,0 +1,230 @@ +# +# python module to read the IMO Working Shower short List +# +# Copyright (C) 2018-2023 Mark McIntyre + +import xmltodict +import datetime +import os +import numpy as np +import copy + +from wmpl.Utils.TrajConversions import jd2Date +from .convertSolLon import sollon2jd + +# imported from $SRC/share +try: + from majorminor import majorlist, minorlist +except Exception: + majorlist = ['QUA', 'LYR', 'ETA', 'CAP', 'SDA', 'PER', 'AUR', 'ORI', 'NTA', 'STA', 'LEO', 'GEM', 'URS'] + minorlist = ['SPE','OCT','DRA','EGE','MON','HYD','COM','NOO'] + + +class IMOshowerList: + """ + Class that loads and parses the IMO Working Shower list, or if needed, the unconfirmed list. + Not all known showers are in the IMO working list. If a shower is not in the Working List then + this library will reference the full shower list curated by Peter Jenniskens which contains + debated and unconfirmed showers. + + These list are updated whenever the library version is bumped, but if you want to override the files, define an + environment variable DATADIR and place your own copies of the files at $DATADIR/share. See the share submodule + for more information. + + """ + def __init__(self, fname=None, fullstreamname=None): + if fname is None: + datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + fname = os.path.join(datadir, 'share', 'IMO_Working_Meteor_Shower_List.xml') + if not os.path.isfile(fname): + datadir=os.path.split(os.path.abspath(__file__))[0] + fname = os.path.join(datadir, '..', 'share', 'IMO_Working_Meteor_Shower_List.xml') + + tmplist = xmltodict.parse(open(fname, 'rb').read()) + self.showerlist = tmplist['meteor_shower_list']['shower'] + if fullstreamname is None: + fullstreamname = os.path.join(datadir, 'share', 'streamfulldata.npy') + if not os.path.isfile(fullstreamname): + datadir=os.path.split(os.path.abspath(__file__))[0] + fullstreamname = os.path.join(datadir, '..', 'share', 'streamfulldata.npy') + self.fullstreamdata = np.load(fullstreamname) + #print('initialised') + + + def getShowerByCode(self, iaucode, useFull=False): + ds = {'@id':None, 'IAU_code':None,'start':None, 'end':None, + 'peak':None, 'r':None, 'name':None, 'V':None, 'ZHR':None, 'RA':None, 'DE':None, 'pksollon': None} + ds2 = {'@id':None, 'IAU_code':None,'start':None, 'end':None, + 'peak':None, 'r':None, 'name':None, 'V':None, 'ZHR':None, 'RA':None, 'DE':None, 'pksollon': None} + for shower in self.showerlist: + if shower['IAU_code'] == iaucode: + ds = shower + if ds['@id'] is None: + ds['@id'] = -1 + pksollong = -1 + #print(ds) + subset = self.fullstreamdata[np.where(self.fullstreamdata[:,3]==iaucode)] + if subset is not None: + mtch = [sh for sh in subset if int(sh[6]) > -1] + if len(mtch) > 0: + ds2 = copy.deepcopy(ds) + ds2['IAU_code'] = mtch[-1][3].strip() + ds2['name'] = mtch[-1][4].strip() + ds2['V'] = mtch[-1][12] + ds2['@id'] = mtch[-1][1] + ds2['RA'] = mtch[-1][8] + ds2['DE'] = mtch[-1][9] + + pksollong = float(mtch[-1][7]) + dt = datetime.datetime.now() + yr = dt.year + mth = dt.month + jd = sollon2jd(yr, mth, pksollong) + pkdt = jd2Date(jd, dt_obj=True) + ds2['peak'] = pkdt.strftime('%h %d') + # start/end pop idx, ZHR not available in the IAU data + ds2['start'] = (pkdt + datetime.timedelta(days=-2)).strftime('%h %d') + ds2['end'] = (pkdt + datetime.timedelta(days=2)).strftime('%h %d') + ds2['pksollon'] = pksollong + #print(ds2) + else: + # okay so its poor quality but lets try it anyway + mtch = subset + ds2 = copy.deepcopy(ds) + ds2['IAU_code'] = mtch[-1][3].strip() + ds2['name'] = mtch[-1][4].strip() + ds2['V'] = mtch[-1][12] + ds2['@id'] = mtch[-1][1] + ds2['RA'] = mtch[-1][8] + ds2['DE'] = mtch[-1][9] + + pksollong = float(mtch[-1][7]) + dt = datetime.datetime.now() + yr = dt.year + mth = dt.month + jd = sollon2jd(yr, mth, pksollong) + pkdt = jd2Date(jd, dt_obj=True) + ds2['peak'] = pkdt.strftime('%h %d') + # start/end pop idx, ZHR not available in the IAU data + ds2['start'] = (pkdt + datetime.timedelta(days=-2)).strftime('%h %d') + ds2['end'] = (pkdt + datetime.timedelta(days=2)).strftime('%h %d') + ds2['pksollon'] = pksollong + if useFull is False: + if 'pksollon' not in ds: + ds['pksollon'] = ds2['pksollon'] + elif ds['pksollon'] is None: + ds['pksollon'] = ds2['pksollon'] + if ds['peak'] is None: + ds['peak'] = ds2['peak'] + ds['@id'] = ds2['@id'] + return ds + else: + ds2['ZHR'] = ds['ZHR'] + ds2['r'] = ds['r'] + return ds2 + + def getStart(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + if shower['start'] is not None: + startdate = datetime.datetime.strptime(shower['start'], '%b %d') + else: + startdate = datetime.datetime.strptime(shower['peak'], '%b %d') + datetime.timedelta(days=-3) + if iaucode == 'QUA' and mth !=12: + # quadrantids straddle yearend + now = now - 1 + startdate = startdate.replace(year=now) + return startdate + + def getEnd(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + #print(shower) + if shower['end'] is not None: + enddate = datetime.datetime.strptime(shower['end'], '%b %d') + else: + enddate = datetime.datetime.strptime(shower['peak'], '%b %d') + datetime.timedelta(days=3) + if iaucode == 'QUA' and mth == 12: + # quadrantids straddle yearend + now = now + 1 + enddate = enddate.replace(year=now) + return enddate + + def getPeak(self, iaucode, currdt=None): + shower = self.getShowerByCode(iaucode) + if currdt is None: + now = datetime.datetime.today().year + mth = datetime.datetime.today().month + else: + now = datetime.datetime.strptime(str(currdt), '%Y%m%d') + mth = now.month + now = now.year + enddate = datetime.datetime.strptime(shower['peak'], '%b %d') + if iaucode == 'QUA' and mth == 12: + # quadrantids straddle yearend + now = now + 1 + enddate = enddate.replace(year=now) + return enddate + + def getRvalue(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['r'] + + def getName(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['name'] + + def getVelocity(self, iaucode): + shower = self.getShowerByCode(iaucode) + return shower['V'] + + def getZHR(self, iaucode): + shower = self.getShowerByCode(iaucode) + zhr = shower['ZHR'] + if zhr is None: + return -1 + else: + return int(zhr) + + def getRaDec(self, iaucode): + shower = self.getShowerByCode(iaucode) + return float(shower['RA']), float(shower['DE']) + + def getActiveShowers(self, datetotest, majorOnly=False, inclMinor=False): + activelist = [] + for shower in self.showerlist: + shwname = shower['IAU_code'] + if shwname == 'ANT': #skip the anthelion source, its not a real shower + continue + start = self.getStart(shwname, datetotest.strftime('%Y%m%d')) + #print(shwname, start,shower) + end = self.getEnd(shwname, datetotest.strftime('%Y%m%d')) + datetime.timedelta(days=3) + if datetotest > start and datetotest < end: + if majorOnly is False or (majorOnly is True and shwname in majorlist): + activelist.append(shwname) + elif inclMinor is True and shwname in minorlist: + activelist.append(shwname) + return activelist + + def getMajorShowers(self, includeSpo=False, stringFmt=False): + majlist = majorlist + if includeSpo is True: + majlist.append('spo') + if stringFmt is True: + tmplist = '' + for shwr in majlist: + tmplist = tmplist + shwr + ' ' + majlist = tmplist + return majlist From 9cb4867e1d60a89e1d34441464972e0d133b25eb Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 15:35:44 +0100 Subject: [PATCH 145/287] updating how shower data is obtained --- archive/analysis/reportActiveShowers.sh | 2 +- archive/cronjobs/getImoWSfile.sh | 2 +- archive/ukmon_pylib/utils/getActiveShowers.py | 95 ++++++++++++++++++- .../ukmon_pylib/utils/imoWorkingShowerList.py | 11 +-- 4 files changed, 98 insertions(+), 12 deletions(-) diff --git a/archive/analysis/reportActiveShowers.sh b/archive/analysis/reportActiveShowers.sh index a42c90d3..e26e0463 100644 --- a/archive/analysis/reportActiveShowers.sh +++ b/archive/analysis/reportActiveShowers.sh @@ -30,7 +30,7 @@ fi logger -s -t reportActiveShowers "report on active showers" python -m reports.reportActiveShowers -m -python -c "from meteortools.utils import getActiveShowers; getActiveShowers('$rundt', inclMinor=True)" | while read shwr +python -c "from utils.getActiveShowers import getActiveShowers;getActiveShowers('$rundt', inclMinor=True)" | while read shwr do aws s3 sync $DATADIR/reports/${yr}/$shwr $WEBSITEBUCKET/reports/${yr}/${shwr} --quiet done diff --git a/archive/cronjobs/getImoWSfile.sh b/archive/cronjobs/getImoWSfile.sh index 021b6bad..69c20be3 100644 --- a/archive/cronjobs/getImoWSfile.sh +++ b/archive/cronjobs/getImoWSfile.sh @@ -19,7 +19,7 @@ git checkout wmpl/share/streamfulldata.csv #git checkout wmpl/share/ShowerLookUpTable.txt git checkout wmpl/share/gmn_shower_table_20230518.txt -python -c "from meteortools.utils.getShowerDates import numpifyShowerData; numpifyShowerData()" +python -c "from utils.getShowerDates import numpifyShowerData; numpifyShowerData()" logger -s -t getImoWSfile "finished" diff --git a/archive/ukmon_pylib/utils/getActiveShowers.py b/archive/ukmon_pylib/utils/getActiveShowers.py index 3e18d632..5e2563b9 100644 --- a/archive/ukmon_pylib/utils/getActiveShowers.py +++ b/archive/ukmon_pylib/utils/getActiveShowers.py @@ -2,8 +2,10 @@ # # simple script to get the active shower list from the IMO working list -import imoWorkingShowerList as iwsl +from utils.imoWorkingShowerList import IMOshowerList as iwsl import datetime +import numpy as np +import os def getActiveShowers(targdate, retlist=False, inclMinor=False): @@ -21,7 +23,7 @@ def getActiveShowers(targdate, retlist=False, inclMinor=False): If retlist is true, returns a python list of shower short-codes eg ['PER','LYR'] """ - sl = iwsl.IMOshowerList() + sl = iwsl() testdate = datetime.datetime.strptime(targdate, '%Y%m%d') listofshowers=sl.getActiveShowers(testdate, True, inclMinor=inclMinor) if retlist is False: @@ -46,3 +48,92 @@ def getActiveShowersStr(targdatestr): shwrs.append('spo') for s in shwrs: print(s) + + +def getShowerDets(shwr, stringFmt=False, dataPth=None): + """ Get details of a shower + + Arguments: + shwr: [string] three-letter shower code eg PER + Keyword Arguments: + stringFmt [bool] default False, return a string rather than a list + dataPth [string] path to the datafiles. Default None means data read from internal files. + + Returns: + (id, full name, peak solar longitude, peak date mm-dd) + """ + sl = iwsl.IMOshowerList() + mtch = sl.getShowerByCode(shwr, useFull=True) + if len(mtch) > 0 and mtch['@id'] is not None: + id = int(mtch['@id']) + nam = mtch['name'] + pkdtstr = mtch['peak'] + dt = datetime.datetime.now() + yr = dt.year + pkdt = datetime.datetime.strptime(f'{yr} {pkdtstr}','%Y %b %d') + dtstr = pkdt.strftime('%m-%d') + pksollong = mtch['pksollon'] + else: + id, nam, pksollong, dtstr = 0, 'Unknown', 0, 'Unknown' + if stringFmt: + return f"{pksollong},{dtstr},{nam},{shwr}" + else: + return id, nam, pksollong, dtstr + + +def getShowerPeak(shwr): + """ Get date of a shower peak in MM-DD format + + Arguments: + shwr: [string] three-letter shower code eg PER + + Returns: + peak date mm-dd + """ + _, _, _, pk = getShowerDets(shwr) + return pk + + +def numpifyShowerData(): + """Refresh the numpy versions of the shower data files """ + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + abs_path = os.getenv('WMPL_LOC', default=os.path.expanduser('~/src/WesternMeteorPyLib')) + + iau_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'streamfulldata.csv') + iau_shower_table_npy = os.path.join(abs_path, 'wmpl', 'share', 'streamfulldata.npy') + iau_shower_list = np.loadtxt(iau_shower_table_file, delimiter="|", usecols=range(20), dtype=str) + np.save(iau_shower_table_npy, iau_shower_list) + + iau_shower_table_npy = os.path.join(datadir, 'share', 'streamfulldata.npy') + np.save(iau_shower_table_npy, iau_shower_list) + + gmn_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'gmn_shower_table_20230518.txt') + gmn_shower_table_npy = os.path.join(abs_path, 'wmpl', 'share', 'gmn_shower_table_20230518.npy') + gmn_shower_list = _loadGMNShowerTable(*os.path.split(gmn_shower_table_file)) + np.save(gmn_shower_table_npy, gmn_shower_list) + + gmn_shower_table_npy = os.path.join(datadir, 'share', 'gmn_shower_table_20230518.npy') + np.save(gmn_shower_table_npy, gmn_shower_list) + + +def _loadGMNShowerTable(dir_path, file_name): + gmn_shower_list = [] + with open(os.path.join(dir_path, file_name), encoding='cp1252') as f: + for line in f: + if line.startswith('#'): + continue + line = line.strip() + line = line.replace('\n', '').replace('\r', '') + if not line: + continue + la_sun, L_g, B_g, v_g, dispersion, IAU_no, IAU_code = line.split() + gmn_shower_list.append([ + np.radians(float(la_sun)), + np.radians(float(L_g)), + np.radians(float(B_g)), + 1000*float(v_g), + np.radians(float(dispersion)), + int(IAU_no)], + IAU_code + ) + return np.array(gmn_shower_list) diff --git a/archive/ukmon_pylib/utils/imoWorkingShowerList.py b/archive/ukmon_pylib/utils/imoWorkingShowerList.py index 9c9e109a..8ccd6064 100644 --- a/archive/ukmon_pylib/utils/imoWorkingShowerList.py +++ b/archive/ukmon_pylib/utils/imoWorkingShowerList.py @@ -10,7 +10,7 @@ import copy from wmpl.Utils.TrajConversions import jd2Date -from .convertSolLon import sollon2jd +from utils.convertSolLon import sollon2jd # imported from $SRC/share try: @@ -27,18 +27,13 @@ class IMOshowerList: this library will reference the full shower list curated by Peter Jenniskens which contains debated and unconfirmed showers. - These list are updated whenever the library version is bumped, but if you want to override the files, define an - environment variable DATADIR and place your own copies of the files at $DATADIR/share. See the share submodule - for more information. - """ def __init__(self, fname=None, fullstreamname=None): if fname is None: - datadir = os.getenv('DATADIR', default='/home/ec2-user/prod/data') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) fname = os.path.join(datadir, 'share', 'IMO_Working_Meteor_Shower_List.xml') if not os.path.isfile(fname): - datadir=os.path.split(os.path.abspath(__file__))[0] - fname = os.path.join(datadir, '..', 'share', 'IMO_Working_Meteor_Shower_List.xml') + return tmplist = xmltodict.parse(open(fname, 'rb').read()) self.showerlist = tmplist['meteor_shower_list']['shower'] From 646f5e7b51914edd87b79133aface6d555c47408 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 15:44:19 +0100 Subject: [PATCH 146/287] corercting location of IMO and gmn shower data --- archive/cronjobs/getImoWSfile.sh | 2 +- archive/ukmon_pylib/utils/getActiveShowers.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/cronjobs/getImoWSfile.sh b/archive/cronjobs/getImoWSfile.sh index 69c20be3..d7cb0399 100644 --- a/archive/cronjobs/getImoWSfile.sh +++ b/archive/cronjobs/getImoWSfile.sh @@ -19,7 +19,7 @@ git checkout wmpl/share/streamfulldata.csv #git checkout wmpl/share/ShowerLookUpTable.txt git checkout wmpl/share/gmn_shower_table_20230518.txt -python -c "from utils.getShowerDates import numpifyShowerData; numpifyShowerData()" +python -c "from utils.getActiveShowers import numpifyShowerData; numpifyShowerData()" logger -s -t getImoWSfile "finished" diff --git a/archive/ukmon_pylib/utils/getActiveShowers.py b/archive/ukmon_pylib/utils/getActiveShowers.py index 5e2563b9..e147693c 100644 --- a/archive/ukmon_pylib/utils/getActiveShowers.py +++ b/archive/ukmon_pylib/utils/getActiveShowers.py @@ -96,7 +96,7 @@ def getShowerPeak(shwr): def numpifyShowerData(): """Refresh the numpy versions of the shower data files """ - datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + srcdir = os.getenv('SRC', default=os.path.expanduser('~/prod')) abs_path = os.getenv('WMPL_LOC', default=os.path.expanduser('~/src/WesternMeteorPyLib')) iau_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'streamfulldata.csv') @@ -104,7 +104,7 @@ def numpifyShowerData(): iau_shower_list = np.loadtxt(iau_shower_table_file, delimiter="|", usecols=range(20), dtype=str) np.save(iau_shower_table_npy, iau_shower_list) - iau_shower_table_npy = os.path.join(datadir, 'share', 'streamfulldata.npy') + iau_shower_table_npy = os.path.join(srcdir, 'share', 'streamfulldata.npy') np.save(iau_shower_table_npy, iau_shower_list) gmn_shower_table_file = os.path.join(abs_path, 'wmpl', 'share', 'gmn_shower_table_20230518.txt') @@ -112,7 +112,7 @@ def numpifyShowerData(): gmn_shower_list = _loadGMNShowerTable(*os.path.split(gmn_shower_table_file)) np.save(gmn_shower_table_npy, gmn_shower_list) - gmn_shower_table_npy = os.path.join(datadir, 'share', 'gmn_shower_table_20230518.npy') + gmn_shower_table_npy = os.path.join(srcdir, 'share', 'gmn_shower_table_20230518.npy') np.save(gmn_shower_table_npy, gmn_shower_list) From c1ed84f51a8eb542fbbddda430b4b894d7b19c65 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 15:47:40 +0100 Subject: [PATCH 147/287] bugfixes --- archive/ukmon_pylib/utils/getActiveShowers.py | 4 ++-- install_or_update.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/utils/getActiveShowers.py b/archive/ukmon_pylib/utils/getActiveShowers.py index e147693c..96b3c244 100644 --- a/archive/ukmon_pylib/utils/getActiveShowers.py +++ b/archive/ukmon_pylib/utils/getActiveShowers.py @@ -133,7 +133,7 @@ def _loadGMNShowerTable(dir_path, file_name): np.radians(float(B_g)), 1000*float(v_g), np.radians(float(dispersion)), - int(IAU_no)], - IAU_code + int(IAU_no)] + ) return np.array(gmn_shower_list) diff --git a/install_or_update.sh b/install_or_update.sh index 21e403e6..0715fbfa 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -28,14 +28,14 @@ mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajd mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs -echo "Updating config for $envname" -~/$envname/utils/makeConfig.sh $RUNTIME_ENV -read -n 1 -p "Update bashrc and aliases? (Y/n) " yesno +read -n 1 -p "Update bashrc and config? (Y/n) " yesno if [[ "$yesno" == "n" || "$yesno" == "N" ]] then - echo skipping bashrc + echo skipping config and bashrc else + echo "Updating config for $envname" + ~/$envname/utils/makeConfig.sh $RUNTIME_ENV for fil in .bashrc .bash_aliases .vimrc .condaon do rsync -a server_setup/$fil ~ From e8e22478a58f7e3076684a1c17e04e9ea0eacbe2 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 15:53:47 +0100 Subject: [PATCH 148/287] fixes in imoWorkinShowerlist --- archive/ukmon_pylib/utils/imoWorkingShowerList.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/archive/ukmon_pylib/utils/imoWorkingShowerList.py b/archive/ukmon_pylib/utils/imoWorkingShowerList.py index 8ccd6064..442cf30d 100644 --- a/archive/ukmon_pylib/utils/imoWorkingShowerList.py +++ b/archive/ukmon_pylib/utils/imoWorkingShowerList.py @@ -12,11 +12,11 @@ from wmpl.Utils.TrajConversions import jd2Date from utils.convertSolLon import sollon2jd -# imported from $SRC/share try: +# imported from $SRC/share from majorminor import majorlist, minorlist except Exception: - majorlist = ['QUA', 'LYR', 'ETA', 'CAP', 'SDA', 'PER', 'AUR', 'ORI', 'NTA', 'STA', 'LEO', 'GEM', 'URS'] + c = ['QUA', 'LYR', 'ETA', 'CAP', 'SDA', 'PER', 'AUR', 'ORI', 'NTA', 'STA', 'LEO', 'GEM', 'URS'] minorlist = ['SPE','OCT','DRA','EGE','MON','HYD','COM','NOO'] @@ -30,18 +30,16 @@ class IMOshowerList: """ def __init__(self, fname=None, fullstreamname=None): if fname is None: - datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - fname = os.path.join(datadir, 'share', 'IMO_Working_Meteor_Shower_List.xml') + srcdir = os.getenv('SRC', default=os.path.expanduser('~/prod')) + fname = os.path.join(srcdir, 'share', 'IMO_Working_Meteor_Shower_List.xml') if not os.path.isfile(fname): + print('unable to open data file') return tmplist = xmltodict.parse(open(fname, 'rb').read()) self.showerlist = tmplist['meteor_shower_list']['shower'] if fullstreamname is None: - fullstreamname = os.path.join(datadir, 'share', 'streamfulldata.npy') - if not os.path.isfile(fullstreamname): - datadir=os.path.split(os.path.abspath(__file__))[0] - fullstreamname = os.path.join(datadir, '..', 'share', 'streamfulldata.npy') + fullstreamname = os.path.join(srcdir, 'share', 'streamfulldata.npy') self.fullstreamdata = np.load(fullstreamname) #print('initialised') From ed8eb85b97d6541aa438131e50020281e5c19b0f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 15:58:40 +0100 Subject: [PATCH 149/287] small error in install_or_update --- archive/ukmon_pylib/utils/imoWorkingShowerList.py | 2 +- install_or_update.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/utils/imoWorkingShowerList.py b/archive/ukmon_pylib/utils/imoWorkingShowerList.py index 442cf30d..2fccc2b2 100644 --- a/archive/ukmon_pylib/utils/imoWorkingShowerList.py +++ b/archive/ukmon_pylib/utils/imoWorkingShowerList.py @@ -13,7 +13,7 @@ from utils.convertSolLon import sollon2jd try: -# imported from $SRC/share + # imported from $SRC/share from majorminor import majorlist, minorlist except Exception: c = ['QUA', 'LYR', 'ETA', 'CAP', 'SDA', 'PER', 'AUR', 'ORI', 'NTA', 'STA', 'LEO', 'GEM', 'URS'] diff --git a/install_or_update.sh b/install_or_update.sh index 0715fbfa..c60b63c9 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -28,6 +28,7 @@ mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajd mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs +$here/cronjobs/getImoWSfile.sh read -n 1 -p "Update bashrc and config? (Y/n) " yesno if [[ "$yesno" == "n" || "$yesno" == "N" ]] From 5118b18d6d783f0b8bb5086dfc06ae570db8eee3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 16:58:23 +0100 Subject: [PATCH 150/287] bugfix when no data found in imo set --- archive/ukmon_pylib/utils/imoWorkingShowerList.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/utils/imoWorkingShowerList.py b/archive/ukmon_pylib/utils/imoWorkingShowerList.py index 2fccc2b2..abc76760 100644 --- a/archive/ukmon_pylib/utils/imoWorkingShowerList.py +++ b/archive/ukmon_pylib/utils/imoWorkingShowerList.py @@ -57,7 +57,7 @@ def getShowerByCode(self, iaucode, useFull=False): pksollong = -1 #print(ds) subset = self.fullstreamdata[np.where(self.fullstreamdata[:,3]==iaucode)] - if subset is not None: + if subset is not None and len(subset) > 0: mtch = [sh for sh in subset if int(sh[6]) > -1] if len(mtch) > 0: ds2 = copy.deepcopy(ds) From 030dbaaed78d71fc3c69f24a1e52f58caaf630d1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 16:58:47 +0100 Subject: [PATCH 151/287] retrieve IMO data after each code update --- install_or_update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index c60b63c9..ffbb1614 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -28,7 +28,7 @@ mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajd mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs -$here/cronjobs/getImoWSfile.sh +~/$envname/cronjobs/getImoWSfile.sh read -n 1 -p "Update bashrc and config? (Y/n) " yesno if [[ "$yesno" == "n" || "$yesno" == "N" ]] From 772f912ab3ae64111b0b626106a739d7e8301626 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 17:02:27 +0100 Subject: [PATCH 152/287] fixing up getShowerDets --- archive/ukmon_pylib/utils/getActiveShowers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/utils/getActiveShowers.py b/archive/ukmon_pylib/utils/getActiveShowers.py index 96b3c244..74d03b4c 100644 --- a/archive/ukmon_pylib/utils/getActiveShowers.py +++ b/archive/ukmon_pylib/utils/getActiveShowers.py @@ -62,7 +62,7 @@ def getShowerDets(shwr, stringFmt=False, dataPth=None): Returns: (id, full name, peak solar longitude, peak date mm-dd) """ - sl = iwsl.IMOshowerList() + sl = iwsl() mtch = sl.getShowerByCode(shwr, useFull=True) if len(mtch) > 0 and mtch['@id'] is not None: id = int(mtch['@id']) From 1bd0033a1dfbe52201c8ea2893b1383c3f71e0cc Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 17:05:32 +0100 Subject: [PATCH 153/287] improve installer --- install_or_update.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/install_or_update.sh b/install_or_update.sh index ffbb1614..3b49e93c 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -16,11 +16,12 @@ cd $here/archive git pull [ -d ~/$envname/data ] && msg="upgrade" || msg="install" -for loc in analysis ukmon_pylib website cronjobs utils share static_content +for loc in analysis ukmon_pylib website cronjobs utils static_content do rsync -a --delete $loc/ ~/${envname}/$loc chmod +x ~/${envname}/$loc/*.sh > /dev/null 2>&1 done +rsync -a share/ ~/${envname}/share DATADIR=~/$envname/data mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls,manualuploads} @@ -28,19 +29,23 @@ mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajd mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs -~/$envname/cronjobs/getImoWSfile.sh +# update the IMO and GMN meteor shower tables if missing +if [ ! -f ~/${envname}/share/IMO_Working_Meteor_Shower_List.xml ] +then + ~/$envname/cronjobs/getImoWSfile.sh +fi -read -n 1 -p "Update bashrc and config? (Y/n) " yesno -if [[ "$yesno" == "n" || "$yesno" == "N" ]] +read -n 1 -p "Update bashrc and config? (y/N) " yesno +if [[ "$yesno" == "y" || "$yesno" == "Y" ]] then - echo skipping config and bashrc -else echo "Updating config for $envname" ~/$envname/utils/makeConfig.sh $RUNTIME_ENV for fil in .bashrc .bash_aliases .vimrc .condaon do rsync -a server_setup/$fil ~ done +else + echo skipping config and bashrc fi echo "" echo "$msg complete" \ No newline at end of file From 127b8148f5127d2e8362acc3dd818186908457f4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 17:11:31 +0100 Subject: [PATCH 154/287] fix createShowerExtracts --- archive/website/createShwrExtracts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/website/createShwrExtracts.sh b/archive/website/createShwrExtracts.sh index beb015b4..d90f0463 100644 --- a/archive/website/createShwrExtracts.sh +++ b/archive/website/createShwrExtracts.sh @@ -44,7 +44,7 @@ cd $DATADIR/browse/showers/ # get a list of files on the website aws s3 ls $WEBSITEBUCKET/browse/showers/ | awk '{ print $4 }' | grep csv > /tmp/browseshwr.txt -shwrs=$(python -c "from meteortools.utils.getActiveShowers import getActiveShowersStr ; getActiveShowersStr('${ymd}')") +shwrs=$(python -c "from utils.getActiveShowers import getActiveShowersStr ; getActiveShowersStr('${ymd}')") for shwr in $shwrs do now=$(date '+%Y-%m-%d %H:%M:%S') From 9a6a39339e3506273809919493ffcefef0816821 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 17:14:49 +0100 Subject: [PATCH 155/287] update extractors and showeranalysis --- archive/ukmon_pylib/analysis/showerAnalysis.py | 10 +++++----- archive/ukmon_pylib/reports/extractors.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index 02551d78..701b75cc 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -10,8 +10,8 @@ from matplotlib import pyplot as plt import datetime -from meteortools.utils import getShowerDates as sd -from meteortools.fileformats import imoWorkingShowerList as imo +from utils.getActiveShowers import getShowerDets +from utils.imoWorkingShowerList import IMOshowerList SMALL_SIZE = 8 MEDIUM_SIZE = 10 @@ -486,7 +486,7 @@ def showerAnalysis(shwr, dtstr): cols = ['Shwr','Dtstamp','Y','M','ID','Mag'] filt = None if shwr != 'ALL': - sl = imo.IMOshowerList() + sl = IMOshowerList() maxdt = sl.getEnd(shwr) + datetime.timedelta(days=10) mindt = sl.getStart(shwr) + datetime.timedelta(days=-10) @@ -498,7 +498,7 @@ def showerAnalysis(shwr, dtstr): # select the required data if shwr != 'ALL': - id, shwrname, sl, dt = sd.getShowerDets(shwr) + id, shwrname, sl, dt = getShowerDets(shwr) sngl = sngl[sngl['Shwr']==shwr] sngl = sngl[sngl.Dtstamp > mindt.timestamp()] sngl = sngl[sngl.Dtstamp < maxdt.timestamp()] @@ -531,7 +531,7 @@ def showerAnalysis(shwr, dtstr): # select the required data if shwr != 'ALL': - id, shwrname, sl, dt = sd.getShowerDets(shwr) + id, shwrname, sl, dt = getShowerDets(shwr) mtch = mtch[mtch['_stream']==shwr] mtch = mtch[mtch.dtstamp > mindt.timestamp()] mtch = mtch[mtch.dtstamp < maxdt.timestamp()] diff --git a/archive/ukmon_pylib/reports/extractors.py b/archive/ukmon_pylib/reports/extractors.py index 65670b38..c3724f47 100644 --- a/archive/ukmon_pylib/reports/extractors.py +++ b/archive/ukmon_pylib/reports/extractors.py @@ -6,8 +6,8 @@ import os import sys import pandas as pd -from meteortools.fileformats import imoWorkingShowerList as imo -from meteortools.utils import getActiveShowers +from utils.imoWorkingShowerList import IMOshowerList +from utils.getActiveShowers import getActiveShowers def createSplitMatchFile(yr, mth=None, shwr=None, matches=None): @@ -143,7 +143,7 @@ def extractAllShowersData(ymd): showerlist = getActiveShowers(ymd, retlist=True) showerlist.append('spo') else: - sl = imo.IMOshowerList() + sl = IMOshowerList() showerlist = sl.getMajorShowers(True, True).strip().split(' ') print(f'processing data for {ymd}') From f45a037508bd03ee30c3927f98eb7c213889f293 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 17:19:05 +0100 Subject: [PATCH 156/287] bug in extractor thats existed since 2023 but nobody noticed... --- archive/ukmon_pylib/reports/extractors.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/archive/ukmon_pylib/reports/extractors.py b/archive/ukmon_pylib/reports/extractors.py index c3724f47..8541e759 100644 --- a/archive/ukmon_pylib/reports/extractors.py +++ b/archive/ukmon_pylib/reports/extractors.py @@ -159,8 +159,9 @@ def extractAllShowersData(ymd): fname = os.path.join(datadir, 'consolidated','M_{}-unified.csv'.format(yr)) if not os.path.isfile(fname): print(f'unable to open {fname}') - return - ufosingles = pd.read_csv(fname, skipinitialspace=True) + ufosingles = None + else: + ufosingles = pd.read_csv(fname, skipinitialspace=True) fname = os.path.join(datadir, 'single','singles-{}.parquet.snap'.format(yr)) if not os.path.isfile(fname): print(f'unable to open {fname}') @@ -174,8 +175,9 @@ def extractAllShowersData(ymd): print(f'processing RMS singles for {shwr}') createRMSSingleMonthlyExtract(yr, shwr=shwr, dta=rmssingles) createRMSSingleMonthlyExtract(yr, shwr=shwr, dta=rmssingles, withshower=True) - print(f'processing UFO singles for {shwr}') - createUFOSingleMonthlyExtract(yr, shwr=shwr, dta=ufosingles) + if ufosingles: + print(f'processing UFO singles for {shwr}') + createUFOSingleMonthlyExtract(yr, shwr=shwr, dta=ufosingles) return From 48e51c960c698b3c61c15024f44a4b194eea5aac Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 17:41:29 +0100 Subject: [PATCH 157/287] trying to fix matplotlib bug --- archive/ukmon_pylib/analysis/showerAnalysis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index 701b75cc..0f0afea7 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -454,7 +454,8 @@ def magDistributionVis(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) + #ax.set_xticklabels(x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) From b524e9430e9bae3070aae924a25b60d7402ed72a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 17:44:58 +0100 Subject: [PATCH 158/287] more working on matplotlib issue --- .../ukmon_pylib/analysis/showerAnalysis.py | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index 0f0afea7..f9b4223d 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -110,7 +110,7 @@ def timeGraph(dta, shwrname, outdir, binmins=10): lab.set_fontsize(SMALL_SIZE) # format x-axes x_labels = binned.index.strftime('%b-%d') - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) ax.set(xlabel="Date", ylabel="Count") plt.title('Observed stream activity {}min intervals ({})'.format(binmins, shwrname)) plt.tight_layout() @@ -157,7 +157,7 @@ def matchesGraphs(dta, shwrname, outdir, binmins=60, startdt=None, enddt=None, t else: x_labels = mbinned.index.strftime('%b-%d') - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) ax.set(xlabel="Date", ylabel="Count") @@ -216,7 +216,7 @@ def velDistribution(dta, shwrname, outdir, vg_or_vs, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -254,7 +254,7 @@ def durationDistribution(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.1f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -292,7 +292,7 @@ def distanceDistribution(dta, shwrname, outdir, binwidth=1.0): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -356,13 +356,6 @@ def radiantDistribution(dta, shwrname, outdir): magdf.plot.scatter(x=idx, y=idx2) ax = plt.gca() ax.set(xlabel='RA (deg)', ylabel="Dec (deg)") - #plt.locator_params(axis='x', nbins=12) - #for lab in ax.get_xticklabels(): - # lab.set_fontsize(SMALL_SIZE) - - # format x-axes - #x_labels=["%.0f" % number for number in bins[:-1]] - #ax.set_xticklabels(x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -394,7 +387,7 @@ def semimajorDistribution(dta, shwrname, outdir, binwidth=0.5): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -424,7 +417,7 @@ def magDistributionAbs(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] - ax.set_xticklabels(x_labels) + plt.xticks(np.arange(len(x_labels)), x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) @@ -455,7 +448,6 @@ def magDistributionVis(dta, shwrname, outdir, binwidth=0.2): # format x-axes x_labels=["%.0f" % number for number in bins[:-1]] plt.xticks(np.arange(len(x_labels)), x_labels) - #ax.set_xticklabels(x_labels) fig = plt.gcf() fig.set_size_inches(11.6, 8.26) From 5a8aa0337c5a231e3c8a86fcc0ec45701d6709b9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 17:48:03 +0100 Subject: [PATCH 159/287] oops overfixed a bug --- archive/ukmon_pylib/analysis/showerAnalysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index f9b4223d..ac2895ae 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -110,7 +110,7 @@ def timeGraph(dta, shwrname, outdir, binmins=10): lab.set_fontsize(SMALL_SIZE) # format x-axes x_labels = binned.index.strftime('%b-%d') - plt.xticks(np.arange(len(x_labels)), x_labels) + ax.set_xticklabels(x_labels) ax.set(xlabel="Date", ylabel="Count") plt.title('Observed stream activity {}min intervals ({})'.format(binmins, shwrname)) plt.tight_layout() From 383dd745719e5a9d7c1845f74f7343f8bbfe91d4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 17:50:54 +0100 Subject: [PATCH 160/287] another incorrect bugfix! --- archive/ukmon_pylib/analysis/showerAnalysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index ac2895ae..c249f82f 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -157,7 +157,7 @@ def matchesGraphs(dta, shwrname, outdir, binmins=60, startdt=None, enddt=None, t else: x_labels = mbinned.index.strftime('%b-%d') - plt.xticks(np.arange(len(x_labels)), x_labels) + ax.set_xticklabels(x_labels) ax.set(xlabel="Date", ylabel="Count") From c2e8b1fca0a3b0edf281735c26ea08db54501afd Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 18:23:26 +0100 Subject: [PATCH 161/287] make bars wider --- archive/ukmon_pylib/analysis/showerAnalysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index c249f82f..c9d01ace 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -98,7 +98,7 @@ def timeGraph(dta, shwrname, outdir, binmins=10): # resample it binned = countcol.resample('{}min'.format(binmins)).count() - binned.plot(kind='bar') + binned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals nbins = max(len(binned)/144, 2) @@ -139,7 +139,7 @@ def matchesGraphs(dta, shwrname, outdir, binmins=60, startdt=None, enddt=None, t mcountcol = mdta['_ID1'] # resample it mbinned = mcountcol.resample('{}min'.format(binmins)).count() - mbinned.plot(kind='bar') + mbinned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals ax = plt.gca() From 550afbed938a1756413710a25ff4f1ace7b94fb7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 18:32:35 +0100 Subject: [PATCH 162/287] more removal of MeteorTools --- .../maintenance/plotStationsOnMap.py | 13 +- .../ukmon_pylib/reports/makeCoverageMap.py | 2 +- archive/ukmon_pylib/utils/kmlHandlers.py | 149 ++++++++++++++++++ 3 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 archive/ukmon_pylib/utils/kmlHandlers.py diff --git a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py index 59b37d3b..e90cca86 100644 --- a/archive/ukmon_pylib/maintenance/plotStationsOnMap.py +++ b/archive/ukmon_pylib/maintenance/plotStationsOnMap.py @@ -6,14 +6,11 @@ import numpy as np import os import sys -import fnmatch import matplotlib.pyplot as plt import json import cartopy.crs as ccrs -from meteortools.fileformats import UFOAnalyzerXML as ua - from PIL import Image @@ -21,15 +18,7 @@ def getBearingsForEvent(stns, fldr): brngs = np.zeros(len(stns)) listOfFiles = os.listdir(fldr) for entry in listOfFiles: - if fnmatch.fnmatch(entry, '*A.XML'): - fullname = os.path.join(fldr, entry) - xmlf = ua.UAXml(fullname) - sta, _, _, _, _, _ = xmlf.getStationDetails() - _, _, _, _, _, _, az1, _ = xmlf.getObjectStart(0) - i = stns.index(sta) - brngs[i] = az1 - - elif entry == 'FTPdetectinfo_UFO.txt': + if entry == 'FTPdetectinfo_UFO.txt': with open(os.path.join(fldr, 'FTPdetectinfo_UFO.txt'), 'r') as f: lis = f.readlines() for s in stns: diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index bbb7f0fb..ca73a073 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -7,7 +7,7 @@ import gmplot import glob from cryptography.fernet import Fernet -from meteortools.fileformats import readCameraKML +from utils.kmlHandlers import readCameraKML def decodeApiKey(enckey): diff --git a/archive/ukmon_pylib/utils/kmlHandlers.py b/archive/ukmon_pylib/utils/kmlHandlers.py new file mode 100644 index 00000000..60f958b2 --- /dev/null +++ b/archive/ukmon_pylib/utils/kmlHandlers.py @@ -0,0 +1,149 @@ +import xmltodict +from shapely.geometry import Polygon +import csv +import simplekml +import numpy as np +import pandas as pd +import os + + +def readCameraKML(kmlFilename, return_poly=False): + """ Load a KML file and return either a list of lats and longs, or a Shapely polygon + + Arguments: + kmlFilename: [string] full path to the KML file to consume + return_poly: [bool] return a Shapely polygon? Default False + + + Returns: + if return_poly is false, returns a tuple of (cameraname, lats, longs) where lats and longs are lists of the + latitudes and longitudes in the KML file. + + If return_poly is true, returns a tuple of (cameranamem, shapely Polygon) + """ + + with open(kmlFilename) as fd: + x = xmltodict.parse(fd.read()) + cname = x['kml']['Folder']['name'] + coords = x['kml']['Folder']['Placemark']['MultiGeometry']['Polygon']['outerBoundaryIs']['LinearRing']['coordinates'] + coords = coords.split('\n') + if return_poly is False: + lats = [] + lngs = [] + for lin in coords: + s = lin.split(',') + lngs.append(float(s[0])) + lats.append(float(s[1])) + return cname, lats, lngs + else: + ptsarr=[] + for lin in coords: + s = lin.split(',') + ptsarr.append((float(s[0]), float(s[1]))) + polyg = Polygon(ptsarr) + return cname, polyg + + +def trackCsvtoKML(trackcsvfile, trackdata=None, saveOutput=True, outdir=None): + """ + Either reads a CSV file containing lat, long, height of an event and + creates a 3d KML file from it or, if trackdata is populated, converts a Pandas dataframe containing + the same data. Output is written to disk unless saveOutput is false. + + Arguments: + trackcsvfile: [string] full path to the file to read from + trackdata: [array] pandas dataframe containing the data. Default None + saveOutput: [bool] write the KML file to disk. Default true + outdir: [string] where to save the file. Default same folder as the source file + + Returns: + the KML file as a tuple + """ + kml=simplekml.Kml() + kml.document.name = trackcsvfile + if trackdata is None: + inputfile = csv.reader(open(trackcsvfile)) + for row in inputfile: + #columns are lat, long, height, times + kml.newpoint(name='', coords=[(row[1], row[0], row[2])]) + else: + for i,r in trackdata.iterrows(): + kml.newpoint(name=f'{r[3]:.5f}', coords=[(r[1], r[0], r[2])], extrude=1, altitudemode='absolute') + if 'csv' in trackcsvfile: + outname = trackcsvfile.replace('.csv','.kml') + else: + outname = f'{trackcsvfile}.kml' + if saveOutput: + if outdir is None: + outdir, _ = os.path.split(trackcsvfile) + os.makedirs(outdir, exist_ok=True) + outname = os.path.join(outdir, outname) + kml.save(outname) + return kml + + +def trackKMLtoCsv(kmlfile, kmldata = None, saveOutput=True, outdir=None): + """ convert a track KML retrieved by ukmondb.trajectoryKML to a 3 dimensional CSV file + containing coordinates of points on the trajectory. + + Arguments: + kmlfile [string] full path to the KML file to read from. + kmldata [kml] the kml data if available. + saveOutput [bool] Default True, save the output to file. + outdir [string] where to save to if saveOutput is True. + + Note: if kmldata is supplied, then kmlfile is ignored. + + Returns: + a Pandas dataframe containing the lat, long, alt and time of each + point on the trajectory, sorted by time. + + """ + with open(kmlfile) as fd: + x = xmltodict.parse(fd.read()) + placemarks=x['kml']['Document']['Placemark'] + lats = [] + lons = [] + alts = [] + tims = [] + for pm in placemarks: + tims.append(float(pm['name'])) + coords = pm['Point']['coordinates'].split(',') + lons.append(float(coords[0])) + lats.append(float(coords[1])) + alts.append(float(coords[2])) + df = pd.DataFrame({"lats": lats, "lons": lons, "alts": alts, "times": tims}) + df = df.sort_values(by=['times', 'lats']) + if saveOutput: + fname = kmlfile + if outdir is None: + outdir, fname = os.path.split(kmlfile) + outf = os.path.join(outdir, fname).replace('.kml', '.csv').replace('.KML','.csv') + df.to_csv(outf, index=False) + return df + + +def getTrackDetails(traj): + """ Get track details from a WMPL trajectory object + + Arguments: + traj: a WMPL trajectory object containing observations + + Returns: + a Pandas dataframe containing the lat, long, alt and time of each point on the trajectory, sorted by time. + """ + lats = [] + lons = [] + alts = [] + lens = [] + # Go through observation from all stations + for obs in traj.observations: + # Go through all observed points + for i in range(obs.kmeas): + lats.append(np.degrees(obs.model_lat[i])) + lons.append(np.degrees(obs.model_lon[i])) + alts.append(obs.model_ht[i]) + lens.append(obs.time_data[i]) + df = pd.DataFrame({"lats": lats, "lons": lons, "alts": alts, "times": lens}) + df = df.sort_values(by=['times', 'lats']) + return df From 66a7fa4843f18b06d7e868263745a44b039d8727 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 18:36:56 +0100 Subject: [PATCH 163/287] bugfix in createannualbarchart --- archive/ukmon_pylib/reports/createAnnualBarChart.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index d628fc9a..e51facc5 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -36,13 +36,14 @@ def createBarChart(datadir=None, yr=None): dts.append(dt) fig, ax = plt.subplots() - width = 0.35 + width = 0.35 nowdt=datetime.datetime.now() ax.set_ylabel('# matches') ax.set_xlabel('Date') ax.set_title('Number of matched events per day. Last updated {}'.format(nowdt.strftime('%Y-%m-%d %H:%M:%S'))) - li.append(0) # comes up one short + if len(dts) > len(li): + li.append(0) # comes up one short ax.bar(dts, li, width, label='Events') fig = plt.gcf() fig.set_size_inches(12, 5) From d55eb05decf01518b71e328d86c945a41fbdce74 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 18:39:19 +0100 Subject: [PATCH 164/287] working on bug --- archive/ukmon_pylib/reports/createAnnualBarChart.py | 1 + 1 file changed, 1 insertion(+) diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index e51facc5..9cd8676b 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -44,6 +44,7 @@ def createBarChart(datadir=None, yr=None): if len(dts) > len(li): li.append(0) # comes up one short + print(len(dts), len(li)) ax.bar(dts, li, width, label='Events') fig = plt.gcf() fig.set_size_inches(12, 5) From e074739b4be418502423b685d072f69ed69fea4c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 18:40:45 +0100 Subject: [PATCH 165/287] more work on lenght bug --- archive/ukmon_pylib/reports/createAnnualBarChart.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 9cd8676b..54878a77 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -42,8 +42,9 @@ def createBarChart(datadir=None, yr=None): ax.set_xlabel('Date') ax.set_title('Number of matched events per day. Last updated {}'.format(nowdt.strftime('%Y-%m-%d %H:%M:%S'))) - if len(dts) > len(li): - li.append(0) # comes up one short + while len(li) < len(dts): + li.append(0) + print(len(dts), len(li)) ax.bar(dts, li, width, label='Events') fig = plt.gcf() From c944a17cc017732b29176bbabe0bd039e460acf2 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 18:44:35 +0100 Subject: [PATCH 166/287] remove some debugging --- archive/ukmon_pylib/reports/createAnnualBarChart.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 54878a77..b7ac9fca 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -44,8 +44,6 @@ def createBarChart(datadir=None, yr=None): while len(li) < len(dts): li.append(0) - - print(len(dts), len(li)) ax.bar(dts, li, width, label='Events') fig = plt.gcf() fig.set_size_inches(12, 5) From a069e8aabda0f1aafa90eda05502b2ce39d15352 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 18:48:17 +0100 Subject: [PATCH 167/287] oops forgot to push the fixed version of createReportIndex --- archive/website/createReportIndex.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/website/createReportIndex.sh b/archive/website/createReportIndex.sh index edf24c4f..f758ae1d 100644 --- a/archive/website/createReportIndex.sh +++ b/archive/website/createReportIndex.sh @@ -60,7 +60,7 @@ if [ -f $curryr/tmp.txt ] ; then rm -f $curryr/tmp.txt ; fi aws s3 ls $WEBSITEBUCKET/reports/$curryr/ | grep PRE | egrep -v "ALL|orbits|stations|fireballs|showers|.js|.html" | awk '{print $2 }' | while read j do - python -c "from meteortools.utils import getShowerDets ; x=getShowerDets('${j:0:3}', True); print(x)" >> $curryr/tmp.txt + python -c "from utils.getActiveShowers import getShowerDets ; x=getShowerDets('${j:0:3}', True); print(x)" >> $curryr/tmp.txt done sort -n $curryr/tmp.txt > $curryr/shwrs.txt rm -f $curryr/tmp.txt From d21bdd0e88c3b3bc9da1dacde91cc843013001f4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 19:03:53 +0100 Subject: [PATCH 168/287] segregating function to get list of images that weren't calibrated by RMS --- archive/analysis/updatePlotsAndDetStatus.sh | 2 +- .../ukmon_pylib/analysis/gatherDetectionData.py | 12 ------------ archive/ukmon_pylib/utils/getUncalImages.py | 16 ++++++++++++++++ 3 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 archive/ukmon_pylib/utils/getUncalImages.py diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index 937f7bb8..158942ea 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -83,7 +83,7 @@ aws ec2 stop-instances --instance-ids $SERVERINSTANCEID log2cw $NJLOGGRP $NJLOGSTREAM "get a list of uncalibrated data" updatePlotsAndDetStatus aws s3 sync $UKMONSHAREDBUCKET/matches/consumed/ $DATADIR/single/used/ --exclude "*" --include "*.txt" --quiet rundate=$(cat $DATADIR/rundate.txt) -python -c "from analysis.gatherDetectionData import getUncalibratedImageList;getUncalibratedImageList('$rundate');" +python -c "from utils.getUnCalImages import getUncalibratedImageList;getUncalibratedImageList('$rundate');" # and then clear the profile again unset AWS_PROFILE diff --git a/archive/ukmon_pylib/analysis/gatherDetectionData.py b/archive/ukmon_pylib/analysis/gatherDetectionData.py index d7e45b9b..bc8be752 100644 --- a/archive/ukmon_pylib/analysis/gatherDetectionData.py +++ b/archive/ukmon_pylib/analysis/gatherDetectionData.py @@ -11,18 +11,6 @@ from meteortools.ukmondb import getECSVs, getLiveJpgs, getFBfiles, getDetections -def getUncalibratedImageList(dtstr=None): - datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') - flines = open(logfile, 'r').readlines() - uncal = [f for f in flines if 'not recalibrated' in f] - imglist = [x.strip(' ').split(' ')[1][:-1] for x in uncal] - with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: - for li in imglist: - outf.write(f'{li}\n') - return imglist - - def getRawData(idlist, outpth): for _,row in idlist.iterrows(): stat =row.ID diff --git a/archive/ukmon_pylib/utils/getUncalImages.py b/archive/ukmon_pylib/utils/getUncalImages.py new file mode 100644 index 00000000..58c890ff --- /dev/null +++ b/archive/ukmon_pylib/utils/getUncalImages.py @@ -0,0 +1,16 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# + +import os + + +def getUncalibratedImageList(dtstr=None): + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') + flines = open(logfile, 'r').readlines() + uncal = [f for f in flines if 'not recalibrated' in f] + imglist = [x.strip(' ').split(' ')[1][:-1] for x in uncal] + with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: + for li in imglist: + outf.write(f'{li}\n') + return imglist From 5fc792094b5c826401c4e2076477c64ef03f624f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 19:09:07 +0100 Subject: [PATCH 169/287] reorganize stuff a bit --- archive/cronjobs/captureBright.sh | 2 +- archive/ukmon_pylib/analysis/README.md | 6 +++--- archive/ukmon_pylib/{analysis => utils}/analyseGmnData.py | 0 .../{analysis => utils}/compareBrightnessData.py | 0 .../ukmon_pylib/{analysis => utils}/gatherDetectionData.py | 0 archive/ukmon_pylib/{analysis => utils}/meteorFlux.py | 0 6 files changed, 4 insertions(+), 4 deletions(-) rename archive/ukmon_pylib/{analysis => utils}/analyseGmnData.py (100%) rename archive/ukmon_pylib/{analysis => utils}/compareBrightnessData.py (100%) rename archive/ukmon_pylib/{analysis => utils}/gatherDetectionData.py (100%) rename archive/ukmon_pylib/{analysis => utils}/meteorFlux.py (100%) diff --git a/archive/cronjobs/captureBright.sh b/archive/cronjobs/captureBright.sh index 06ce1561..9644b6b2 100644 --- a/archive/cronjobs/captureBright.sh +++ b/archive/cronjobs/captureBright.sh @@ -6,7 +6,7 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} cd $DATADIR/brightness rundt=$(date -d "yesterday" +%Y%m%d) -python -m analysis.compareBrightnessData $rundt +python -m utils.compareBrightnessData $rundt mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD use ukmon; diff --git a/archive/ukmon_pylib/analysis/README.md b/archive/ukmon_pylib/analysis/README.md index 732ae6a4..5cf38005 100644 --- a/archive/ukmon_pylib/analysis/README.md +++ b/archive/ukmon_pylib/analysis/README.md @@ -4,6 +4,6 @@ This folder contains functions to analyse the single station and matched data. * ShowerAnalysis.py - analysis for a named shower for a specific year eg GEM 2022 * stationAnalysis.py - analysis for a named year or year+month, for one or all stations. -* meteorFlux - plot a graph of meteor flux -* gatherDetectionData.py -collects all available data on a detection or detections matching a pattern YYYYMMDD_HHMMSS. If the seconds are omitted, then all detections within the specified minute will be returned. - +* summaryAnalysis.py - summary analysis for a given period +* meteorFlux.py - plot a graph of meteor flux + diff --git a/archive/ukmon_pylib/analysis/analyseGmnData.py b/archive/ukmon_pylib/utils/analyseGmnData.py similarity index 100% rename from archive/ukmon_pylib/analysis/analyseGmnData.py rename to archive/ukmon_pylib/utils/analyseGmnData.py diff --git a/archive/ukmon_pylib/analysis/compareBrightnessData.py b/archive/ukmon_pylib/utils/compareBrightnessData.py similarity index 100% rename from archive/ukmon_pylib/analysis/compareBrightnessData.py rename to archive/ukmon_pylib/utils/compareBrightnessData.py diff --git a/archive/ukmon_pylib/analysis/gatherDetectionData.py b/archive/ukmon_pylib/utils/gatherDetectionData.py similarity index 100% rename from archive/ukmon_pylib/analysis/gatherDetectionData.py rename to archive/ukmon_pylib/utils/gatherDetectionData.py diff --git a/archive/ukmon_pylib/analysis/meteorFlux.py b/archive/ukmon_pylib/utils/meteorFlux.py similarity index 100% rename from archive/ukmon_pylib/analysis/meteorFlux.py rename to archive/ukmon_pylib/utils/meteorFlux.py From 0a8d20331ccfcd297972bd1d78f46ff350317a16 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 20:51:18 +0100 Subject: [PATCH 170/287] add email sending functionality to ukmon_pylib --- archive/ukmon_pylib/utils/sendAnEmail.py | 192 +++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 archive/ukmon_pylib/utils/sendAnEmail.py diff --git a/archive/ukmon_pylib/utils/sendAnEmail.py b/archive/ukmon_pylib/utils/sendAnEmail.py new file mode 100644 index 00000000..a7d215fb --- /dev/null +++ b/archive/ukmon_pylib/utils/sendAnEmail.py @@ -0,0 +1,192 @@ +# Copyright (C) 2018-2023 Mark McIntyre +import os +import platform +import base64 +import email +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart + +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from google_auth_oauthlib.flow import InstalledAppFlow +from googleapiclient.discovery import build + +SCOPES =['https://mail.google.com/'] + + +def _getGmailCreds(tokfile=None, crdfile=None): + creds = None + # The token stores the user's access and refresh tokens, and is + # created automatically when the authorization flow completes for the first + # time. + if tokfile is None: + tokfile = '~/.ssh/gmailtoken.json' + if crdfile is None: + crdfile = '~/.ssh/gmailcreds.json' + tokfile = os.path.expanduser(tokfile) + crdfile = os.path.expanduser(crdfile) + if os.path.exists(tokfile): + creds = Credentials.from_authorized_user_file(tokfile, SCOPES) + + # If there are no (valid) credentials available, ask the user to log in interactively. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + if not os.path.isfile(crdfile): + print(f'to use this function you must have stored your Google OAUTH2 secret json file in {crdfile}') + print('To set up OAUTH2 go to your google cloud console, select APIs then Credentials, and add an OAUTH2 desktop client ID.') + return None + flow = InstalledAppFlow.from_client_secrets_file(crdfile, SCOPES) + creds = flow.run_local_server(port=0) + # Save the credentials for the next run + with open(tokfile, 'w') as token: + token.write(creds.to_json()) + return creds + + +def _refreshCreds(tokfile=None, crdfile=None): + creds = None + if tokfile is None: + tokfile = '~/.ssh/gmailtoken.json' + if crdfile is None: + crdfile = '~/.ssh/gmailcreds.json' + tokfile = os.path.expanduser(tokfile) + crdfile = os.path.expanduser(crdfile) + if os.path.exists(tokfile): + creds = Credentials.from_authorized_user_file(tokfile, SCOPES) + + # If there are no (valid) credentials available, let the user log in. + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + try: + creds.refresh(Request()) + except: + flow = InstalledAppFlow.from_client_secrets_file(crdfile, SCOPES) + creds = flow.run_local_server(port=0) + # Save the credentials for the next run + if creds.valid: + with open(tokfile, 'w') as token: + token.write(creds.to_json()) + else: + print('credentials not valid, try again') + return creds + + +def _create_message(sender, to, subject, message_text, msg_text_html=None): + if msg_text_html is None: + msg = MIMEText(message_text) + else: + msg = MIMEMultipart('alternative') + msg.attach(MIMEText(message_text, 'plain')) + msg.attach(MIMEText(msg_text_html, 'html')) # html must be the last part + msg['to'] = to + msg['from'] = sender + msg['subject'] = subject + return {'raw': base64.urlsafe_b64encode(msg.as_string().encode('utf-8')).decode('utf-8')} + + + +def sendAnEmail(mailrecip, message, subject, mailfrom, files=None, tokfile=None, crdfile=None, msg_html=None): + """ sends an email using gmail. + + Arguments: + mailrecip: [string] email address of recipient. + message: [string] the message to send. + subject: [string] Subject line. + mailfrom: [string] email address of sender. + Keyword Args: + files: [list] list of files to attach, not currently implemented. + tokfile: [string] full path to token file, if not ~/.ssh/gmailtoken.json + crdfile: [string] full path to credentials file, if not ~/.ssh/gmailcreds.json + msg_html: [string] HTML version of the message body, if any + + Returns: + Nothing, though a message is printed onscreen. + + Notes: + You must have gmail OAUTH2 set up. The gmail credentials default to gmailtoken.json and + gmailcreds.json in the $HOME/.ssh folder. + """ + + if subject is None: + subject = platform.uname()[1] + + # email a summary to the mailrecip + creds = _getGmailCreds(tokfile, crdfile) + if not creds: + return + service = build('gmail', 'v1', credentials=creds) + + subj = subject + mailmsg = _create_message(mailfrom, mailrecip, subj, message, msg_text_html=msg_html) + + try: + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + print('Message Id: %s' % retval['id']) + except: + print('An error occurred sending the message') + + +def forwardAnEmail(reciplist, msgid=None, tokfile=None, crdfile=None): + """ Forward email from a gmail account to a list of recipients + + Arguments: + recplist: [list of strings] list of addresses to forward to + + Keyword arguments: + msgid: [string] gmail message id. If None, then all unread mail is forwarded + tokfile: [string] full path to a gmail oauth2 json token file + crdfile: [string] full path to a gmail oauth2 json credentials file for initial authorization + + To obtain the oauth2 credentials file, go to the google cloud console, select APIs and enable gmail. Then + go to Credentials and create an OAUTH2 Client ID for Desktop and download the JSON credentials file. + Upon first run, you'll be taken to the gmail authorisation screen and the token file will be created. + Subsequent runs will use the token file. The creds file will be used to reauthorise periodically. + """ + creds = _getGmailCreds(tokfile, crdfile) + if not creds: + return + service = build('gmail', 'v1', credentials=creds) + userid = 'me' + if msgid is None: + try: + msglist = (service.users().messages().list(userId=userid, labelIds=['INBOX','UNREAD']).execute()) + for msg in msglist['messages']: + msgid = msg['id'] + # retrieve raw mail + message = (service.users().messages().get(userId='me', id=msgid, format='raw').execute()) + # decode it from base64 + decmsg = base64.urlsafe_b64decode(message['raw']) + newmsg = email.message_from_bytes(decmsg) + newmsg.add_header('Reply-To',newmsg['from']) + newmsg.replace_header('Subject','Fwd: ' + newmsg['subject']) + # Send it + for recip in reciplist: + newmsg.replace_header('To', recip) + mailmsg = {'raw': base64.urlsafe_b64encode(newmsg.as_string().encode('utf-8')).decode('utf-8')} + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + # This will mark the messagea as read + service.users().messages().modify(userId=userid, id=msgid, body={'removeLabelIds': ['UNREAD']}).execute() + print(retval) + except Exception: + print('Nothing to forward') + else: + try: + message = (service.users().messages().get(userId='me', id=msgid, format='raw').execute()) + # decode it from base64 + decmsg = base64.urlsafe_b64decode(message['raw']) + newmsg = email.message_from_bytes(decmsg) + newmsg.add_header('Reply-To',newmsg['from']) + newmsg.replace_header('Subject','Fwd: ' + newmsg['subject']) + # Send it + for recip in reciplist: + newmsg.replace_header('To', recip) + mailmsg = {'raw': base64.urlsafe_b64encode(newmsg.as_string().encode('utf-8')).decode('utf-8')} + retval = (service.users().messages().send(userId='me', body=mailmsg).execute()) + # This will mark the messagea as read + service.users().messages().modify(userId=userid, id=msgid, body={'removeLabelIds': ['UNREAD']}).execute() + print(retval) + except Exception: + print('Nothing to forward') + return From 12a05c3204a99c4093bc1f4878213e792801c870 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 20:56:38 +0100 Subject: [PATCH 171/287] use send email functionality in code --- archive/analysis/findAllMatches.sh | 2 +- archive/cronjobs/nightlyJob.sh | 10 ++++++---- archive/ukmon_pylib/analysis/README.md | 3 +-- archive/ukmon_pylib/reports/dailyReport.py | 2 +- archive/ukmon_pylib/reports/reportBadCameras.py | 2 +- archive/utils/statsToMqtt.sh | 4 ++-- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 2fba436f..b52fc2ae 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -65,7 +65,7 @@ success=$(grep "Total run time:" $SRC/logs/matchJob.log) if [ "$success" == "" ] then - python -c "from meteortools.utils import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')" + python -c "from utils.sendAnEmail import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')" echo problems with solver fi log2cw $NJLOGGRP $NJLOGSTREAM "Solving Run Done" findAllMatches diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index 1c9eb120..7d16b7f1 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -66,7 +66,7 @@ ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" nightlyJob $SRC/website/updateIndexPages.sh -# FIRST CONSOLIDATE THE DATA AND CREATE SEARCH INDEXES +# CONSOLIDATE THE DATA AND CREATE SEARCH INDEXES # consolidate the output of the match process for further analysos log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob $SRC/analysis/consolidateOutput.sh ${yr} @@ -79,8 +79,10 @@ fi log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 2" nightlyJob $SRC/analysis/createSearchable.sh $yr matches +############################################################ # FROM HERE DOWN WE'RE CREATING REPORTS - +# and everything can be rerun safely if the data are present +############################################################ # add daily report to the website log2cw $NJLOGGRP $NJLOGSTREAM "start publishDailyReport" nightlyJob $SRC/website/publishDailyReport.sh @@ -129,12 +131,12 @@ ${SRC}/website/cameraStatusReport.sh $rundate log2cw $NJLOGGRP $NJLOGSTREAM "start createExchangeFiles" nightlyJob python -c "from reports.createExchangeFiles import createAll;createAll();" -aws s3 sync $DATADIR/browse/daily/ $WEBSITEBUCKET/browse/daily/ --region eu-west-2 --quiet +aws s3 sync $DATADIR/browse/daily/ $WEBSITEBUCKET/browse/daily/ --quiet cd $DATADIR # do this manually when on PC required as it requires too much memory for the batch server; closes #61 # python $PYLIB/maintenance/plotStationsOnMap.py False -aws s3 cp $DATADIR/stations.png $WEBSITEBUCKET/ --region eu-west-2 --quiet +aws s3 cp $DATADIR/stations.png $WEBSITEBUCKET/ --quiet rm -f $SRC/data/.nightly_running diff --git a/archive/ukmon_pylib/analysis/README.md b/archive/ukmon_pylib/analysis/README.md index 5cf38005..a4b09f7d 100644 --- a/archive/ukmon_pylib/analysis/README.md +++ b/archive/ukmon_pylib/analysis/README.md @@ -1,9 +1,8 @@ # Analysis -This folder contains functions to analyse the single station and matched data. +This folder contains functions to analyse the single station and matched data for website reporting. * ShowerAnalysis.py - analysis for a named shower for a specific year eg GEM 2022 * stationAnalysis.py - analysis for a named year or year+month, for one or all stations. * summaryAnalysis.py - summary analysis for a given period -* meteorFlux.py - plot a graph of meteor flux diff --git a/archive/ukmon_pylib/reports/dailyReport.py b/archive/ukmon_pylib/reports/dailyReport.py index a511c405..69f6eb64 100644 --- a/archive/ukmon_pylib/reports/dailyReport.py +++ b/archive/ukmon_pylib/reports/dailyReport.py @@ -6,7 +6,7 @@ import sys import datetime -from meteortools.utils import sendAnEmail +from utils.sendAnEmail import sendAnEmail def AddHeader(body, bodytext, stats): diff --git a/archive/ukmon_pylib/reports/reportBadCameras.py b/archive/ukmon_pylib/reports/reportBadCameras.py index cc0938ee..adf4d64c 100644 --- a/archive/ukmon_pylib/reports/reportBadCameras.py +++ b/archive/ukmon_pylib/reports/reportBadCameras.py @@ -10,7 +10,7 @@ import textwrap from reports.CameraDetails import loadLocationDetails -from meteortools.utils import sendAnEmail +from utils.sendAnEmail import sendAnEmail mailfrom = 'ukmonhelper2@ukmeteors.co.uk' diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh index e2334bfe..170ebb0b 100644 --- a/archive/utils/statsToMqtt.sh +++ b/archive/utils/statsToMqtt.sh @@ -10,7 +10,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then msg="$(hostname) diskspace was ${diskpct}%" subj="Diskspace alert for $(hostname)" hn="$(hostname)@aws" - python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" + python -c "from utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi elif [ "$(hostname)" == "ukmcalcserver" ] ; then source $HOME/venvs/wmpl/bin/activate @@ -31,6 +31,6 @@ else msg="$(hostname) diskspace is ${diskpct}%" subj="Diskspace alert for $(hostname)" hn="$(hostname)@aws" - python -c "from meteortools.utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" + python -c "from utils.sendAnEmail import sendAnEmail;sendAnEmail('markmcintyre99@googlemail.com', '$msg', '$subj', '$hn')" fi fi From cfd7111aa8ba10a919f96490c62f53d9ed30a884 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 21:23:11 +0100 Subject: [PATCH 172/287] add database and gmaps keys/passwords as secure variables --- archive/terraform/ukmda/dev_ssm_variables.tf | 21 ++++++++++++++++++++ archive/terraform/ukmda/ssm_variables.tf | 20 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/archive/terraform/ukmda/dev_ssm_variables.tf b/archive/terraform/ukmda/dev_ssm_variables.tf index fcfaed25..bca54359 100644 --- a/archive/terraform/ukmda/dev_ssm_variables.tf +++ b/archive/terraform/ukmda/dev_ssm_variables.tf @@ -32,6 +32,17 @@ resource "aws_ssm_parameter" "dev_dbpw" { } } +resource "aws_ssm_parameter" "dev_rootdbpw" { + provider = aws.eu-west-1-prov + name = "dev_rootdbpw" + type = "SecureString" + value = "Wombat33mdb" + tags = { + "billingtag" = "ukmda" + } +} + + resource "aws_ssm_parameter" "dev_dbuser" { provider = aws.eu-west-1-prov name = "dev_dbuser" @@ -168,3 +179,13 @@ resource "aws_ssm_parameter" "dev_batchloggroup" { "billingtag" = "ukmda" } } + +resource "aws_ssm_parameter" "dev_gmapsapikey" { + name = "dev_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmda" + } +} + diff --git a/archive/terraform/ukmda/ssm_variables.tf b/archive/terraform/ukmda/ssm_variables.tf index 11b3fdaa..690de73b 100644 --- a/archive/terraform/ukmda/ssm_variables.tf +++ b/archive/terraform/ukmda/ssm_variables.tf @@ -32,6 +32,16 @@ resource "aws_ssm_parameter" "prod_dbpw" { } } +resource "aws_ssm_parameter" "prod_rootdbpw" { + provider = aws.eu-west-1-prov + name = "prod_rootdbpw" + type = "SecureString" + value = "Wombat33mdb" + tags = { + "billingtag" = "ukmda" + } +} + resource "aws_ssm_parameter" "prod_dbuser" { provider = aws.eu-west-1-prov name = "prod_dbuser" @@ -168,3 +178,13 @@ resource "aws_ssm_parameter" "prod_batchloggroup" { "billingtag" = "ukmda" } } + +resource "aws_ssm_parameter" "prod_gmapsapikey" { + name = "prod_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmda" + } +} + From 2e7d4b277dc1638822abbd1114d05216f280690e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 21:23:34 +0100 Subject: [PATCH 173/287] update to use secure keys --- .../ukmon_pylib/reports/makeCoverageMap.py | 21 +++++++++++++++---- archive/utils/makeConfig.sh | 4 +--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index ca73a073..413b3c9a 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -6,6 +6,7 @@ import os import gmplot import glob +import boto3 from cryptography.fernet import Fernet from utils.kmlHandlers import readCameraKML @@ -24,9 +25,20 @@ def encodeApiKey(plainkey): return apikey.decode('utf-8') +def getApiKey(): + ssm = boto3.client('ssm') + res = ssm.get_parameter(Name='prod_gmapsapikey', WithDecryption=True) + if 'Parameter' in res: + return res['Parameter']['Value'] + else: + return False + + def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): - apikey = os.getenv('APIKEY') - apikey = decodeApiKey(apikey) + + apikey = getApiKey() + if not apikey: + return kmltempl = os.getenv('KMLTEMPLATE', default='*70km.kml') heightval = kmltempl[1:-4] @@ -56,8 +68,9 @@ def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): def createCoveragePage(): - apikey = os.getenv('APIKEY') - apikey = decodeApiKey(apikey) + apikey = getApiKey() + if not apikey: + return templdir = os.getenv('TEMPLATES', default=os.path.expanduser('~/prod/website/templates')) datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) with open(os.path.join(templdir, 'coverage-maps.html'), 'r') as inf: diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index c8cc9e14..50ffc086 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -34,7 +34,6 @@ MATCHSTART=2 MATCHEND=0 RMS_ENV=RMS WMPL_ENV=wmpl -APIKEY=$(cat ~/.ssh/gmapsapikey) KMLTEMPLATE=*70km.kml # variables that can be used in scripts and python files to ensure test data doesnt overwrite live data @@ -71,7 +70,6 @@ echo "WMPL_LOC=${WMPL_LOC}" >> ${CFGFILE} echo "MATCHSTART=${MATCHSTART}" >> ${CFGFILE} echo "MATCHEND=${MATCHEND}" >> ${CFGFILE} echo "SERVERSSHKEY=${SERVERSSHKEY}" >> ${CFGFILE} -echo "APIKEY=${APIKEY}" >> ${CFGFILE} echo "KMLTEMPLATE=${KMLTEMPLATE}" >> ${CFGFILE} echo "NJLOGGRP=${NJLOGGRP}" >> ${CFGFILE} echo "TESTMODE=${TESTMODE}" >> ${CFGFILE} @@ -83,7 +81,7 @@ echo "export PYLIB TEMPLATES DATADIR AWS_DEFAULT_REGION" >> ${CFGFILE} echo "export RMS_ENV RMS_LOC WMPL_ENV WMPL_LOC" >> ${CFGFILE} echo "export PYTHONPATH=${RMS_LOC}:${WMPL_LOC}:${PYLIB}:${SRC}/share" >> ${CFGFILE} echo "export MATCHSTART MATCHEND SERVERSSHKEY" >> ${CFGFILE} -echo "export APIKEY KMLTEMPLATE SERVERINSTANCEID SERVERUSERID CALCSERVERIP NJLOGGRP" >> ${CFGFILE} +echo "export KMLTEMPLATE SERVERINSTANCEID SERVERUSERID CALCSERVERIP NJLOGGRP" >> ${CFGFILE} echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib" >> ${CFGFILE} echo "export TESTMODE TESTSUFF" >> ${CFGFILE} From 3d511d33a72f9df78dc8d0a241455ef79be591df Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 21:23:52 +0100 Subject: [PATCH 174/287] remove dbpw from environment --- archive/utils/loadMatchCsvMDB.sh | 3 ++- archive/utils/loadSingleCsvMDB.sh | 3 ++- archive/utils/updateDb.sh | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/archive/utils/loadMatchCsvMDB.sh b/archive/utils/loadMatchCsvMDB.sh index b2fd845f..884706b8 100644 --- a/archive/utils/loadMatchCsvMDB.sh +++ b/archive/utils/loadMatchCsvMDB.sh @@ -17,7 +17,8 @@ if [ ! -f $csvname ] ; then echo "$csvname not found" ; exit ; fi echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch -passwd=$(cat ~/.ssh/db_batch.passwd ) +# leading space to prevent being inserted into environment + passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) mysql -u${user} -p${passwd} << EOD select count(*) from ukmon.matches; LOAD DATA LOCAL INFILE '$csvname' diff --git a/archive/utils/loadSingleCsvMDB.sh b/archive/utils/loadSingleCsvMDB.sh index 09fdaaaa..9b7910bc 100644 --- a/archive/utils/loadSingleCsvMDB.sh +++ b/archive/utils/loadSingleCsvMDB.sh @@ -17,7 +17,8 @@ if [ ! -f $csvname ] ; then echo "$csvname not found" ; exit ; fi echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch -passwd=$(cat ~/.ssh/db_batch.passwd ) +# leading space to prevent being inserted into environment + passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) mysql -u${user} -p${passwd} << EOD select count(*) from ukmon.singles; LOAD DATA LOCAL INFILE '$csvname' diff --git a/archive/utils/updateDb.sh b/archive/utils/updateDb.sh index 501497f0..904fa624 100644 --- a/archive/utils/updateDb.sh +++ b/archive/utils/updateDb.sh @@ -12,7 +12,9 @@ else fi cd ~/prod/data/brightness -mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD +# leading space to prevent being inserted into environment + passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) +mysql -u batch -p$passwd -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; load data local infile './CaptureNight_${rundt}.csv' From 1d4b6df4c965829c8116fc9ea43182d849ebba72 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 21:24:20 +0100 Subject: [PATCH 175/287] remove dbpw from environment --- archive/cronjobs/captureBright.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/archive/cronjobs/captureBright.sh b/archive/cronjobs/captureBright.sh index 9644b6b2..e8ce844a 100644 --- a/archive/cronjobs/captureBright.sh +++ b/archive/cronjobs/captureBright.sh @@ -8,7 +8,9 @@ cd $DATADIR/brightness rundt=$(date -d "yesterday" +%Y%m%d) python -m utils.compareBrightnessData $rundt -mysql -u batch -p$(cat ~/.ssh/db_batch.passwd) -h localhost << EOD +# leading space to prevent being inserted into environment + pwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) +mysql -u batch -p$pwd -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; load data local infile '${DATADIR}/brightness/CaptureNight_${rundt}.csv' From 67b026c8c4e8f261d3b120843c16942430adb2bf Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 21:27:05 +0100 Subject: [PATCH 176/287] add script to retrieve db passwd --- archive/utils/getDbPwd.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 archive/utils/getDbPwd.sh diff --git a/archive/utils/getDbPwd.sh b/archive/utils/getDbPwd.sh new file mode 100644 index 00000000..fd749db6 --- /dev/null +++ b/archive/utils/getDbPwd.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright (C) 2018- Mark McIntyre + +if [ "$1" == "root" ] then + passwd=$(aws ssm get-parameters --names prod_rootdbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) +else + passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) +fi +echo $passwd +unset passwd From b283c2f2dea54d49353805fa308d72697ab35905 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 21:48:52 +0100 Subject: [PATCH 177/287] add script to retrieve db passwords --- archive/utils/getDbPwd.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/utils/getDbPwd.sh b/archive/utils/getDbPwd.sh index fd749db6..cbbc5212 100644 --- a/archive/utils/getDbPwd.sh +++ b/archive/utils/getDbPwd.sh @@ -1,7 +1,8 @@ #!/bin/bash # Copyright (C) 2018- Mark McIntyre -if [ "$1" == "root" ] then +if [ "$1" == "root" ] +then passwd=$(aws ssm get-parameters --names prod_rootdbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) else passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) From 9e212e505a6fcf5ed144faa7c7e26f6f16b070be Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 21:50:25 +0100 Subject: [PATCH 178/287] bug in createTables --- archive/database/ddl/create_tables.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/database/ddl/create_tables.sql b/archive/database/ddl/create_tables.sql index c5102756..ba036622 100644 --- a/archive/database/ddl/create_tables.sql +++ b/archive/database/ddl/create_tables.sql @@ -28,8 +28,9 @@ create table singles( AngVel float, Shwr varchar(6), filname varchar(64), - Dtstamp double(18,6)), - status varchar(3); + Dtstamp double(18,6), + status varchar(3) + ); alter ignore table ukmon.singles add unique uniq_Dtstamp (Dtstamp); From 814368f85b773ab74a2b8545bb6e01feb56d73c6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 22:13:38 +0100 Subject: [PATCH 179/287] updates to migration instructions --- archive/database/Setup.md | 2 +- archive/server_setup/migratingBatchServer.md | 33 +++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/archive/database/Setup.md b/archive/database/Setup.md index 1aef81b1..84dff4cf 100644 --- a/archive/database/Setup.md +++ b/archive/database/Setup.md @@ -29,7 +29,7 @@ make sure you've installed mariadb on the new server and created databases and u make sure you can ssh from old to new then on the old server run ``` -mysqldump -u batch -p'passwd' ukmon | ssh ukmonhelper2 mysql -u batch -p'passwd' ukmon +mysqldump -u batch -p'passwd' ukmon | ssh newserver mysql -u batch -p'passwd' ukmon ``` ## Connecting from Python diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index e4b5ed8a..751a540b 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -54,6 +54,16 @@ aws ssm get-parameters --region eu-west-2 --names prod_siteurl --query Parameter "https://archive.ukmeteors.co.uk" ``` +### SSH and other API keys +copy the contents of ~/.ssh on the old server, and make sure permissions are correct (should all be 0600). At a minimum you need the following files +``` bash +gmailcreds.json +gmailtoken.json +``` +These are used to authenticate against gmail. The keys are currently specific to Mark McIntyre's account and will need to be replaced with keys for `ukmeteors@gmail.com` + +You may also need the `github` keys though this depends on how you authenticate with GitHub. + ## data Replicate `~/prod/data` to the new server. (once for prod and once for dev). This will have to be repeated every day till golive. ``` bash @@ -66,7 +76,28 @@ rsync -avz ukmonhelper2:keymgmt/ ./keymgmt ``` ## Mariadb database -TBC +Sudo to root and run the following to set a root password +``` bash +mysql -u root -p +ALTER USER 'root'@'localhost' IDENTIFIED VIA mysql_native_password USING PASSWORD('ROOTPASSWORD'); +FLUSH PRIVILEGES; +quit +``` +(obviously, replace `ROOTPASSWORD` with the password in the SSM variable `prod_dbpw`) + +Exit the root shell, and as a 'normal' user execute the following: +``` bash +cd $SRC/database/ +mysql -u root -pROOTPASSWORD < ddl/create_dbs_and_users.sql +mysql -u root -pROOTPASSWORD < ddl/create_tables.sql +mysql -u root -pROOTPASSWORD < ddl/create_brightness_table.sq +``` +Now dump and reload the databases from the old server. +Make sure the old server can connect to the new one then on the old server, run the following: +``` bash +mysqldump -u batch -p'passwd' ukmon | ssh newserver mysql -u batch -p'passwd' ukmon +``` + ### To install PyQt5 * needs at least 3GB memory so add 2GB swap if the server has less. From d2594c294ca3639bf50a187b3ab9771b03fa3aba Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 22:16:40 +0100 Subject: [PATCH 180/287] include pip installs in the script --- install_or_update.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/install_or_update.sh b/install_or_update.sh index 3b49e93c..f450fe71 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -29,6 +29,14 @@ mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajd mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs +if [ -f ~/.condaon ] +then + cd $here + source ~/.condaon + conda activate wmpl + pip install -r archive/ukmon_pylib/additional_requirements.txt +fi + # update the IMO and GMN meteor shower tables if missing if [ ! -f ~/${envname}/share/IMO_Working_Meteor_Shower_List.xml ] then From 1e79073605d324249f734fee3399768716977d5f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 22:22:08 +0100 Subject: [PATCH 181/287] tweak installer --- install_or_update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index f450fe71..16dcc108 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -29,7 +29,7 @@ mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajd mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs -if [ -f ~/.condaon ] +if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] then cd $here source ~/.condaon From 474434d86108f80fa4a7eecb197cd95d72fa56b3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 27 Apr 2026 23:32:41 +0100 Subject: [PATCH 182/287] remove glob.glob1 --- archive/ukmon_pylib/additional_requirements.txt | 1 - archive/ukmon_pylib/metrics/camMetrics.py | 4 ++-- archive/ukmon_pylib/reports/findFailedMatches.py | 2 +- archive/ukmon_pylib/reports/makeCoverageMap.py | 2 +- archive/ukmon_pylib/reports/reportActiveShowers.py | 4 ++-- archive/ukmon_pylib/traj/distributeCandidates.py | 2 +- archive/ukmon_pylib/traj/manualBulkReruns.py | 2 +- archive/ukmon_pylib/traj/pickleAnalyser.py | 2 +- 8 files changed, 9 insertions(+), 10 deletions(-) diff --git a/archive/ukmon_pylib/additional_requirements.txt b/archive/ukmon_pylib/additional_requirements.txt index 4752c6cf..c5942af2 100644 --- a/archive/ukmon_pylib/additional_requirements.txt +++ b/archive/ukmon_pylib/additional_requirements.txt @@ -16,7 +16,6 @@ pyproj pyswarms Shapely xmltodict -tweepy fastparquet cryptography python-crontab diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index e85ff3d6..04a09053 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -137,9 +137,9 @@ def backPopulate(stationid): s3bucket = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] basepath = os.path.expanduser('~/prod/ukmon-shared/matches/RMSCorrelate') - fldrs = glob.glob1(os.path.join(basepath, stationid), '*') + fldrs = glob.glob('*', root_dir=os.path.join(basepath, stationid)) for fldr in fldrs: - s3objects = glob.glob1(os.path.join(basepath, stationid. fldr), 'FTPd*') + s3objects = glob.glob('FTPd*', root_dir=os.path.join(basepath, stationid. fldr)) if len(s3objects) > 0: s3obj = s3objects[0] fullobj = f'matches/RMSCorrelate/{stationid}/{fldr}/{s3obj}' diff --git a/archive/ukmon_pylib/reports/findFailedMatches.py b/archive/ukmon_pylib/reports/findFailedMatches.py index c6005fe7..1b110a1c 100644 --- a/archive/ukmon_pylib/reports/findFailedMatches.py +++ b/archive/ukmon_pylib/reports/findFailedMatches.py @@ -46,7 +46,7 @@ def processOneLog(logfile, repfile): srcdir = os.getenv('SRC', default='.') logdir = os.path.join(srcdir, 'logs','distrib') - logs = glob.glob1(logdir, f'{repdt}*.log') + logs = glob.glob(f'{repdt}*.log', root_dir=logdir) for logf in logs: processOneLog(os.path.join(logdir, logf), reportfile) reportfile.close() diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index 413b3c9a..db5c8017 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -45,7 +45,7 @@ def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): gmap = gmplot.GoogleMapPlotter(52, -1.0, 5, apikey=apikey, title=f'Camera Coverage at {heightval}', map_type='satellite') - flist = glob.glob1(kmlsource, kmltempl) + flist = glob.glob(kmltempl, root_dir=kmlsource) cols = list(gmplot.color._HTML_COLOR_CODES.keys()) for col, fn in zip(cols,flist): diff --git a/archive/ukmon_pylib/reports/reportActiveShowers.py b/archive/ukmon_pylib/reports/reportActiveShowers.py index 596b7b73..bf2a171a 100644 --- a/archive/ukmon_pylib/reports/reportActiveShowers.py +++ b/archive/ukmon_pylib/reports/reportActiveShowers.py @@ -109,8 +109,8 @@ def createShowerIndexPage(dtstr, shwr, shwrname, outdir, datadir): outf.write('The graphs and histograms below show more information about the velocity, magnitude \n') outf.write('start and end altitude and other parameters. Click for larger view. \n') # add the charts and stuff - jpglist = glob.glob1(outdir, '*.jpg') - pnglist = glob.glob1(outdir, '*.png') + jpglist = glob.glob('*.jpg', root_dir=outdir) + pnglist = glob.glob('*.png', root_dir=outdir) outf.write('
\n') for j in jpglist: outf.write(f'\n') diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index e81d2f1e..9255534e 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -110,7 +110,7 @@ def distributeCandidates(rundate, srcdir, maxcount=20, istest=False): print(f'Reading from {srcdir}') # obtain a list of picklefiles and sort by name - flist = glob.glob1(srcdir, '*.pickle') + flist = glob.glob('*.pickle', root_dir=srcdir) if len(flist) == 0: print('no candidates to process') return False diff --git a/archive/ukmon_pylib/traj/manualBulkReruns.py b/archive/ukmon_pylib/traj/manualBulkReruns.py index 4d391af5..98dd19b9 100644 --- a/archive/ukmon_pylib/traj/manualBulkReruns.py +++ b/archive/ukmon_pylib/traj/manualBulkReruns.py @@ -200,7 +200,7 @@ def getOverlappingStations(datadir, stationid): checker = EventChecker(traj_constraints=trajcons) ppdir = os.path.join(datadir, 'consolidated','platepars') rp = loadPlatepar(ppdir, stationid) - ppfs = glob.glob1(ppdir, '*.json') + ppfs = glob.glob('*.json', root_dir=ppdir) overlaps=[] for ppf in ppfs: ppfname, _ = os.path.splitext(ppf) diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index 028f60fe..b9da8919 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -79,7 +79,7 @@ def getBestView(picklename): except Exception: pass beststatid = statids[vmags.index(bestvmag)] - imgfn = glob.glob1(outdir, '*{}*.jpg'.format(beststatid)) + imgfn = glob.glob(f'*{beststatid}*.jpg', root_dir=outdir) if len(imgfn) > 0: bestimg = imgfn[0] elif os.path.isfile(os.path.join(outdir, 'jpgs.lst')): From 2b1c332f1876f2dca8def9bffa4f17b73ded3be8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 16:34:27 +0100 Subject: [PATCH 183/287] move unused scripts - will delete later --- archive/server_setup/ssh-setup.sh | 15 --------------- .../server_setup/{ => unused}/install-cartopy.sh | 0 archive/server_setup/{ => unused}/install-geos.sh | 0 .../server_setup/{ => unused}/install-proj4.sh | 0 .../server_setup/{ => unused}/install-sqlite3.sh | 0 .../server_setup/{ => unused}/install_opencv.sh | 0 archive/server_setup/{ => unused}/remount-s3.sh | 0 7 files changed, 15 deletions(-) delete mode 100644 archive/server_setup/ssh-setup.sh rename archive/server_setup/{ => unused}/install-cartopy.sh (100%) rename archive/server_setup/{ => unused}/install-geos.sh (100%) rename archive/server_setup/{ => unused}/install-proj4.sh (100%) rename archive/server_setup/{ => unused}/install-sqlite3.sh (100%) rename archive/server_setup/{ => unused}/install_opencv.sh (100%) rename archive/server_setup/{ => unused}/remount-s3.sh (100%) diff --git a/archive/server_setup/ssh-setup.sh b/archive/server_setup/ssh-setup.sh deleted file mode 100644 index f2c0cbf7..00000000 --- a/archive/server_setup/ssh-setup.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -echo $1 -scp -i ~/.ssh/markskey.pem ~/.ssh/ukmon-shared-keys $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/markskey.pem $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/config $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/gm* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/ukmon* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/marksk* $1:.ssh -scp -i ~/.ssh/markskey.pem ~/.ssh/s3* $1:.ssh - -ssh -i ~/.ssh/markskey.pem $1 mkdir setup -ssh -i ~/.ssh/markskey.pem $1 mkdir data -ssh -i ~/.ssh/markskey.pem $1 mkdir src - diff --git a/archive/server_setup/install-cartopy.sh b/archive/server_setup/unused/install-cartopy.sh similarity index 100% rename from archive/server_setup/install-cartopy.sh rename to archive/server_setup/unused/install-cartopy.sh diff --git a/archive/server_setup/install-geos.sh b/archive/server_setup/unused/install-geos.sh similarity index 100% rename from archive/server_setup/install-geos.sh rename to archive/server_setup/unused/install-geos.sh diff --git a/archive/server_setup/install-proj4.sh b/archive/server_setup/unused/install-proj4.sh similarity index 100% rename from archive/server_setup/install-proj4.sh rename to archive/server_setup/unused/install-proj4.sh diff --git a/archive/server_setup/install-sqlite3.sh b/archive/server_setup/unused/install-sqlite3.sh similarity index 100% rename from archive/server_setup/install-sqlite3.sh rename to archive/server_setup/unused/install-sqlite3.sh diff --git a/archive/server_setup/install_opencv.sh b/archive/server_setup/unused/install_opencv.sh similarity index 100% rename from archive/server_setup/install_opencv.sh rename to archive/server_setup/unused/install_opencv.sh diff --git a/archive/server_setup/remount-s3.sh b/archive/server_setup/unused/remount-s3.sh similarity index 100% rename from archive/server_setup/remount-s3.sh rename to archive/server_setup/unused/remount-s3.sh From f8bf4442c98fe151011d0e6cfd2c07713fd02fda Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 16:43:27 +0100 Subject: [PATCH 184/287] fix mariadb connections --- archive/utils/loadMatchCsvMDB.sh | 4 ++-- archive/utils/loadSingleCsvMDB.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/utils/loadMatchCsvMDB.sh b/archive/utils/loadMatchCsvMDB.sh index 884706b8..101049f3 100644 --- a/archive/utils/loadMatchCsvMDB.sh +++ b/archive/utils/loadMatchCsvMDB.sh @@ -18,8 +18,8 @@ echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch # leading space to prevent being inserted into environment - passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) -mysql -u${user} -p${passwd} << EOD +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's'"//g') +mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.matches; LOAD DATA LOCAL INFILE '$csvname' INTO TABLE ukmon.matches diff --git a/archive/utils/loadSingleCsvMDB.sh b/archive/utils/loadSingleCsvMDB.sh index 9b7910bc..507f0675 100644 --- a/archive/utils/loadSingleCsvMDB.sh +++ b/archive/utils/loadSingleCsvMDB.sh @@ -18,8 +18,8 @@ echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch # leading space to prevent being inserted into environment - passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) -mysql -u${user} -p${passwd} << EOD +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's'"//g') +mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.singles; LOAD DATA LOCAL INFILE '$csvname' INTO TABLE ukmon.singles From 7e4a62e39d000e9b1ca37d2f7ebd133f8910983c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 16:45:15 +0100 Subject: [PATCH 185/287] typo --- archive/utils/loadMatchCsvMDB.sh | 2 +- archive/utils/loadSingleCsvMDB.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/utils/loadMatchCsvMDB.sh b/archive/utils/loadMatchCsvMDB.sh index 101049f3..4dfaccb5 100644 --- a/archive/utils/loadMatchCsvMDB.sh +++ b/archive/utils/loadMatchCsvMDB.sh @@ -18,7 +18,7 @@ echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch # leading space to prevent being inserted into environment -passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's'"//g') +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.matches; LOAD DATA LOCAL INFILE '$csvname' diff --git a/archive/utils/loadSingleCsvMDB.sh b/archive/utils/loadSingleCsvMDB.sh index 507f0675..6d7b7b02 100644 --- a/archive/utils/loadSingleCsvMDB.sh +++ b/archive/utils/loadSingleCsvMDB.sh @@ -18,7 +18,7 @@ echo "loading $csvname into mariadb" logger -s -t loadSQL "starting" user=batch # leading space to prevent being inserted into environment -passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's'"//g') +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') mysql -p${passwd} -u${user} << EOD select count(*) from ukmon.singles; LOAD DATA LOCAL INFILE '$csvname' From 6643b2b291e26c9b5bb61de75ac3664f36db0441 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 16:45:33 +0100 Subject: [PATCH 186/287] tidy up docs --- archive/database/Setup.md | 43 ++++++++------------------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/archive/database/Setup.md b/archive/database/Setup.md index 84dff4cf..328cab34 100644 --- a/archive/database/Setup.md +++ b/archive/database/Setup.md @@ -1,42 +1,16 @@ # Installing MariaDB -## Create /etc/yum.repos.d/MariaDB.repo - -[mariadb] -name = MariaDB -baseurl = http://yum.mariadb.org/10.3/centos7-amd64 -gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB -gpgcheck=1 - -## Install packages - -yum update -yum install mariadb-server mariadb-client mariadb-devel mariadb-shared mariadb-common - -## Create Databases and Tables -``` -mysql -u root -h localhost -set root password with -update mysql.user set password= PASSWORD('newpassword') where user = 'root'; -flush privileges; -exit; -``` -execute contents of create_dbs_and_users.sql -execute contents of create_tables.sql - -## migrating to new server -make sure you've installed mariadb on the new server and created databases and users -make sure you can ssh from old to new -then on the old server run -``` -mysqldump -u batch -p'passwd' ukmon | ssh newserver mysql -u batch -p'passwd' ukmon -``` +## Installing and Migrating +For installation and migration instructions see `migratingBatchServer.md` in the `server_setup` folder ## Connecting from Python +These packages are installed as part of the ukmda tooling. +``` bash pip install mysql-connector-python pip install pymysql - +``` +``` python import pymysql.cursors connection = pymysql.connect(host='localhost', @@ -46,12 +20,13 @@ pip install pymysql cursorclass=pymysql.cursors.DictCursor) try: with connection.cursor() as cursor: - sql = "INSERT INTO books VALUES ({},'{}', {}, {})".format(sys.argv[1],'foo',34,56) + #sql = "INSERT INTO books VALUES ({},'{}', {}, {})".format(sys.argv[1],'foo',34,56) cursor.execute(sql) connection.commit() - sql = "SELECT * from `books` " + sql = "SELECT * from matches limit 10 " cursor.execute(sql) result = cursor.fetchall() print(result) finally: connection.close() +``` \ No newline at end of file From 0df64d248d1b347b33626f8c992ff5bf43d8c3e5 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 16:48:14 +0100 Subject: [PATCH 187/287] fixing mysql calls --- archive/cronjobs/captureBright.sh | 4 ++-- archive/utils/updateDb.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/cronjobs/captureBright.sh b/archive/cronjobs/captureBright.sh index e8ce844a..ca33ec02 100644 --- a/archive/cronjobs/captureBright.sh +++ b/archive/cronjobs/captureBright.sh @@ -9,8 +9,8 @@ rundt=$(date -d "yesterday" +%Y%m%d) python -m utils.compareBrightnessData $rundt # leading space to prevent being inserted into environment - pwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) -mysql -u batch -p$pwd -h localhost << EOD +pwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') +mysql -p$pwd -ubatch -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; load data local infile '${DATADIR}/brightness/CaptureNight_${rundt}.csv' diff --git a/archive/utils/updateDb.sh b/archive/utils/updateDb.sh index 904fa624..98bf4a5a 100644 --- a/archive/utils/updateDb.sh +++ b/archive/utils/updateDb.sh @@ -13,8 +13,8 @@ fi cd ~/prod/data/brightness # leading space to prevent being inserted into environment - passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) -mysql -u batch -p$passwd -h localhost << EOD +passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value| sed 's/"//g') +mysql -p$passwd -ubatch -h localhost << EOD use ukmon; select count(*) from ukmon.brightness; load data local infile './CaptureNight_${rundt}.csv' From 8184c805d7f8fe1803d16195abbad65ed1024698 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 22:09:38 +0100 Subject: [PATCH 188/287] small changes to documentation --- archive/cronjobs/nightlyJob.sh | 15 ++++++++------- archive/shwrinfo/costnotes.html | 17 +---------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index 7d16b7f1..fdea560c 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -63,11 +63,16 @@ $SRC/analysis/createSearchable.sh $yr singles # Run the match process - run this only once as it scoops up all unprocessed data ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 + +############################################################ +# FROM HERE DOWN WE'RE CONSOLIDATING DATA AND CREATING REPORTS +# and everything can be rerun safely if the data are present +############################################################ + log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" nightlyJob $SRC/website/updateIndexPages.sh -# CONSOLIDATE THE DATA AND CREATE SEARCH INDEXES -# consolidate the output of the match process for further analysos +# consolidate the output of the match process for further analysis log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob $SRC/analysis/consolidateOutput.sh ${yr} if [ "$(date +%m%d)" == "0101" ] ; then @@ -75,14 +80,10 @@ if [ "$(date +%m%d)" == "0101" ] ; then $SRC/analysis/consolidateOutput.sh $(date -d 'last year' +%Y) fi -# create the search indexes used on the website +# update the search indexes used on the website log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 2" nightlyJob $SRC/analysis/createSearchable.sh $yr matches -############################################################ -# FROM HERE DOWN WE'RE CREATING REPORTS -# and everything can be rerun safely if the data are present -############################################################ # add daily report to the website log2cw $NJLOGGRP $NJLOGSTREAM "start publishDailyReport" nightlyJob $SRC/website/publishDailyReport.sh diff --git a/archive/shwrinfo/costnotes.html b/archive/shwrinfo/costnotes.html index 09a5714c..08b211c3 100644 --- a/archive/shwrinfo/costnotes.html +++ b/archive/shwrinfo/costnotes.html @@ -4,19 +4,4 @@
  • Amounts of less than $0.10 per day are consolidated under Other.
  • AWS cost data may lag by up to a day, so yesterday's costs data are incomplete.
  • AWS charge VAT at the start of the month for the previous month's total bill.
    • -
  • Until September 2023, costs were split across two accounts for historical reasons. We've now consolidated into one account.
    • -
    -2023 Costs prior to 13th September were as follows: -
  • S3: $1135
    • -
  • ECS: $212
    • -
  • EC2-other: $159
    • -
  • EC2-instance: $131
    • -
  • Cloudwatch: $44
    • -
  • Registrar: $29
    • -
  • KMS: $23
    • -
  • DynamoDB: $15
    • -
  • Route53: $6
    • -
  • ECR: $5
    • -
  • Other: ÂŖ8
    • - -Total: $1768 + VAT +
    \ No newline at end of file From 99dc404f9f9550a2bd9353d52a4aeaba8bdb2ccf Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 22:13:49 +0100 Subject: [PATCH 189/287] suppress pip messages if already installed --- install_or_update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index 16dcc108..3a52a384 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -34,7 +34,7 @@ then cd $here source ~/.condaon conda activate wmpl - pip install -r archive/ukmon_pylib/additional_requirements.txt + pip install -r archive/ukmon_pylib/additional_requirements.txt | grep -v "already satisfied" fi # update the IMO and GMN meteor shower tables if missing From 378da820611245525933e2feb47fe3b6e0322523 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 22:15:27 +0100 Subject: [PATCH 190/287] tweaks to the installer --- install_or_update.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install_or_update.sh b/install_or_update.sh index 3a52a384..9a6bd9e9 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -35,12 +35,14 @@ then source ~/.condaon conda activate wmpl pip install -r archive/ukmon_pylib/additional_requirements.txt | grep -v "already satisfied" + echo "" fi # update the IMO and GMN meteor shower tables if missing if [ ! -f ~/${envname}/share/IMO_Working_Meteor_Shower_List.xml ] then ~/$envname/cronjobs/getImoWSfile.sh + echo "" fi read -n 1 -p "Update bashrc and config? (y/N) " yesno @@ -53,6 +55,7 @@ then rsync -a server_setup/$fil ~ done else + echo "" echo skipping config and bashrc fi echo "" From c3392e128ccccec6132f4d92c16adea86d0809b5 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 23:16:21 +0100 Subject: [PATCH 191/287] add logrotate stuff --- archive/server_setup/logs.conf | 24 ++++++++++++++++++++++++ install_or_update.sh | 4 ++++ 2 files changed, 28 insertions(+) create mode 100644 archive/server_setup/logs.conf diff --git a/archive/server_setup/logs.conf b/archive/server_setup/logs.conf new file mode 100644 index 00000000..9554fd49 --- /dev/null +++ b/archive/server_setup/logs.conf @@ -0,0 +1,24 @@ +daily +rotate 7 +notifempty +/home/ec2-user/prod/logs/*.log { + daily + rotate 45 + compress + delaycompress + dateext + notifempty + maxage 45 + # create +} + +/home/ec2-user/dev/logs/*.log { + daily + rotate 45 + compress + delaycompress + dateext + notifempty + maxage 45 + # create +} diff --git a/install_or_update.sh b/install_or_update.sh index 9a6bd9e9..a819d74d 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -28,6 +28,8 @@ mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls,man mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs +mkdir -p ~/.logrotate +mkdir -p ~/.aws if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] then @@ -44,6 +46,8 @@ then ~/$envname/cronjobs/getImoWSfile.sh echo "" fi +mkdir -p ~/.logrotate +rsync -a server_setup/logs.conf ~/.logrotate read -n 1 -p "Update bashrc and config? (y/N) " yesno if [[ "$yesno" == "y" || "$yesno" == "Y" ]] From 2ea7fd13e892bc279c07abf874041ecf3321d451 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 23:20:01 +0100 Subject: [PATCH 192/287] bugfixes in installer --- install_or_update.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index a819d74d..862c2c4e 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -33,11 +33,12 @@ mkdir -p ~/.aws if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] then - cd $here + pushd $here source ~/.condaon conda activate wmpl pip install -r archive/ukmon_pylib/additional_requirements.txt | grep -v "already satisfied" echo "" + popd fi # update the IMO and GMN meteor shower tables if missing From c73fe5503ecec8793d083833c46438b9353228c9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 23:21:29 +0100 Subject: [PATCH 193/287] tidy up installer --- install_or_update.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install_or_update.sh b/install_or_update.sh index 862c2c4e..5663e4e6 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -47,10 +47,8 @@ then ~/$envname/cronjobs/getImoWSfile.sh echo "" fi -mkdir -p ~/.logrotate -rsync -a server_setup/logs.conf ~/.logrotate -read -n 1 -p "Update bashrc and config? (y/N) " yesno +read -n 1 -p "Update $envname configuration files? (y/N) " yesno if [[ "$yesno" == "y" || "$yesno" == "Y" ]] then echo "Updating config for $envname" @@ -59,6 +57,8 @@ then do rsync -a server_setup/$fil ~ done + mkdir -p ~/.logrotate + rsync -a server_setup/logs.conf ~/.logrotate else echo "" echo skipping config and bashrc From 6f7c51e1a1ecd357501851cb35ad30eb858ad5f4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 28 Apr 2026 23:32:55 +0100 Subject: [PATCH 194/287] bugfix in key mgmt --- archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py index 2dadf8cb..85b3f04b 100644 --- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py +++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py @@ -26,7 +26,7 @@ lnxdir='/home/ec2-user/keymgmt/csvkeys' else: csvdir=os.path.expanduser('~/keymgmt/csvkeys') - csvdir=os.path.expanduser('~/keymgmt/csvkeys') + lnxdir=os.path.expanduser('~/keymgmt/csvkeys') def checkIfOnGMN(camid): From db8e111a0cbf87de05680ca4600becf9fb1af8d9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 12:35:43 +0100 Subject: [PATCH 195/287] move some functionality into new script to tidy up nightly job and simplify rerunning --- archive/ukmon_pylib/reports/CameraDetails.py | 20 ++++++----- archive/website/updateCameraDets.sh | 37 ++++++++++++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 archive/website/updateCameraDets.sh diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index ab3fd50d..8ea77d71 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -71,7 +71,7 @@ def findLocationInfo(srchstring, ddb=None, statdets=None): return srchres -def createCDCsv(targetloc): +def createCDCsv(targetloc, srchidxdir): camdets = loadLocationDetails() cd4csv = camdets[['site','stationid','oldcode','direction','camtype','active']] pd.options.mode.chained_assignment = None @@ -85,26 +85,30 @@ def createCDCsv(targetloc): cd4csv.to_csv(outfname, index=False) with open(os.path.join(datadir, 'activecamcount.txt'), 'w') as outf: outf.write(str(len(cd4csv[cd4csv.active==1]))) - createStatOptsHtml(camdets, datadir) - createStatOptsHtml(camdets, datadir, True) - createStatOptsHtml(camdets, datadir, True, True) + + outdir = os.path.join(datadir, srchidxdir) + os.makedirs(outdir, exist_ok=True) + + createStatOptsHtml(camdets, outdir) + createStatOptsHtml(camdets, outdir, True) + createStatOptsHtml(camdets, outdir, True, True) return -def createStatOptsHtml(sitedets, datadir, active=False, locs=False): +def createStatOptsHtml(sitedets, outdir, active=False, locs=False): """ Create an HTML file containing a list of stations or locations These files are used by the website search functions """ if active: if locs: - siteidx = os.path.join(datadir, 'activestatlocs.html') + siteidx = os.path.join(outdir, 'activestatlocs.html') sitedets = sitedets.sort_values(by=['site', 'stationid']) else: - siteidx = os.path.join(datadir, 'activestatopts.html') + siteidx = os.path.join(outdir, 'activestatopts.html') sitedets = sitedets[sitedets.active==1] else: - siteidx = os.path.join(datadir, 'statopts.html') + siteidx = os.path.join(outdir, 'statopts.html') with open(siteidx, 'w') as outf: if not active: outf.write('\n') diff --git a/archive/website/updateCameraDets.sh b/archive/website/updateCameraDets.sh new file mode 100644 index 00000000..c620bde5 --- /dev/null +++ b/archive/website/updateCameraDets.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Copyright (C) 2018-2023 Mark McIntyre +# create various html and js files of camera info for the website +# +# Parameters +# none +# +# Consumes +# +# +# Produces +# camera locations file cameraLocs.json +# camera details file camera-details.csv +# three html files used by the search page + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# load the configuration +source $here/../config.ini >/dev/null 2>&1 +conda activate ${WMPL_ENV} + +logger -s -t updateCameraDets "starting" + +python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();" +aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --quiet +aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --quiet +aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --quiet + +# create the CSV file of camera info and the html versions for search functions on the website + +python -c "from reports.CameraDetails import createCDCsv; createCDCsv('consolidated','searchidx');" +aws s3 cp $DATADIR/consolidated/camera-details.csv $UKMONSHAREDBUCKET/consolidated/ --quiet +aws s3 cp $DATADIR/searchidx/statopts.html $WEBSITEBUCKET/search/ --quiet +aws s3 cp $DATADIR/searchidx/activestatopts.html $WEBSITEBUCKET/search/ --quiet +aws s3 cp $DATADIR/searchidx/activestatlocs.html $WEBSITEBUCKET/search/ --quiet + +logger -s -t updateCameraDets "finished" From 7cf10c9f7424a0bd4731983ea730492019483771 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 13:13:44 +0100 Subject: [PATCH 196/287] improve install script --- install_or_update.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/install_or_update.sh b/install_or_update.sh index 5663e4e6..409e1acb 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -10,11 +10,12 @@ envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -mkdir -p ~/${envname} - +echo "Updating codebase..." cd $here/archive git pull +echo "Updating code..." +mkdir -p ~/${envname} [ -d ~/$envname/data ] && msg="upgrade" || msg="install" for loc in analysis ukmon_pylib website cronjobs utils static_content do @@ -23,6 +24,7 @@ do done rsync -a share/ ~/${envname}/share +echo "Creating data folders..." DATADIR=~/$envname/data mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls,manualuploads} mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} @@ -31,6 +33,7 @@ mkdir -p ~/$envname/logs mkdir -p ~/.logrotate mkdir -p ~/.aws +echo "Checking conda environment..." if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] then pushd $here @@ -41,16 +44,18 @@ then popd fi +echo "Updating meteor shower tables..." # update the IMO and GMN meteor shower tables if missing if [ ! -f ~/${envname}/share/IMO_Working_Meteor_Shower_List.xml ] then ~/$envname/cronjobs/getImoWSfile.sh echo "" fi - +echo "" read -n 1 -p "Update $envname configuration files? (y/N) " yesno if [[ "$yesno" == "y" || "$yesno" == "Y" ]] then + echo "" echo "Updating config for $envname" ~/$envname/utils/makeConfig.sh $RUNTIME_ENV for fil in .bashrc .bash_aliases .vimrc .condaon From d4fa6382217da15ade7446cf8a24ec9ce9a56d6f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 13:20:43 +0100 Subject: [PATCH 197/287] update location of camera count file --- archive/ukmon_pylib/reports/CameraDetails.py | 2 +- archive/website/createSummaryTable.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index 8ea77d71..44e96241 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -83,7 +83,7 @@ def createCDCsv(targetloc, srchidxdir): datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) outfname = os.path.join(datadir, targetloc, 'camera-details.csv') cd4csv.to_csv(outfname, index=False) - with open(os.path.join(datadir, 'activecamcount.txt'), 'w') as outf: + with open(os.path.join(datadir, targetloc, 'activecamcount.txt'), 'w') as outf: outf.write(str(len(cd4csv[cd4csv.active==1]))) outdir = os.path.join(datadir, srchidxdir) diff --git a/archive/website/createSummaryTable.sh b/archive/website/createSummaryTable.sh index fab46348..220de239 100644 --- a/archive/website/createSummaryTable.sh +++ b/archive/website/createSummaryTable.sh @@ -9,6 +9,7 @@ # Consumes # the single station and matched station data files in $DATADIR/single and $DATADIR/matched # the most recent match run logfile +# information about the number of active cameras # # Produces # a webpage showing the current summary - the site homepage @@ -51,7 +52,7 @@ python -c "from reports.createAnnualBarChart import createBarChart; createBarCha popd # update index page -numcams=$(cat $DATADIR/activecamcount.txt) +numcams=$(cat $DATADIR/consolidated/activecamcount.txt) cat $TEMPLATES/frontpage.html | sed "s/#NUMCAMS#/$numcams/g" > $DATADIR/newindex.html logger -s -t createSummaryTable "copying to website" From f8165a3bd4a2a61e85a3cf94fa738eb65e6edb68 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 13:23:50 +0100 Subject: [PATCH 198/287] move annual bar charts into a more logical location --- archive/ukmon_pylib/reports/createAnnualBarChart.py | 2 +- archive/website/createSummaryTable.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index b7ac9fca..132850ff 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -49,7 +49,7 @@ def createBarChart(datadir=None, yr=None): fig.set_size_inches(12, 5) fig.tight_layout() #plt.gca().invert_yaxis() - plt.savefig(os.path.join(datadir,f'Annual-{yr}.jpg'), dpi=100) + plt.savefig(os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg'), dpi=100) plt.close() return matches diff --git a/archive/website/createSummaryTable.sh b/archive/website/createSummaryTable.sh index 220de239..044ae9ee 100644 --- a/archive/website/createSummaryTable.sh +++ b/archive/website/createSummaryTable.sh @@ -62,7 +62,7 @@ aws s3 cp $DATADIR/coverage-100km.html $WEBSITEBUCKET/data/ --quiet aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/ --quiet aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/coverage.html --quiet aws s3 cp $DATADIR/coverage-25km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/Annual-${yr}.jpg $WEBSITEBUCKET/YearToDate.jpg --quiet +aws s3 cp $DATADIR/reports/${yr}/Annual-${yr}.jpg $WEBSITEBUCKET/YearToDate.jpg --quiet aws s3 cp $DATADIR/newindex.html $WEBSITEBUCKET/index.html --quiet aws s3 cp $SRC/website/templates/header.html $WEBSITEBUCKET/templates/ --quiet aws s3 cp $SRC/website/templates/footer.html $WEBSITEBUCKET/templates/ --quiet From b1e7187bfbec445de5bb83235b9cebeaac7744d0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 13:24:57 +0100 Subject: [PATCH 199/287] update test script --- .../ukmon_pylib/tests/test_reports_createAnnualBarChart.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py index 793c3ee1..49009654 100644 --- a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py +++ b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py @@ -17,14 +17,14 @@ def test_createBarChart(): # pass year and datadir res = createBarChart(datadir=datadir, yr=yr) - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) os.remove(outf) # pass datadir only - will fail if we're not in 2023 res = createBarChart(datadir=datadir, yr=None) - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: @@ -35,7 +35,7 @@ def test_createBarChart(): # no params passed res = createBarChart() - outf=os.path.join(datadir, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: From 531d817fbbbb22477f0b7e08e9d0be2b7739b346 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 14:03:23 +0100 Subject: [PATCH 200/287] add gmaps param to my acct --- archive/terraform/mjmm/ssm_parameters.tf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/archive/terraform/mjmm/ssm_parameters.tf b/archive/terraform/mjmm/ssm_parameters.tf index 5eb11b0b..8889f3a6 100644 --- a/archive/terraform/mjmm/ssm_parameters.tf +++ b/archive/terraform/mjmm/ssm_parameters.tf @@ -176,3 +176,12 @@ resource "aws_ssm_parameter" "prod_batchloggroup" { "billingtag" = "ukmon" } } + +resource "aws_ssm_parameter" "prod_gmapsapikey" { + name = "prod_gmapsapikey" + type = "SecureString" + value = "AIzaSyBFadTuzvLfkUhz8CwY2CtRDJ_lYlHUYyA" + tags = { + "billingtag" = "ukmon" + } +} From ef54c94fa37e5bd3738e10228cebd0498733aa54 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 14:04:51 +0100 Subject: [PATCH 201/287] relocate coverage maps and front page index --- .../ukmon_pylib/reports/createSummaryTable.py | 2 +- .../ukmon_pylib/reports/makeCoverageMap.py | 4 +-- archive/website/createSummaryTable.sh | 32 +++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/archive/ukmon_pylib/reports/createSummaryTable.py b/archive/ukmon_pylib/reports/createSummaryTable.py index a0d861bb..5155e332 100644 --- a/archive/ukmon_pylib/reports/createSummaryTable.py +++ b/archive/ukmon_pylib/reports/createSummaryTable.py @@ -19,7 +19,7 @@ def createSummaryTable(curryr=None, datadir=None): curryr = str(datetime.datetime.now().year) if datadir is None: datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - fname = os.path.join(datadir, 'summarytable.js') + fname = os.path.join(datadir, 'reports', curryr, 'summarytable.js') with open(fname, 'w') as f: f.write('$(function() {\n') f.write('var table = document.createElement("table");\n') diff --git a/archive/ukmon_pylib/reports/makeCoverageMap.py b/archive/ukmon_pylib/reports/makeCoverageMap.py index db5c8017..b25f8bec 100644 --- a/archive/ukmon_pylib/reports/makeCoverageMap.py +++ b/archive/ukmon_pylib/reports/makeCoverageMap.py @@ -67,7 +67,7 @@ def makeCoverageMap(kmlsource, outdir, showMarker=False, useName=False): return -def createCoveragePage(): +def createCoveragePage(targdir): apikey = getApiKey() if not apikey: return @@ -75,7 +75,7 @@ def createCoveragePage(): datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) with open(os.path.join(templdir, 'coverage-maps.html'), 'r') as inf: lis = inf.readlines() - with open(os.path.join(datadir, 'latest','coverage-maps.html'), 'w') as outf: + with open(os.path.join(datadir, targdir,'coverage-maps.html'), 'w') as outf: for li in lis: outf.write(li.replace('{{MAPSAPIKEY}}', apikey)) return diff --git a/archive/website/createSummaryTable.sh b/archive/website/createSummaryTable.sh index 044ae9ee..caa0fd83 100644 --- a/archive/website/createSummaryTable.sh +++ b/archive/website/createSummaryTable.sh @@ -33,37 +33,37 @@ python -c "from reports.createSummaryTable import createSummaryTable; createSumm logger -s -t createSummaryTable "create a coverage map from the kmls" # make sure correct version of GEOS and PROJ4 available for mapping routines -LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib -export LD_LIBRARY_PATH +#LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/geos/lib:/usr/local/proj4/lib +#export LD_LIBRARY_PATH mkdir -p $DATADIR/kmls aws s3 sync $WEBSITEBUCKET/img/kmls/ $DATADIR/kmls --quiet export KMLTEMPLATE="*25km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} export KMLTEMPLATE="*70km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} export KMLTEMPLATE="*100km.kml" -python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR +python -m reports.makeCoverageMap $DATADIR/kmls $DATADIR/reports/${yr} -python -c "from reports.makeCoverageMap import createCoveragePage as ccp ; ccp() ;" +python -c "from reports.makeCoverageMap import createCoveragePage as ccp ; ccp('reports/${yr}') ;" logger -s -t createSummaryTable "create year-to-date barchart" -pushd $DATADIR python -c "from reports.createAnnualBarChart import createBarChart; createBarChart('${DATADIR}','${yr}')" -popd # update index page numcams=$(cat $DATADIR/consolidated/activecamcount.txt) -cat $TEMPLATES/frontpage.html | sed "s/#NUMCAMS#/$numcams/g" > $DATADIR/newindex.html +cat $TEMPLATES/frontpage.html | sed "s/#NUMCAMS#/$numcams/g" > $DATADIR/reports/${yr}/newindex.html logger -s -t createSummaryTable "copying to website" -aws s3 cp $DATADIR/latest/coverage-maps.html $WEBSITEBUCKET/latest/ --quiet -aws s3 cp $DATADIR/summarytable.js $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-100km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/ --quiet -aws s3 cp $DATADIR/coverage-70km.html $WEBSITEBUCKET/data/coverage.html --quiet -aws s3 cp $DATADIR/coverage-25km.html $WEBSITEBUCKET/data/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-maps.html $WEBSITEBUCKET/latest/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-100km.html $WEBSITEBUCKET/data/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-70km.html $WEBSITEBUCKET/data/ --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-70km.html $WEBSITEBUCKET/data/coverage.html --quiet +aws s3 cp $DATADIR/reports/${yr}/coverage-25km.html $WEBSITEBUCKET/data/ --quiet + aws s3 cp $DATADIR/reports/${yr}/Annual-${yr}.jpg $WEBSITEBUCKET/YearToDate.jpg --quiet -aws s3 cp $DATADIR/newindex.html $WEBSITEBUCKET/index.html --quiet + +aws s3 cp $DATADIR/reports/${yr}/newindex.html $WEBSITEBUCKET/index.html --quiet +aws s3 cp $DATADIR/reports/${yr}/summarytable.js $WEBSITEBUCKET/data/ --quiet aws s3 cp $SRC/website/templates/header.html $WEBSITEBUCKET/templates/ --quiet aws s3 cp $SRC/website/templates/footer.html $WEBSITEBUCKET/templates/ --quiet From 91dc6884ffa241781baaaa054b018f3f6c7d5907 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 14:30:43 +0100 Subject: [PATCH 202/287] tidy up useraudit process --- archive/utils/userAudit.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/archive/utils/userAudit.sh b/archive/utils/userAudit.sh index 205f7e20..98e7123f 100644 --- a/archive/utils/userAudit.sh +++ b/archive/utils/userAudit.sh @@ -8,22 +8,24 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -cd $DATADIR -echo "{\"Subject\": {\"Data\": \"User Audit\",\"Charset\": \"utf-8\"}," > auditmsg.json -echo -n "\"Body\": {\"Text\": {\"Data\": \"" >> auditmsg.json +echo "{\"Subject\": {\"Data\": \"User Audit\",\"Charset\": \"utf-8\"}," > $DATADIR/admin/auditmsg.json +echo -n "\"Body\": {\"Text\": {\"Data\": \"" >> $DATADIR/admin/auditmsg.json # general audit of all users export AWS_PROFILE=ukmonshared -python -m maintenance.getUserAndKeyInfo audit > $DATADIR/useraudit.txt +python -m maintenance.getUserAndKeyInfo audit > $DATADIR/admin/useraudit.txt # accounts not used for 30 days or more export MAXAGE=30 -python -m maintenance.getUserAndKeyInfo dormant >> $DATADIR/useraudit.txt +python -m maintenance.getUserAndKeyInfo dormant >> $DATADIR/admin/useraudit.txt -awk -v ORS='\\n' '1' useraudit.txt >> auditmsg.json -echo "\",\"Charset\": \"utf-8\"}}}" >> auditmsg.json +awk -v ORS='\\n' '1' $DATADIR/admin/useraudit.txt >> $DATADIR/admin/auditmsg.json +echo "\",\"Charset\": \"utf-8\"}}}" >> $DATADIR/admin/auditmsg.json + +pushd $DATADIR/admin aws ses send-email --destination ToAddresses="markmcintyre99@googlemail.com" --message file://auditmsg.json --from "noreply@ukmeteors.co.uk" --region eu-west-2 +popd +rm $DATADIR/admin/auditmsg.json unset MAXAGE unset AWS_PROFILE -rm auditmsg.json From 8cc2ed424f6e8ef48f2e95214dc397ffed633353 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 14:36:03 +0100 Subject: [PATCH 203/287] bugfixes in testing --- .../ukmon_pylib/tests/test_reports_createAnnualBarChart.py | 6 +++--- archive/ukmon_pylib/tests/test_reports_various.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py index 49009654..7dac55f6 100644 --- a/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py +++ b/archive/ukmon_pylib/tests/test_reports_createAnnualBarChart.py @@ -17,14 +17,14 @@ def test_createBarChart(): # pass year and datadir res = createBarChart(datadir=datadir, yr=yr) - outf=os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) os.remove(outf) # pass datadir only - will fail if we're not in 2023 res = createBarChart(datadir=datadir, yr=None) - outf=os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: @@ -35,7 +35,7 @@ def test_createBarChart(): # no params passed res = createBarChart() - outf=os.path.join(datadir,'reports',yr, f'Annual-{yr}.jpg') + outf=os.path.join(datadir,'reports',str(yr), f'Annual-{yr}.jpg') print(f'looking for {outf}') assert os.path.isfile(outf) is False try: diff --git a/archive/ukmon_pylib/tests/test_reports_various.py b/archive/ukmon_pylib/tests/test_reports_various.py index d9aeac7b..27455d33 100644 --- a/archive/ukmon_pylib/tests/test_reports_various.py +++ b/archive/ukmon_pylib/tests/test_reports_various.py @@ -10,8 +10,9 @@ def test_createSummaryTable(): + yr=2023 createSummaryTable(curryr=None, datadir=datadir) - fname = os.path.join(datadir, 'summarytable.js') + fname = os.path.join(datadir,'reports',str(yr), 'summarytable.js') assert os.path.isfile(fname) lis = open(fname, 'r').readlines() assert 'cell.innerHTML = ' in lis[9] From 47fd55b316cb19e0e0415738d61bea032eb34745 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 14:46:34 +0100 Subject: [PATCH 204/287] bugfix in createAnnualBarchart --- archive/ukmon_pylib/reports/createAnnualBarChart.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/reports/createAnnualBarChart.py b/archive/ukmon_pylib/reports/createAnnualBarChart.py index 132850ff..944c2ba0 100644 --- a/archive/ukmon_pylib/reports/createAnnualBarChart.py +++ b/archive/ukmon_pylib/reports/createAnnualBarChart.py @@ -16,7 +16,8 @@ def createBarChart(datadir=None, yr=None): yr=datetime.datetime.now().year if datadir is None: datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - fname = os.path.join(datadir, 'matched', 'matches-full-{}.parquet.snap'.format(yr)) + yr = str(yr) + fname = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') if not os.path.isfile(fname): print('{} missing', fname) return None From aacc917ca678162669f034b3a611867e21bd4e27 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 14:53:22 +0100 Subject: [PATCH 205/287] bug in testscript --- archive/ukmon_pylib/tests/test_reports_various.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/tests/test_reports_various.py b/archive/ukmon_pylib/tests/test_reports_various.py index 27455d33..9ddc3ba6 100644 --- a/archive/ukmon_pylib/tests/test_reports_various.py +++ b/archive/ukmon_pylib/tests/test_reports_various.py @@ -11,7 +11,7 @@ def test_createSummaryTable(): yr=2023 - createSummaryTable(curryr=None, datadir=datadir) + createSummaryTable(curryr=yr, datadir=datadir) fname = os.path.join(datadir,'reports',str(yr), 'summarytable.js') assert os.path.isfile(fname) lis = open(fname, 'r').readlines() From b1419049268c84d8bbe4f2f3ce96350f0ba5cdde Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 14:56:37 +0100 Subject: [PATCH 206/287] another small bug in createSummaryTable --- archive/ukmon_pylib/reports/createSummaryTable.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/reports/createSummaryTable.py b/archive/ukmon_pylib/reports/createSummaryTable.py index 5155e332..5948f194 100644 --- a/archive/ukmon_pylib/reports/createSummaryTable.py +++ b/archive/ukmon_pylib/reports/createSummaryTable.py @@ -19,7 +19,9 @@ def createSummaryTable(curryr=None, datadir=None): curryr = str(datetime.datetime.now().year) if datadir is None: datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - fname = os.path.join(datadir, 'reports', curryr, 'summarytable.js') + outdir = os.path.join(datadir, 'reports', curryr) + os.makedirs(outdir, exist_ok=True) + fname = os.path.join(outdir, 'summarytable.js') with open(fname, 'w') as f: f.write('$(function() {\n') f.write('var table = document.createElement("table");\n') From 564b6559d9bb39fc3988b8c0e6ca06a3d9e333ef Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 15:01:39 +0100 Subject: [PATCH 207/287] still fighting that bug --- archive/ukmon_pylib/reports/createSummaryTable.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/reports/createSummaryTable.py b/archive/ukmon_pylib/reports/createSummaryTable.py index 5948f194..b3150002 100644 --- a/archive/ukmon_pylib/reports/createSummaryTable.py +++ b/archive/ukmon_pylib/reports/createSummaryTable.py @@ -16,9 +16,10 @@ def createSummaryTable(curryr=None, datadir=None): """ if curryr is None: - curryr = str(datetime.datetime.now().year) + curryr = datetime.datetime.now().year if datadir is None: datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + curryr = str(curryr) outdir = os.path.join(datadir, 'reports', curryr) os.makedirs(outdir, exist_ok=True) fname = os.path.join(outdir, 'summarytable.js') From 55872fa7096f128845957b0561f59e7acd4eab35 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 15:15:46 +0100 Subject: [PATCH 208/287] tidy up lastlog data --- archive/analysis/getLogData.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/archive/analysis/getLogData.sh b/archive/analysis/getLogData.sh index 1f46b696..67342a51 100644 --- a/archive/analysis/getLogData.sh +++ b/archive/analysis/getLogData.sh @@ -10,7 +10,7 @@ if [ "$1" != "" ] ; then logfile=$DATADIR/lastlogs/lastlog-${rundate}.html else rundate=$(date +%Y%m%d) - logfile=$DATADIR/lastlog.html + logfile=$DATADIR/lastlogs/lastlog.html fi # create performance metrics @@ -78,7 +78,7 @@ echo "

    > $logfile -if [ ! -d $DATADIR/lastlogs ] ; then mkdir $DATADIR/lastlogs ; fi +mkdir -p $DATADIR/lastlogs if [ "$1" == "" ] ; then aws s3 cp $logfile $WEBSITEBUCKET/reports/ --quiet cp $logfile $DATADIR/lastlogs/lastlog-${rundate}.html @@ -92,3 +92,5 @@ ls -1r $DATADIR/lastlogs/last*.html | head -90 | while read i ; do done cat $TEMPLATES/footer.html >> $DATADIR/lastlogs/index.html aws s3 cp $DATADIR/lastlogs/index.html $WEBSITEBUCKET/reports/lastlogs/ --quiet + +find $DATADIR/lastlogs -name "lastlog*" -mtime +90 -exec rm -f {} \; From cceb1a4c5a4eb1bf9ef24bfbb8fcd36eae8cd16d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 15:22:16 +0100 Subject: [PATCH 209/287] remove unused file --- .../ukmon_pylib/utils/gatherDetectionData.py | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 archive/ukmon_pylib/utils/gatherDetectionData.py diff --git a/archive/ukmon_pylib/utils/gatherDetectionData.py b/archive/ukmon_pylib/utils/gatherDetectionData.py deleted file mode 100644 index bc8be752..00000000 --- a/archive/ukmon_pylib/utils/gatherDetectionData.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Collect data for a potential match at a specific time -# -# Copyright (C) 2018-2023 Mark McIntyre -# -import os -import sys -import datetime -import pandas as pd - -from meteortools.ukmondb import getECSVs, getLiveJpgs, getFBfiles, getDetections - - -def getRawData(idlist, outpth): - for _,row in idlist.iterrows(): - stat =row.ID - dts = datetime.datetime.strptime(row.Dtstamp, '%Y%m%d_%H%M%S') - dt = dts.strftime('%Y-%m-%dT%H:%M:%S') - fboutdir = os.path.join(outpth, stat) - dtstr = row.Filename[10:25] - getLiveJpgs(dtstr, outpth, False) - getFBfiles(f'{stat}_{dtstr}', fboutdir) - getECSVs(stat, dt, True, fboutdir) - return - - -def updateSingleDB(yr, consdt): - datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - sngls = pd.read_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv')) - consdf = pd.read_csv(os.path.join(datadir, 'single', 'used', f'consumed_{consdt}.txt'), names=['Filename']) - conslist = list(consdf.Filename) - for ff in conslist: - mtch = sngls.Filename == ff - if mtch.any(): - print(f'updating {ff}') - sngls.loc[mtch, 'status'] = 'Y' - else: - print(f'skipping {ff}') - sngls.to_csv(os.path.join(datadir, 'single', f'singles-{yr}.csv'), index=False) - - -if __name__ == '__main__': - outpth = f'./{sys.argv[1]}' - os.makedirs(outpth, exist_ok=True) - idlist = getDetections(sys.argv[1]) - if not isinstance(idlist, bool): - getRawData(idlist, outpth) - ids = list(idlist.ID) - with open(os.path.join(outpth, 'ids.txt'), 'w') as outf: - outf.writelines(line + '\n' for line in ids) - print(ids) - else: - print('no matches') From c467a9c71652dfdea1ebe36c113a0e64202c9deb Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 17:27:09 +0100 Subject: [PATCH 210/287] update cam status reporting --- archive/ukmon_pylib/metrics/camMetrics.py | 43 ++++++++++++----------- archive/website/cameraStatusReport.sh | 5 ++- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 04a09053..e36f49f3 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -187,35 +187,38 @@ def backPopulate(stationid): pd.options.mode.chained_assignment = 'warn' caminfo = camlist.drop(columns=['site','direction','oldcode','active','camtype','eMail', 'humanName']) - logindf = pd.read_csv(os.path.join(datadir, 'reports', 'lastlogins.txt'), names=['dateval','timeval','siteid'], skipinitialspace=True) - logindf['timeval'] = logindf.timeval.astype('str').str.pad(6,fillchar='0') - logindf.dateval.fillna('Jan-01',inplace=True) - logindf.timeval.fillna('00:00:00',inplace=True) - # handle case round yearend where the log may have prev year's details as well as current year - nowdt = datetime.datetime.now() - yrval = str(nowdt.year) + '-' - yrvalback = str(nowdt.year-1) + '-' - logindf['lastseen'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrval + logindf.dateval+'_'+logindf.timeval] - try: # will fail on 29th Feb in a leapyear, as previous year is not leap - logindf['lastseen2'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrvalback + logindf.dateval+'_'+logindf.timeval] - except Exception: - logindf['lastseen2'] = [datetime.datetime.strptime(x, '%Y-%b-%d_%H%M%S') for x in yrval + logindf.dateval+'_'+logindf.timeval] - logindf.loc[logindf.lastseen > nowdt, 'lastseen'] = logindf.lastseen2 + # process the last-login data from the SSHD log + lastlogs = open(os.path.join(datadir, 'reports', 'lastlogins.txt'),'r').readlines() + lodata = [] + now = datetime.datetime.now(tz=datetime.timezone.utc) + for li in lastlogs: + spls = li.split('ssh') + # could be ubuntu (auth.log) or amazon linux style log + dtstr = spls[0][spls[0].find(':')+1:][:19] + if ' ' in dtstr: + dtval = datetime.datetime.strptime(dtstr[:15], '%b %d %H:%M:%S') + dtval = dtval.replace(year=now.year, tzinfo=datetime.timezone.utc) + if dtval > now: + dtval = dtval.replace(year=now.year-1, tzinfo=datetime.timezone.utc) + else: + dtval = datetime.datetime.strptime(dtstr, '%Y-%m-%dT%H:%M:%S') + dtval = dtval.replace(tzinfo=datetime.timezone.utc) + location = spls[1].split(' for ')[1].split()[0] + lodata.append({'location':location, 'lastseen':dtval}) + + + logindf = pd.DataFrame(lodata) logindf = logindf.sort_values(by=['lastseen']) - logindf.drop_duplicates(subset=['siteid'], inplace=True, keep='last') - logindf.rename(columns={'siteid':'location'}, inplace=True) - logindf.drop(columns = ['lastseen2'], inplace=True) + logindf.drop_duplicates(subset=['location'], inplace=True, keep='last') # create a merged dataframe with siteid and stationid intdf = pd.merge(logindf,caminfo, on=['location'], how='outer') df = pd.merge(intdf, fulldf, on=['stationid']) - df.dateval.fillna('Jan-01',inplace=True) - df.timeval.fillna('00:00:00',inplace=True) df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] - df = df.drop(columns=['timeval','stationid','manual','rundate', 'upddate','uploadtime', 'dateval']) + df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) df['dateval']=[x.strftime('%b-%d') for x in df.lastupload] df = df.sort_values(by=['lastupload']) diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh index 537a7b2f..00cf151e 100644 --- a/archive/website/cameraStatusReport.sh +++ b/archive/website/cameraStatusReport.sh @@ -27,9 +27,12 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} export PYTHONPATH=$PYLIB logger -s -t cameraStatusReport "starting" -sudo grep publickey /var/log/secure* | grep -v ec2-user | awk '{printf("%s-%s, %s, %s\n", $1, $2, $3,$9)}' | awk -F ":" '{print $2$3$4}' > $DATADIR/reports/lastlogins.txt +platform=$(grep "^NAME" /etc/os-release | awk -F'"' '{print $2}') +[ "$platform" == "Ubuntu" ] && authlog=/var/log/auth.log || authlog=/var/log/secure +sudo grep publickey $authlog* | grep -v $USER > $DATADIR/reports/lastlogins.txt python -m metrics.camMetrics $rundate +rm -f $DATADIR/reports/lastlogins.txt aws s3 cp $DATADIR/reports/stationlogins.txt $WEBSITEBUCKET/reports/stationlogins.txt --region eu-west-2 --quiet From 5757d2c9ac95e738f7b8b49fc39b9307fefe4286 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 17:56:50 +0100 Subject: [PATCH 211/287] clear down the old brightness data --- archive/cronjobs/captureBright.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/cronjobs/captureBright.sh b/archive/cronjobs/captureBright.sh index ca33ec02..ff38a264 100644 --- a/archive/cronjobs/captureBright.sh +++ b/archive/cronjobs/captureBright.sh @@ -19,4 +19,4 @@ select count(*) from ukmon.brightness; EOD find ${DATADIR}/brightness -name "CaptureNight*" -mtime +30 -exec rm -f {} \; - +find ${DATADIR}/brightness -name "matcheddata*" -mtime +30 -exec rm -f {} \; \ No newline at end of file From 043986a7746369e3819de034eac7f207b18fff7b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 18:02:44 +0100 Subject: [PATCH 212/287] simplification of logging and of the nightlyJob script --- archive/cronjobs/nightlyJob.sh | 96 ++++++++++++---------------------- 1 file changed, 32 insertions(+), 64 deletions(-) diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index fdea560c..b955b709 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -7,12 +7,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) -export NJLOGSTREAM -aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared - -# log2cw is a function defined in config.ini and logs to cloudwatch, syslog and console -log2cw $NJLOGGRP $NJLOGSTREAM "start nightlyJob" nightlyJob +logger -s -t nightlyJob "start nightlyJob" # dates to process for rundate=$(date +%Y%m%d) @@ -25,55 +20,53 @@ mkdir -p $DATADIR/{admin,browse,consolidated,costs,dailyreports,distrib,kmls} mkdir -p $DATADIR/{lastlogs,latest,matched,orbits,reports,searchidx,single,trajdb,videos} mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} -# update the JSON file containing camera location details. This is used by the website -log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera location/dir/fov database" nightlyJob -python -c "from reports.CameraDetails import updateCamLocDirFovDB; updateCamLocDirFovDB();" -aws s3 cp $DATADIR/admin/cameraLocs.json $UKMONSHAREDBUCKET/admin/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/admin/cameraLocs.json $WEBSITEBUCKET/browse/ --profile ukmonshared --quiet -aws s3 sync $UKMONSHAREDBUCKET/admin/ $DATADIR/admin --profile ukmonshared --quiet - -# create the CSV file of camera info and the html versions for search functions on the website -log2cw $NJLOGGRP $NJLOGSTREAM "updating the camera details files for searching" nightlyJob -python -c "from reports.CameraDetails import createCDCsv; createCDCsv('consolidated');" -aws s3 cp $DATADIR/consolidated/camera-details.csv $UKMONSHAREDBUCKET/consolidated/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/statopts.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/activestatopts.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet -aws s3 cp $DATADIR/activestatlocs.html $WEBSITEBUCKET/search/ --profile ukmonshared --quiet +#################################################################################### +# START OF DATA PROCESSING. FIRST WE UPDATE THE SEARCH PAGE WITH SINGLE-STATION +# AND CAMERA DETAILS SO IT CAN BE SEARCHED WITHOUT WAITING FOR THE MATCHING ENGINE +#################################################################################### -# set up logging for the match process -log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" nightlyJob -matchlog=matchJob.log -if [ -f $SRC/logs/$matchlog ] ; then - suff=$(stat matchJob.log -c %X) - mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff -fi +# update the JSON and html files containing camera and location details, used by the website +$SRC/website/updateCameraDets.sh -log2cw $NJLOGGRP $NJLOGSTREAM "start getRMSSingleData" nightlyJob -# this creates the parquet table for Athena +# consolidate the single-station data - we do this here so the search index can be updated earlier +# and we can search for single-station events without waiting for the main batch $SRC/analysis/getRMSSingleData.sh if [ "$(date +%m%d)" == "0101" ] ; then # catch any data uploaded on 01/01 that is for 31/12 the previous year $SRC/analysis/getRMSSingleData.sh $(date -d 'last year' +%Y) fi -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 1" nightlyJob - +# now update the search index with single-station data $SRC/analysis/createSearchable.sh $yr singles +#################################################################################### +# NOW THE RUN THE MATCHING ENGINE VIA findAllmatches.sh +# Take care rerunning it as it will re-create the daily report +#################################################################################### + +# set up logging for the match process +matchlog=matchJob.log + +# save the existing log, in case the process is being rerun on the same day +if [ -f $SRC/logs/$matchlog ] ; then + suff=$(stat $SRC/logs/$matchjob -c %X) + mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff +fi + +logger -s -t nightlyJob "start findAllMatches" # Run the match process - run this only once as it scoops up all unprocessed data ${SRC}/analysis/findAllMatches.sh > ${SRC}/logs/${matchlog} 2>&1 -############################################################ +#################################################################################### # FROM HERE DOWN WE'RE CONSOLIDATING DATA AND CREATING REPORTS -# and everything can be rerun safely if the data are present -############################################################ +# and everything can be rerun safely provided the data are present +#################################################################################### -log2cw $NJLOGGRP $NJLOGSTREAM "start updateIndexPages" nightlyJob +# update the website daily, monthly and annual index pages where needed $SRC/website/updateIndexPages.sh # consolidate the output of the match process for further analysis -log2cw $NJLOGGRP $NJLOGSTREAM "start consolidateOutput" nightlyJob $SRC/analysis/consolidateOutput.sh ${yr} if [ "$(date +%m%d)" == "0101" ] ; then # catch any data uploaded on 01/01 that is for 31/12 the previous year @@ -81,56 +74,41 @@ if [ "$(date +%m%d)" == "0101" ] ; then fi # update the search indexes used on the website -log2cw $NJLOGGRP $NJLOGSTREAM "start createSearchable pass 2" nightlyJob $SRC/analysis/createSearchable.sh $yr matches # add daily report to the website -log2cw $NJLOGGRP $NJLOGSTREAM "start publishDailyReport" nightlyJob $SRC/website/publishDailyReport.sh # create monthly and per-shower CSV extracts of the data -log2cw $NJLOGGRP $NJLOGSTREAM "start createMthlyExtracts" nightlyJob ${SRC}/website/createMthlyExtracts.sh ${mth} - -log2cw $NJLOGGRP $NJLOGSTREAM "start createShwrExtracts" nightlyJob ${SRC}/website/createShwrExtracts.sh ${rundate} # create the fireballs page -log2cw $NJLOGGRP $NJLOGSTREAM "start createFireballPage" nightlyJob #requires search index to have been updated first ${SRC}/website/createFireballPage.sh ${yr} -3.99 # create a report of activity for the current month and whole year -log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $mth" nightlyJob $SRC/analysis/showerReport.sh ALL ${mth} force - -log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $yr" nightlyJob $SRC/analysis/showerReport.sh ALL ${yr} force # if we ran on the 1st of the month we need to catch any late-arrivals for last month if [ $(date +%d) -eq 1 ] ; then lastmth=$(date -d '-1 month' +%Y%m) - log2cw $NJLOGGRP $NJLOGSTREAM "start showerMthlyExtracts ALL $lastmth" nightlyJob ${SRC}/website/createMthlyExtracts.sh ${lastmth} - log2cw $NJLOGGRP $NJLOGSTREAM "start createShwrExtracts ALL $lastmth" nightlyJob ${SRC}/website/createShwrExtracts.sh ${lastmth}28 - log2cw $NJLOGGRP $NJLOGSTREAM "start showerReport ALL $lastmth" nightlyJob $SRC/analysis/showerReport.sh ALL ${lastmth} force fi # create a per-shower report for any currently active showers -log2cw $NJLOGGRP $NJLOGSTREAM "start reportActiveShowers" nightlyJob ${SRC}/analysis/reportActiveShowers.sh ${yr} # create the website front page -log2cw $NJLOGGRP $NJLOGSTREAM "start createSummaryTable" nightlyJob ${SRC}/website/createSummaryTable.sh # create the camera status reports -log2cw $NJLOGGRP $NJLOGSTREAM "start cameraStatusReport" nightlyJob ${SRC}/website/cameraStatusReport.sh $rundate -log2cw $NJLOGGRP $NJLOGSTREAM "start createExchangeFiles" nightlyJob +# create the files for exchange with other networks - i think this is unused python -c "from reports.createExchangeFiles import createAll;createAll();" aws s3 sync $DATADIR/browse/daily/ $WEBSITEBUCKET/browse/daily/ --quiet @@ -142,25 +120,17 @@ aws s3 cp $DATADIR/stations.png $WEBSITEBUCKET/ --quiet rm -f $SRC/data/.nightly_running # various reports for management - bad stations, costs, next batch start time. -log2cw $NJLOGGRP $NJLOGSTREAM "start getBadStations" nightlyJob $SRC/analysis/getBadStations.sh - -log2cw $NJLOGGRP $NJLOGSTREAM "start costReport" nightlyJob $SRC/website/costReport.sh - -# set time of next run python $PYLIB/maintenance/getNextBatchStart.py 150 # create station reports. This takes a while hence done after everything else -log2cw $NJLOGGRP $NJLOGSTREAM "start stationReports" nightlyJob $SRC/analysis/stationReports.sh # clear down space where possible -log2cw $NJLOGGRP $NJLOGSTREAM "start clearSpace" nightlyJob $SRC/utils/clearSpace.sh # load the MariaDB with the latest data. The mariadb database isn't used much -log2cw $NJLOGGRP $NJLOGSTREAM "update MariaDB tables" nightlyJob $SRC/utils/loadMatchCsvMDB.sh $SRC/utils/loadSingleCsvMDB.sh @@ -168,12 +138,10 @@ $SRC/utils/loadSingleCsvMDB.sh # has to be run quite late as not all trajectories have synced to the website earlier $SRC/analysis/updatePlotsAndDetStatus.sh +# push the API data dictionary to the website for end-user use aws s3 sync $SRC/share/ s3://ukmda-website/browse --exclude "*" --include "datadictionary.xlsx" --quiet -log2cw $NJLOGGRP $NJLOGSTREAM "finished nightlyJob" nightlyJob +logger -s -t nightlyJog "finished nightlyJob" # grab the logs for the website - run this last to capture the above Finished message $SRC/analysis/getLogData.sh - - - From 0e748e941f5af06fad07b9db5e10d100a1f104e7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 18:45:10 +0100 Subject: [PATCH 213/287] bugfixes in process to gather consumed image ifo --- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 3 ++- archive/ukmon_pylib/utils/getUncalImages.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index d3b6c99c..99c28d1c 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -108,6 +108,7 @@ def SyncRawData(outf, matchstart, matchend, shbucket): # def gatherUsedImageList(outf, matchstart, matchend, shbucket): + shbucket = shbucket.replace('RMSCorrelate','consumed') for d in range(matchend, matchstart+1): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) yr = thisdt.year @@ -116,7 +117,7 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): trajloc = f'trajectories/{yr}/{yr}{mth:02d}/{yr}{mth:02d}{dy:02d}' out_dir = '~/data/distrib' outf.write(f'python -c "from traj.pickleAnalyser import getAllImages;getAllImages(\'{trajloc}\', \'{out_dir}\');"\n') - outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt --quiet"\n') + outf.write(f'aws s3 sync {out_dir} {shbucket}/ --exclude "*" --include "consumed_*.txt" --quiet\n') outf.write(f'rm {out_dir}/consumed_*.txt\n') return diff --git a/archive/ukmon_pylib/utils/getUncalImages.py b/archive/ukmon_pylib/utils/getUncalImages.py index 58c890ff..332268ea 100644 --- a/archive/ukmon_pylib/utils/getUncalImages.py +++ b/archive/ukmon_pylib/utils/getUncalImages.py @@ -9,7 +9,7 @@ def getUncalibratedImageList(dtstr=None): logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') flines = open(logfile, 'r').readlines() uncal = [f for f in flines if 'not recalibrated' in f] - imglist = [x.strip(' ').split(' ')[1][:-1] for x in uncal] + imglist = [x.split('Skipping ')[1].split(',')[0] for x in uncal] with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: for li in imglist: outf.write(f'{li}\n') From 6b257607e8340b9d1c7d44f6a258084ce46767a2 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 19:37:38 +0100 Subject: [PATCH 214/287] update getUncal to work for historic data --- archive/lambdas/matchDataApi/matchDataApi.py | 2 +- archive/ukmon_pylib/utils/getUncalImages.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/archive/lambdas/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py index 3000292d..73c393db 100644 --- a/archive/lambdas/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -81,7 +81,7 @@ def getStationData(statid, dtstr, period=None): def getSummaryData(dtstr, period=None): host, user, passwd, db = getSqlLoginDetails() perfrag = periodToSqlFragment(period) if period is not None else "" - connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' diff --git a/archive/ukmon_pylib/utils/getUncalImages.py b/archive/ukmon_pylib/utils/getUncalImages.py index 332268ea..c07c4b25 100644 --- a/archive/ukmon_pylib/utils/getUncalImages.py +++ b/archive/ukmon_pylib/utils/getUncalImages.py @@ -6,7 +6,10 @@ def getUncalibratedImageList(dtstr=None): datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') + if dtstr is not None: + logfile = os.path.join(datadir, '..', 'logs', f'matchJob.log.{dtstr}') + else: + logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') flines = open(logfile, 'r').readlines() uncal = [f for f in flines if 'not recalibrated' in f] imglist = [x.split('Skipping ')[1].split(',')[0] for x in uncal] From 57e4486536d20f2c00ead994e508db1b616ecd82 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 19:39:13 +0100 Subject: [PATCH 215/287] typo --- archive/analysis/updatePlotsAndDetStatus.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index 158942ea..36ab4908 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -83,7 +83,7 @@ aws ec2 stop-instances --instance-ids $SERVERINSTANCEID log2cw $NJLOGGRP $NJLOGSTREAM "get a list of uncalibrated data" updatePlotsAndDetStatus aws s3 sync $UKMONSHAREDBUCKET/matches/consumed/ $DATADIR/single/used/ --exclude "*" --include "*.txt" --quiet rundate=$(cat $DATADIR/rundate.txt) -python -c "from utils.getUnCalImages import getUncalibratedImageList;getUncalibratedImageList('$rundate');" +python -c "from utils.getUncalImages import getUncalibratedImageList;getUncalibratedImageList('$rundate');" # and then clear the profile again unset AWS_PROFILE From 192f366120f942c8fc09bfd3892027320dc1acd6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 19:41:36 +0100 Subject: [PATCH 216/287] improvements in getUncalImages --- archive/ukmon_pylib/utils/getUncalImages.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/utils/getUncalImages.py b/archive/ukmon_pylib/utils/getUncalImages.py index c07c4b25..11f975c3 100644 --- a/archive/ukmon_pylib/utils/getUncalImages.py +++ b/archive/ukmon_pylib/utils/getUncalImages.py @@ -1,12 +1,14 @@ # Copyright (C) 2018-2023 Mark McIntyre # -import os +import os +import datetime def getUncalibratedImageList(dtstr=None): datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - if dtstr is not None: + now = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y%m%d') + if dtstr is not None and dtstr != now: logfile = os.path.join(datadir, '..', 'logs', f'matchJob.log.{dtstr}') else: logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') From 4d1eb921031bf1efae35c7cc36b4a0949ab8fa2b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 19:43:26 +0100 Subject: [PATCH 217/287] bugfix --- archive/ukmon_pylib/utils/getUncalImages.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/archive/ukmon_pylib/utils/getUncalImages.py b/archive/ukmon_pylib/utils/getUncalImages.py index 11f975c3..381c96a7 100644 --- a/archive/ukmon_pylib/utils/getUncalImages.py +++ b/archive/ukmon_pylib/utils/getUncalImages.py @@ -9,13 +9,17 @@ def getUncalibratedImageList(dtstr=None): datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) now = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%Y%m%d') if dtstr is not None and dtstr != now: - logfile = os.path.join(datadir, '..', 'logs', f'matchJob.log.{dtstr}') + logfile = os.path.join(datadir, '..', 'logs', f'matchJob.log-{dtstr}') else: logfile = os.path.join(datadir, '..', 'logs', 'matchJob.log') - flines = open(logfile, 'r').readlines() - uncal = [f for f in flines if 'not recalibrated' in f] - imglist = [x.split('Skipping ')[1].split(',')[0] for x in uncal] - with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: - for li in imglist: - outf.write(f'{li}\n') + if os.path.isfile(logfile): + flines = open(logfile, 'r').readlines() + uncal = [f for f in flines if 'not recalibrated' in f] + imglist = [x.split('Skipping ')[1].split(',')[0] for x in uncal] + with open(os.path.join(datadir, 'single', 'used', f'uncal_{dtstr}.txt'), 'w') as outf: + for li in imglist: + outf.write(f'{li}\n') + else: + print('file not found', logfile) + imglist=[] return imglist From 06b2b19fa68084eb7d381bb04ec1b59c33ccd642 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 20:06:21 +0100 Subject: [PATCH 218/287] add python to get unused images --- archive/analysis/getUsedUnused.py | 65 +++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 archive/analysis/getUsedUnused.py diff --git a/archive/analysis/getUsedUnused.py b/archive/analysis/getUsedUnused.py new file mode 100644 index 00000000..3d1b6f54 --- /dev/null +++ b/archive/analysis/getUsedUnused.py @@ -0,0 +1,65 @@ +# +# Python code to analyse meteor shower data +# +# Copyright (C) 2018-2023 Mark McIntyre + +import os +import boto3 +import json +import pymysql.cursors + + +def getSqlLoginDetails(): + # retrieve password and host from SSM. This allows me to manage them from Terraform + ssm = boto3.client('ssm', region_name='eu-west-1') + res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) + password = res['Parameter']['Value'] + res = ssm.get_parameter(Name='prod_dbhost') + host = res['Parameter']['Value'] + # should really do these too but they won't change often if at all + user = 'batch' + db = 'ukmon' + return host, user, password, db + + +def getAllImages(dtstr): + """ + Retrieve a list of all images for a given date from the SQL database + This is a list of all images containing detections, whether or not used + in a solution. + """ + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, database=db, cursorclass=pymysql.cursors.DictCursor) + sqlstr = f'select filname from singles where Y={dtstr[:4]} and M={int(dtstr[4:6])} and D={int(dtstr[6:8])}' + with connection.cursor() as cursor: + cursor.execute(sqlstr) + result = cursor.fetchall() + connection.close() + res=[] + for event in result: + res.append(event['filname']) + return res + + +def getKnownImages(dtstr, datatype='consumed'): + """ + Get a list of all images that were involved in a match or were uncalibrated, + depending on the value of datatype (consumed or uncal) + The raw data are generated by a process on the matching engine server + """ + datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + srcfile = os.path.join(datadir,'single','used',f'{datatype}_{dtstr}.txt') + imglist = [] + if os.path.isfile(srcfile): + used = open(srcfile,'r').readlines() + for img in used: + imglist.append(img.strip()) + return imglist + + +def getUnusedImages(dtstr): + + allimages = getAllImages(dtstr) + uncal = getKnownImages(dtstr, 'uncal') + used = getKnownImages(dtstr, 'consumed') + unused = list(set(allimages)-set(used)-set(uncal)) \ No newline at end of file From c6d4893b40e02c419a4dba633c19748553e81207 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 29 Apr 2026 20:09:51 +0100 Subject: [PATCH 219/287] move script to right place --- archive/{ => ukmon_pylib}/analysis/getUsedUnused.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename archive/{ => ukmon_pylib}/analysis/getUsedUnused.py (100%) diff --git a/archive/analysis/getUsedUnused.py b/archive/ukmon_pylib/analysis/getUsedUnused.py similarity index 100% rename from archive/analysis/getUsedUnused.py rename to archive/ukmon_pylib/analysis/getUsedUnused.py From db3e093d3e4039806cdfe81ccc60fec80546a8f3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 08:43:32 +0100 Subject: [PATCH 220/287] comment out unnecessary echo --- archive/ukmon_pylib/reports/createSearchableFormat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/reports/createSearchableFormat.py b/archive/ukmon_pylib/reports/createSearchableFormat.py index 4424cc4b..c567fa57 100644 --- a/archive/ukmon_pylib/reports/createSearchableFormat.py +++ b/archive/ukmon_pylib/reports/createSearchableFormat.py @@ -23,7 +23,7 @@ def checkUrl(s3, url): except Exception: #print(e) retval = '/img/missing-white.png' - print(url, retval) + #print(url, retval) return retval From 44c07e931412076fb9a47e7fc00aedd9d2f3a0a8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 10:40:39 +0100 Subject: [PATCH 221/287] bugfixes in camMetrics while migrating user accts --- archive/ukmon_pylib/metrics/camMetrics.py | 75 ++++++++++------------- archive/website/cameraStatusReport.sh | 7 +++ 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index e36f49f3..db31fe14 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -151,20 +151,6 @@ def backPopulate(stationid): datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) ddb = boto3.resource('dynamodb', region_name='eu-west-2') - if os.path.isfile('/sys/devices/virtual/dmi/id/board_asset_tag'): # crude check for EC2 - #print('asset tag file exists') - lis = open('/sys/devices/virtual/dmi/id/board_asset_tag').readlines() - if 'i-' in lis[0]: - sts_client = boto3.client('sts') - assumed_role_object=sts_client.assume_role( - RoleArn="arn:aws:iam::183798037734:role/service-role/S3FullAccess", - RoleSessionName="AssumeRoleSession1") - credentials=assumed_role_object['Credentials'] - - ddb = boto3.resource('dynamodb', region_name='eu-west-2', - aws_access_key_id=credentials['AccessKeyId'], - aws_secret_access_key=credentials['SecretAccessKey'], - aws_session_token=credentials['SessionToken']) s,d,t,m,r = getDayCamTimings(sys.argv[1], ddb=ddb) newdata=pd.DataFrame(zip(s,d,t,m,r), columns=['stationid','upddate','uploadtime','manual','rundate']) @@ -206,33 +192,34 @@ def backPopulate(stationid): location = spls[1].split(' for ')[1].split()[0] lodata.append({'location':location, 'lastseen':dtval}) - - logindf = pd.DataFrame(lodata) - logindf = logindf.sort_values(by=['lastseen']) - logindf.drop_duplicates(subset=['location'], inplace=True, keep='last') - - # create a merged dataframe with siteid and stationid - intdf = pd.merge(logindf,caminfo, on=['location'], how='outer') - - df = pd.merge(intdf, fulldf, on=['stationid']) - df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") - df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime - df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] - df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) - df['dateval']=[x.strftime('%b-%d') for x in df.lastupload] - df = df.sort_values(by=['lastupload']) - - outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') - zerodate = datetime.datetime(1970,1,1,0,0,0) - with open(outfile,'w') as outf: - outf.write('Last Upload, StationID, Last Login\n') - for _,rw in df.iterrows(): - dtval = rw.dateval - lastup = rw.lastupload.strftime('%H:%M:%S') - if pd.isnull(rw.lastseen): - lastseen = '> 1 month' - else: - lastseen = rw.lastseen.strftime('%b-%d %H:%M:%S') - if lastseen == 'Jan-01 00:00:00': - lastseen = '> 1 month' - outf.write(f'{dtval}, {lastup}, {rw.location:20s}, {lastseen}\n') + + if len(lodata) > 0: + logindf = pd.DataFrame(lodata) + logindf = logindf.sort_values(by=['lastseen']) + logindf.drop_duplicates(subset=['location'], inplace=True, keep='last') + + # create a merged dataframe with siteid and stationid + intdf = pd.merge(logindf,caminfo, on=['location'], how='outer') + + df = pd.merge(intdf, fulldf, on=['stationid']) + df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") + df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime + df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] + df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) + df['dateval']=[x.strftime('%b-%d') for x in df.lastupload] + df = df.sort_values(by=['lastupload']) + + outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') + zerodate = datetime.datetime(1970,1,1,0,0,0) + with open(outfile,'w') as outf: + outf.write('Last Upload, StationID, Last Login\n') + for _,rw in df.iterrows(): + dtval = rw.dateval + lastup = rw.lastupload.strftime('%H:%M:%S') + if pd.isnull(rw.lastseen): + lastseen = '> 1 month' + else: + lastseen = rw.lastseen.strftime('%b-%d %H:%M:%S') + if lastseen == 'Jan-01 00:00:00': + lastseen = '> 1 month' + outf.write(f'{dtval}, {lastup}, {rw.location:20s}, {lastseen}\n') diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh index 00cf151e..6a2f4eb4 100644 --- a/archive/website/cameraStatusReport.sh +++ b/archive/website/cameraStatusReport.sh @@ -31,6 +31,13 @@ platform=$(grep "^NAME" /etc/os-release | awk -F'"' '{print $2}') [ "$platform" == "Ubuntu" ] && authlog=/var/log/auth.log || authlog=/var/log/secure sudo grep publickey $authlog* | grep -v $USER > $DATADIR/reports/lastlogins.txt +echo TEMPORARY HACK REMOVE ME +if [ "$platform" == "Ubuntu" ] +then + ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | grep -v ec2-user" > $DATADIR/reports/lastlogins.txt +fi +echo TO HERE + python -m metrics.camMetrics $rundate rm -f $DATADIR/reports/lastlogins.txt aws s3 cp $DATADIR/reports/stationlogins.txt $WEBSITEBUCKET/reports/stationlogins.txt --region eu-west-2 --quiet From 99ff82f773a9a530a0871a574dca6c3b5e6f2971 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 11:01:07 +0100 Subject: [PATCH 222/287] fix #448 --- archive/ukmon_pylib/analysis/showerAnalysis.py | 4 ++-- archive/ukmon_pylib/analysis/stationAnalysis.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/archive/ukmon_pylib/analysis/showerAnalysis.py b/archive/ukmon_pylib/analysis/showerAnalysis.py index c9d01ace..7defd593 100644 --- a/archive/ukmon_pylib/analysis/showerAnalysis.py +++ b/archive/ukmon_pylib/analysis/showerAnalysis.py @@ -97,7 +97,7 @@ def timeGraph(dta, shwrname, outdir, binmins=10): countcol = dta['Shwr'] # resample it - binned = countcol.resample('{}min'.format(binmins)).count() + binned = countcol.resample(f'{binmins}min').count() binned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals @@ -138,7 +138,7 @@ def matchesGraphs(dta, shwrname, outdir, binmins=60, startdt=None, enddt=None, t #select just the shower ID col mcountcol = mdta['_ID1'] # resample it - mbinned = mcountcol.resample('{}min'.format(binmins)).count() + mbinned = mcountcol.resample(f'{binmins}min').count() mbinned.plot(kind='bar', width=10) # set ticks and labels every 144 intervals diff --git a/archive/ukmon_pylib/analysis/stationAnalysis.py b/archive/ukmon_pylib/analysis/stationAnalysis.py index fb7a2e9d..7e888ae4 100644 --- a/archive/ukmon_pylib/analysis/stationAnalysis.py +++ b/archive/ukmon_pylib/analysis/stationAnalysis.py @@ -77,7 +77,7 @@ def timeGraph(dta, loc, outdir, when, sampleinterval): #select just the shower ID col countcol = dta['ID'] # resample it - binned = countcol.resample('{}'.format(sampleinterval)).count() + binned = countcol.resample(f'{sampleinterval}').count() binned.plot(kind='bar') # set ticks and labels every 144 intervals @@ -100,7 +100,7 @@ def timeGraph(dta, loc, outdir, when, sampleinterval): pass fname = os.path.join(outdir, 'station_plot_timeline_single.jpg') - plt.title('Observed stream activity {} intervals for {} in {}'.format(sampleinterval, loc, when)) + plt.title(f'Observed stream activity {sampleinterval} intervals for {loc} in {when}') try: plt.tight_layout() plt.savefig(fname) @@ -311,7 +311,7 @@ def pushToWebsite(fuloutdir, outdir, websitebucket): idlist = list(camlistfltr.stationid) if mth is None: - sampleinterval="1M" + sampleinterval="1ME" shortoutdir = os.path.join('reports', str(yr), 'stations', loc) outdir = os.path.join(datadir, shortoutdir) else: From 57d9bf3f04f24d76040c547df86d9ef04b59b333 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 11:44:11 +0100 Subject: [PATCH 223/287] make sure window title is sensible --- archive/server_setup/.bashrc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/archive/server_setup/.bashrc b/archive/server_setup/.bashrc index a456684e..a7f39fa3 100644 --- a/archive/server_setup/.bashrc +++ b/archive/server_setup/.bashrc @@ -55,6 +55,14 @@ fi unset __conda_setup # <<< conda initialize <<< +# If this is an xterm set the title to user@host:dir +case "$TERM" in + xterm*|rxvt*) + PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" + ;; + *) + ;; +esac export rundate=$(date +%Y%m%d) conda activate wmpl From 3eed2b21fce6e57644ea9bd8edae8b482343af2c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 12:10:11 +0100 Subject: [PATCH 224/287] bugfix in replot and get unused --- archive/analysis/runDistrib.sh | 12 +++++++----- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 3 +-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index eb690991..536527c4 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -76,11 +76,13 @@ while [ $? -ne 0 ] ; do scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist done # push the python code and ECS templates required -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$$CALCSERVERIP:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj -rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/clusdetails-* $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/taskrunner*.json $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/consolidateDistTraj.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/distributeCandidates.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/ShowerAssociation.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/traj/ +rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/utils/convertSolLon.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/utils/ # now run the script log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 99c28d1c..953e7d72 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -108,7 +108,6 @@ def SyncRawData(outf, matchstart, matchend, shbucket): # def gatherUsedImageList(outf, matchstart, matchend, shbucket): - shbucket = shbucket.replace('RMSCorrelate','consumed') for d in range(matchend, matchstart+1): thisdt=datetime.datetime.now() + datetime.timedelta(days=-d) yr = thisdt.year @@ -117,7 +116,7 @@ def gatherUsedImageList(outf, matchstart, matchend, shbucket): trajloc = f'trajectories/{yr}/{yr}{mth:02d}/{yr}{mth:02d}{dy:02d}' out_dir = '~/data/distrib' outf.write(f'python -c "from traj.pickleAnalyser import getAllImages;getAllImages(\'{trajloc}\', \'{out_dir}\');"\n') - outf.write(f'aws s3 sync {out_dir} {shbucket}/ --exclude "*" --include "consumed_*.txt" --quiet\n') + outf.write(f'aws s3 sync {out_dir} {shbucket}/matches/consumed/ --exclude "*" --include "consumed_*.txt" --quiet\n') outf.write(f'rm {out_dir}/consumed_*.txt\n') return From 46ef42f5b5d6841bcc5e3695ad0dadf62754aaa2 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 12:21:51 +0100 Subject: [PATCH 225/287] bugfix in removeDeletedTraj for search index --- archive/ukmon_pylib/maintenance/dataMaintenance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index d67cff95..7373c314 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -128,6 +128,7 @@ def removeDeletedTraj(csvfile): # loop over the deleted trajectories, removing the corresponding one from the CSV file. # Note that there's no need to update the MariaDB database, as it is populated # from the CSV file *after* it has been purged + offs = 4 if 'search' in csvfile else 131 for traj in deltrajs: cur = masterdb.dbhandle.execute(f'select jdt_ref, participating_stations from trajectories where traj_file_path="{traj[0]}" and status=0') thistraj = cur.fetchall() @@ -141,7 +142,7 @@ def removeDeletedTraj(csvfile): obs_ids = thistraj[0][1] obs_ids_str = ';'.join(json.loads(obs_ids)) + ';' for thismtch in match: - if thismtch.split(',')[131] == obs_ids_str: + if thismtch.split(',')[offs] == obs_ids_str: print(f'removing {fldr}') idx = csvdata.index(thismtch) _ = csvdata.pop(idx) From 2e511b4d39aefdd9ecf598d172ce6acbc8a488d1 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 12:32:31 +0100 Subject: [PATCH 226/287] remove deprecation warning issue #450 --- archive/ukmon_pylib/metrics/camMetrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index db31fe14..2155a7e1 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -182,8 +182,7 @@ def backPopulate(stationid): # could be ubuntu (auth.log) or amazon linux style log dtstr = spls[0][spls[0].find(':')+1:][:19] if ' ' in dtstr: - dtval = datetime.datetime.strptime(dtstr[:15], '%b %d %H:%M:%S') - dtval = dtval.replace(year=now.year, tzinfo=datetime.timezone.utc) + dtval = datetime.datetime.strptime(f'{now.year} {dtstr[:15]}', '%Y %b %d %H:%M:%S') if dtval > now: dtval = dtval.replace(year=now.year-1, tzinfo=datetime.timezone.utc) else: From 41c37c0542cb7c05a247f82c70273e571fce7cd0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 12:37:23 +0100 Subject: [PATCH 227/287] whoops naive vs tzaware dates --- archive/ukmon_pylib/metrics/camMetrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 2155a7e1..cd0d3fa2 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -182,7 +182,7 @@ def backPopulate(stationid): # could be ubuntu (auth.log) or amazon linux style log dtstr = spls[0][spls[0].find(':')+1:][:19] if ' ' in dtstr: - dtval = datetime.datetime.strptime(f'{now.year} {dtstr[:15]}', '%Y %b %d %H:%M:%S') + dtval = datetime.datetime.strptime(f'{now.year} {dtstr[:15]}', '%Y %b %d %H:%M:%S').replace(tzinfo=datetime.timezone.utc) if dtval > now: dtval = dtval.replace(year=now.year-1, tzinfo=datetime.timezone.utc) else: From f68185d28ce64d663fa16baee8579436686fac8f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 12:49:15 +0100 Subject: [PATCH 228/287] remove warning in cameraStatusReport issue #453 --- archive/ukmon_pylib/reports/cameraStatusReport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/reports/cameraStatusReport.py b/archive/ukmon_pylib/reports/cameraStatusReport.py index 256ad076..83e4850e 100644 --- a/archive/ukmon_pylib/reports/cameraStatusReport.py +++ b/archive/ukmon_pylib/reports/cameraStatusReport.py @@ -30,7 +30,7 @@ def getLastUpdateDate(datadir=None, camfname=None, ddb=None): fldrlist = pd.merge(tmplist, caminfo, on=['stationid'], how='inner') nowdt = datetime.datetime.now() - fldrlist.rundate.fillna(fldrlist.upddate.astype(str) + '_' + fldrlist.uploadtime.astype(str).str.pad(width=6,fillchar='0'), inplace=True) + fldrlist.fillna({'rundate': fldrlist.upddate.astype(str) + '_' + fldrlist.uploadtime.astype(str).str.pad(width=6,fillchar='0')}, inplace=True) fldrlist['dtstamp'] = [datetime.datetime.strptime(f,'%Y%m%d_%H%M%S') for f in fldrlist.rundate] fldrlist = fldrlist.sort_values(by=['stationid','dtstamp']) fldrlist.drop_duplicates(keep='last', subset=['stationid'], inplace=True) From 7ed5362e90d067a598415e4444637bfd0132ee8b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 12:57:34 +0100 Subject: [PATCH 229/287] remove warning issue #454 --- archive/ukmon_pylib/converters/toParquet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/converters/toParquet.py b/archive/ukmon_pylib/converters/toParquet.py index c9454b7b..a147501c 100644 --- a/archive/ukmon_pylib/converters/toParquet.py +++ b/archive/ukmon_pylib/converters/toParquet.py @@ -15,7 +15,7 @@ if 'match' in fn: # fill in any #NAs in the mjd column - df.mjd.fillna(df._mjd, inplace=True) + df.fillna({'mjd': df._mjd}, inplace=True) if 'single' in fn: df = df.drop_duplicates() From f981ef5be688c970e8959ca3a36c4549d9f3f7e0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 13:20:38 +0100 Subject: [PATCH 230/287] widen constraints on analysing pickles to get images --- archive/ukmon_pylib/traj/pickleAnalyser.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/traj/pickleAnalyser.py b/archive/ukmon_pylib/traj/pickleAnalyser.py index b9da8919..16f1fd4b 100644 --- a/archive/ukmon_pylib/traj/pickleAnalyser.py +++ b/archive/ukmon_pylib/traj/pickleAnalyser.py @@ -585,19 +585,20 @@ def getListOfImages(picklename): class TrajQualityParams(object): def __init__(self): - self.min_traj_points = 6 - self.min_qc = 5.0 - self.max_e = 1.5 - self.max_radiant_err = 2.0 - self.max_vg_err = 10.0 + self.min_traj_points = 4 + self.min_qc = 3.0 + self.max_e = 2.5 + self.max_radiant_err = 5.0 + self.max_vg_err = 20.0 self.max_vg = 120.0 self.max_begin_ht = 160 - self.min_end_ht = 20 + self.min_end_ht = 0 def getAllImages(dir_path, outdir): outdir = os.path.expanduser(outdir) traj_quality_params = TrajQualityParams() + trajs = loadTrajectoryPickles(dir_path, traj_quality_params, verbose=True) imglist = [] for traj in trajs: From 36a8ed9b51956acd49267c9e8cbf6c29b664ed73 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 14:21:36 +0100 Subject: [PATCH 231/287] add migration script --- archive/server_setup/migrateSftpAccts.sh | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 archive/server_setup/migrateSftpAccts.sh diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh new file mode 100644 index 00000000..50a63ad7 --- /dev/null +++ b/archive/server_setup/migrateSftpAccts.sh @@ -0,0 +1,26 @@ +#!/bin/bash + + +addOneUser() { + userid=$1 + grep -w $userid /etc/passwd + if [ $? -eq 1 ] ; then + dt=$(date +%Y-%m-%d) + logger -s -t addSftpUser "Creating unix user $userid" + sudo useradd --system --shell /usr/sbin/nologin --groups sftp --home /var/sftp/$userid --comment "${dt}" $userid + sudo mkdir /var/sftp/$userid + sudo chown root:sftp /var/sftp/$userid + sudo chmod 751 /var/sftp/$userid + # create the .ssh folder, platepar folder and empty client copy of the ini file + sudo mkdir /var/sftp/$userid/.ssh + sudo mkdir /var/sftp/$userid/platepar + sudo touch /var/sftp/$userid/ukmon.ini.client + # make these three writeable by the client + sudo chown $userid:$userid /var/sftp/$userid/platepar /var/sftp/$userid/.ssh /var/sftp/$userid/ukmon.ini.client + else + logger -s -t addSftpUser "Unix user $userid already exists" + fi + rsync -avn 3.11.55.160:/var/sftp/$userid/ /var/sftp/$userid +} + +addOneUser tackley_sw \ No newline at end of file From 0dedbe632a9d527ef9dfc73cf7179c97c86c964a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 14:24:58 +0100 Subject: [PATCH 232/287] add sudo --- archive/server_setup/migrateSftpAccts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh index 50a63ad7..ee9b3699 100644 --- a/archive/server_setup/migrateSftpAccts.sh +++ b/archive/server_setup/migrateSftpAccts.sh @@ -20,7 +20,7 @@ addOneUser() { else logger -s -t addSftpUser "Unix user $userid already exists" fi - rsync -avn 3.11.55.160:/var/sftp/$userid/ /var/sftp/$userid + sudo rsync -avn 3.11.55.160:/var/sftp/$userid/ /var/sftp/$userid } addOneUser tackley_sw \ No newline at end of file From 0235adf42b8cd6c1521be2010a0b9505c62de39a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 14:26:33 +0100 Subject: [PATCH 233/287] actually do it --- archive/server_setup/migrateSftpAccts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh index ee9b3699..c01f430a 100644 --- a/archive/server_setup/migrateSftpAccts.sh +++ b/archive/server_setup/migrateSftpAccts.sh @@ -20,7 +20,7 @@ addOneUser() { else logger -s -t addSftpUser "Unix user $userid already exists" fi - sudo rsync -avn 3.11.55.160:/var/sftp/$userid/ /var/sftp/$userid + sudo rsync -av 3.11.55.160:/var/sftp/$userid/ /var/sftp/$userid } addOneUser tackley_sw \ No newline at end of file From dc95620d8418b74dc346fa46abe81d1dc4466b98 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 15:59:43 +0100 Subject: [PATCH 234/287] update sftp migration script --- archive/server_setup/migrateSftpAccts.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh index c01f430a..18426a8b 100644 --- a/archive/server_setup/migrateSftpAccts.sh +++ b/archive/server_setup/migrateSftpAccts.sh @@ -23,4 +23,7 @@ addOneUser() { sudo rsync -av 3.11.55.160:/var/sftp/$userid/ /var/sftp/$userid } -addOneUser tackley_sw \ No newline at end of file +addOneUser tackley_sw +addOneUser tackley_se +addOneUser tackley_nw +addOneUser tackley_ne \ No newline at end of file From 4b559f0b7a99e5a7b6b1225f7c5f3947125979df Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 16:19:02 +0100 Subject: [PATCH 235/287] more changes to migration script --- archive/server_setup/migrateSftpAccts.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh index 18426a8b..1a1daa5e 100644 --- a/archive/server_setup/migrateSftpAccts.sh +++ b/archive/server_setup/migrateSftpAccts.sh @@ -3,6 +3,7 @@ addOneUser() { userid=$1 + $srchost=$2 grep -w $userid /etc/passwd if [ $? -eq 1 ] ; then dt=$(date +%Y-%m-%d) @@ -20,10 +21,13 @@ addOneUser() { else logger -s -t addSftpUser "Unix user $userid already exists" fi - sudo rsync -av 3.11.55.160:/var/sftp/$userid/ /var/sftp/$userid + echo sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid } -addOneUser tackley_sw -addOneUser tackley_se -addOneUser tackley_nw -addOneUser tackley_ne \ No newline at end of file +oldserver=$1 +srcfile=$2 + +cat $srcfile | while read stn +do + addOneUser $stn $oldserver +done \ No newline at end of file From a91ac87b2cd33d88ed3b66dbd33ba876c275b5c0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 16:21:38 +0100 Subject: [PATCH 236/287] moving sftp migration script --- archive/{server_setup => utils}/migrateSftpAccts.sh | 6 ++++++ 1 file changed, 6 insertions(+) rename archive/{server_setup => utils}/migrateSftpAccts.sh (84%) diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/utils/migrateSftpAccts.sh similarity index 84% rename from archive/server_setup/migrateSftpAccts.sh rename to archive/utils/migrateSftpAccts.sh index 1a1daa5e..bc46d62f 100644 --- a/archive/server_setup/migrateSftpAccts.sh +++ b/archive/utils/migrateSftpAccts.sh @@ -24,6 +24,12 @@ addOneUser() { echo sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid } +echo "Warning: this must only be run on the new server" +read -p "press ctrl-c to quit or enter to continue" +if [ $# -lt 3 ] ; then + echo "Usage: ./migrateSftpAccounts.sh oldservername userfile" + exit +fi oldserver=$1 srcfile=$2 From 27ab5352d36012f70ba1bcfc26f4b31b43eebe7e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 16:25:28 +0100 Subject: [PATCH 237/287] update messages and docos --- archive/server_setup/migratingBatchServer.md | 92 +++++++++----------- archive/utils/migrateSftpAccts.sh | 6 +- 2 files changed, 46 insertions(+), 52 deletions(-) diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index 751a540b..bfd61ab9 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -43,8 +43,6 @@ python setup.py cd ~/src git clone git@github.com:ukmda/ukmda-dataprocessing.git cd ukmda-dataprocessing -conda activate wmpl -pip install -r additional_requirements.txt ./install_or_upgrade.sh PROD ``` * double check that all required SSM variables exist in the account holding the server. These @@ -55,7 +53,7 @@ aws ssm get-parameters --region eu-west-2 --names prod_siteurl --query Parameter ``` ### SSH and other API keys -copy the contents of ~/.ssh on the old server, and make sure permissions are correct (should all be 0600). At a minimum you need the following files +copy the below files from ~/.ssh on the old server, and make sure permissions are correct (should all be 0600). At a minimum you need the following files ``` bash gmailcreds.json gmailtoken.json @@ -65,16 +63,20 @@ These are used to authenticate against gmail. The keys are currently specific to You may also need the `github` keys though this depends on how you authenticate with GitHub. ## data -Replicate `~/prod/data` to the new server. (once for prod and once for dev). This will have to be repeated every day till golive. +Replicate `~/prod/data`, `~/prod/logs` and `~/keymgmt` to the new server. (once for prod and once for dev). + + This will have to be repeated every day till golive (strictly, the key data only needs to be replicated if a new camera is added). Also keep the MariaDB SQL database up to date. ``` bash cd $DATADIR rsync -avz ukmonhelper2:prod/data/ . -``` -Replicate the contents of `~/keymgmt` to the new server. Repeat if any new cameras added before golive. -``` bash +cd $SRC/logs +rsync -avz ukmonhelper2:prod/logs/ . +cd ~ rsync -avz ukmonhelper2:keymgmt/ ./keymgmt -``` +$SRC/utils/loadSingleCsvMDB.sh +$SRC/utils/loadMatchCsvMDB.sh +``` ## Mariadb database Sudo to root and run the following to set a root password ``` bash @@ -83,7 +85,7 @@ ALTER USER 'root'@'localhost' IDENTIFIED VIA mysql_native_password USING PASSWOR FLUSH PRIVILEGES; quit ``` -(obviously, replace `ROOTPASSWORD` with the password in the SSM variable `prod_dbpw`) +Pbviously, replace `ROOTPASSWORD` with the password in the SSM variable `prod_dbpw` Exit the root shell, and as a 'normal' user execute the following: ``` bash @@ -121,54 +123,44 @@ conda activate ${WMPL_ENV} ``` ## how to move accounts to a new server -The basic process is to extract the user accounts from the current server along with -group and password info, then import it back in on the new server. Its important to avoid -accidentally overwriting system or otherwise-existing accounts on the new server. -On most systems, accounts below 500 are system accounts. Check though, as some AWS servers create -new accounts starting at 1000 and working both upwards and downwards! +### Install and configure sftp +First set up the SFTP server on the new host. +Run the following as root: -Steps in brief. NB must all be done as root, of course. - -### On the old server ``` bash -mkdir /root/move/ -export UGIDLIMIT=500 -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/passwd > /root/move/passwd.mig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/group > /root/move/group.mig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534) {print $1}' /etc/passwd | tee - |egrep -f - /etc/shadow > /root/move/shadow.mig -cp /etc/gshadow /root/move/gshadow.mig - -# Also backup the user homedirs. In our case, they're all in /var/sftp -tar cvfz /root/move/varsftp.tar.gz /var/sftp -# now copy the files to target server, -scp /root/move/* newserver:/tmp - +groupadd sftp +mkdir -p /var/sftp +chown root:root /var/sftp +chmod 751 /var/sftp ``` -### On the new server -In summary: backup the existing files, remove any accounts from the .mig files -that are already present in the target, then append the filtered data. - -When comparing groups, remember new ids will get added to the sftp group. - - +Add this to /etc/ssh/sshd_config ``` bash -mkdir -p /root/move/bkp -mv /tmp/*.mig /tmp/varsftp.tar.gz /root/move -cp /etc/passwd /etc/group /etc/shadow /etc/gshadow /root/move/bkp +Match group sftp +ChrootDirectory /var/sftp/%u +AllowTCPForwarding no +X11Forwarding no +ForceCommand internal-sftp +``` -cd / -tar -xvf /root/move/varsftp.tar.gz . +and then reload sshd +``` bash +service sshd reload +``` +### Now move the user accounts -export UGIDLIMIT=500 -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/passwd > /root/move/passwd.orig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534)' /etc/group > /root/move/group.orig -awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3!=65534) {print $1}' /etc/passwd | tee - |egrep -f - /etc/shadow > /root/move/shadow.orig +First, ensure that root on the new server can connect to the old server as the batch user by creating a default SSH keypair for root, and adding its public half to root's authorized_keys file on the old server. -cd /root/move -diff passwd.orig passwd.mig -diff shadow.orig shadow.mig -diff group.orig group.mig +* on the new server as the standard batch user: + * create a folder `move` + * Create a list of the desired SFTP user accounts using +``` bash +ssh oldserver "sudo ls -1 /var/sftp" > ./move/sftp_accts.txt ``` - + * Edit the list to exclude defunct accounts and other entries not related to a camera account. + * use `$SRC/$utils/migrateSftpAccts.sh` to create accounts on the new server and copy over the user data +``` bash +$SRC/utils/migrateSftpAccts.sh oldservername ./move/sftp_accts.txt +``` +Once you've completed the process you can remove the `move` folder. \ No newline at end of file diff --git a/archive/utils/migrateSftpAccts.sh b/archive/utils/migrateSftpAccts.sh index bc46d62f..d009e936 100644 --- a/archive/utils/migrateSftpAccts.sh +++ b/archive/utils/migrateSftpAccts.sh @@ -24,12 +24,14 @@ addOneUser() { echo sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid } -echo "Warning: this must only be run on the new server" -read -p "press ctrl-c to quit or enter to continue" if [ $# -lt 3 ] ; then echo "Usage: ./migrateSftpAccounts.sh oldservername userfile" exit fi + +echo "Warning: this must only be run on the new server" +read -p "press ctrl-c to quit or enter to continue" + oldserver=$1 srcfile=$2 From b829c17e3107ff9fb3e990796e698c43f476efb5 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 16:29:44 +0100 Subject: [PATCH 238/287] slight tweak to migration script --- archive/utils/migrateSftpAccts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/utils/migrateSftpAccts.sh b/archive/utils/migrateSftpAccts.sh index d009e936..091c3876 100644 --- a/archive/utils/migrateSftpAccts.sh +++ b/archive/utils/migrateSftpAccts.sh @@ -24,7 +24,7 @@ addOneUser() { echo sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid } -if [ $# -lt 3 ] ; then +if [ $# -lt 2 ] ; then echo "Usage: ./migrateSftpAccounts.sh oldservername userfile" exit fi From 54324dc8c180dcd208187916e7ec35082788b69e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 16:34:19 +0100 Subject: [PATCH 239/287] further work on sftp account migration --- archive/server_setup/migratingBatchServer.md | 2 +- archive/utils/migrateSftpAccts.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index bfd61ab9..7d5463c4 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -161,6 +161,6 @@ ssh oldserver "sudo ls -1 /var/sftp" > ./move/sftp_accts.txt * Edit the list to exclude defunct accounts and other entries not related to a camera account. * use `$SRC/$utils/migrateSftpAccts.sh` to create accounts on the new server and copy over the user data ``` bash -$SRC/utils/migrateSftpAccts.sh oldservername ./move/sftp_accts.txt +$SRC/utils/migrateSftpAccts.sh oldserverFQDN ./move/sftp_accts.txt ``` Once you've completed the process you can remove the `move` folder. \ No newline at end of file diff --git a/archive/utils/migrateSftpAccts.sh b/archive/utils/migrateSftpAccts.sh index 091c3876..79edc657 100644 --- a/archive/utils/migrateSftpAccts.sh +++ b/archive/utils/migrateSftpAccts.sh @@ -3,7 +3,7 @@ addOneUser() { userid=$1 - $srchost=$2 + srchost=$2 grep -w $userid /etc/passwd if [ $? -eq 1 ] ; then dt=$(date +%Y-%m-%d) @@ -21,7 +21,7 @@ addOneUser() { else logger -s -t addSftpUser "Unix user $userid already exists" fi - echo sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid + sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid } if [ $# -lt 2 ] ; then From c65b741f887b94b034c519cfa8fe5694eb3ff2c6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 22:14:35 +0100 Subject: [PATCH 240/287] update ukmon ini files after migration --- archive/utils/migrateSftpAccts.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/archive/utils/migrateSftpAccts.sh b/archive/utils/migrateSftpAccts.sh index 79edc657..c77f5414 100644 --- a/archive/utils/migrateSftpAccts.sh +++ b/archive/utils/migrateSftpAccts.sh @@ -22,6 +22,9 @@ addOneUser() { logger -s -t addSftpUser "Unix user $userid already exists" fi sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid + sudo cat /var/sftp/$userid/ukmon.ini | sed s/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini + sudo mv /tmp/$userid.ini /var/sftp/$userid/ukmon.ini + sudo scp /var/sftp/$userid/ukmon.ini $srchost:/var/sftp/$userid/ } if [ $# -lt 2 ] ; then From dbd346c99c29e7c008406ea7f89b2312763fd195 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 22:16:00 +0100 Subject: [PATCH 241/287] typo --- archive/utils/migrateSftpAccts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/utils/migrateSftpAccts.sh b/archive/utils/migrateSftpAccts.sh index c77f5414..52d5add0 100644 --- a/archive/utils/migrateSftpAccts.sh +++ b/archive/utils/migrateSftpAccts.sh @@ -22,7 +22,7 @@ addOneUser() { logger -s -t addSftpUser "Unix user $userid already exists" fi sudo rsync -av $srchost:/var/sftp/$userid/ /var/sftp/$userid - sudo cat /var/sftp/$userid/ukmon.ini | sed s/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini + sudo cat /var/sftp/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini sudo mv /tmp/$userid.ini /var/sftp/$userid/ukmon.ini sudo scp /var/sftp/$userid/ukmon.ini $srchost:/var/sftp/$userid/ } From 1daa675ecaac1ebc25b600f41fc2f19e04c00a24 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 23:26:28 +0100 Subject: [PATCH 242/287] collect old server login records integration with new --- archive/website/cameraStatusReport.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh index 6a2f4eb4..ea81986a 100644 --- a/archive/website/cameraStatusReport.sh +++ b/archive/website/cameraStatusReport.sh @@ -34,12 +34,15 @@ sudo grep publickey $authlog* | grep -v $USER > $DATADIR/reports/lastlogins.tx echo TEMPORARY HACK REMOVE ME if [ "$platform" == "Ubuntu" ] then - ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | grep -v ec2-user" > $DATADIR/reports/lastlogins.txt + ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | grep -v ec2-user" > $DATADIR/reports/lastlogins_old.txt fi +cat $DATADIR/reports/lastlogins_old.txt >> $DATADIR/reports/lastlogins.txt echo TO HERE python -m metrics.camMetrics $rundate rm -f $DATADIR/reports/lastlogins.txt +rm -f $DATADIR/reports/lastlogins_old.txt + aws s3 cp $DATADIR/reports/stationlogins.txt $WEBSITEBUCKET/reports/stationlogins.txt --region eu-west-2 --quiet From 995c886cf0eaa94cec67ada70bf1c5caf0553965 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 30 Apr 2026 23:52:19 +0100 Subject: [PATCH 243/287] update migration instructions --- archive/server_setup/migratingBatchServer.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index 7d5463c4..7985e0e1 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -148,7 +148,7 @@ and then reload sshd ``` bash service sshd reload ``` -### Now move the user accounts +### Now move the user accounts and update camera config First, ensure that root on the new server can connect to the old server as the batch user by creating a default SSH keypair for root, and adding its public half to root's authorized_keys file on the old server. @@ -163,4 +163,6 @@ ssh oldserver "sudo ls -1 /var/sftp" > ./move/sftp_accts.txt ``` bash $SRC/utils/migrateSftpAccts.sh oldserverFQDN ./move/sftp_accts.txt ``` -Once you've completed the process you can remove the `move` folder. \ No newline at end of file +This will also update the `ukmon.ini` file for each station on both server. Upon next connection, the station should download the new ini file and thereafter connect to the new server. + +This means that the old server must be kept running for a few days, till all cameras have switched over. From d16a0e2eb7b121e02a29a95ac21856594c3ec14a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 1 May 2026 11:59:25 +0100 Subject: [PATCH 244/287] update cam metrics report to include hostname --- archive/ukmon_pylib/metrics/camMetrics.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index cd0d3fa2..36aeaf98 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -185,11 +185,13 @@ def backPopulate(stationid): dtval = datetime.datetime.strptime(f'{now.year} {dtstr[:15]}', '%Y %b %d %H:%M:%S').replace(tzinfo=datetime.timezone.utc) if dtval > now: dtval = dtval.replace(year=now.year-1, tzinfo=datetime.timezone.utc) + targhost = 'ukmonhelper2' else: dtval = datetime.datetime.strptime(dtstr, '%Y-%m-%dT%H:%M:%S') dtval = dtval.replace(tzinfo=datetime.timezone.utc) + targhost = 'batchserver' location = spls[1].split(' for ')[1].split()[0] - lodata.append({'location':location, 'lastseen':dtval}) + lodata.append({'location':location, 'lastseen':dtval,'host': targhost}) if len(lodata) > 0: @@ -205,20 +207,14 @@ def backPopulate(stationid): df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) - df['dateval']=[x.strftime('%b-%d') for x in df.lastupload] df = df.sort_values(by=['lastupload']) outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') zerodate = datetime.datetime(1970,1,1,0,0,0) with open(outfile,'w') as outf: - outf.write('Last Upload, StationID, Last Login\n') + outf.write('Last Upload, StationID, Last Login, Via\n') for _,rw in df.iterrows(): - dtval = rw.dateval - lastup = rw.lastupload.strftime('%H:%M:%S') - if pd.isnull(rw.lastseen): - lastseen = '> 1 month' - else: - lastseen = rw.lastseen.strftime('%b-%d %H:%M:%S') - if lastseen == 'Jan-01 00:00:00': - lastseen = '> 1 month' - outf.write(f'{dtval}, {lastup}, {rw.location:20s}, {lastseen}\n') + lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') + lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') + via = '' if pd.isnull(rw.host) else rw.host + outf.write(f'{lastup} , {rw.location:20s}, {lastlo}, {via}\n') From b565e4d4f05bb6f414b02ec18682d753b13afc1d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 3 May 2026 13:41:04 +0100 Subject: [PATCH 245/287] bugfix as $USER not available in cron --- archive/website/cameraStatusReport.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/archive/website/cameraStatusReport.sh b/archive/website/cameraStatusReport.sh index ea81986a..385fb035 100644 --- a/archive/website/cameraStatusReport.sh +++ b/archive/website/cameraStatusReport.sh @@ -27,11 +27,17 @@ conda activate $HOME/miniconda3/envs/${WMPL_ENV} export PYTHONPATH=$PYLIB logger -s -t cameraStatusReport "starting" +echo TEMPORARY HACK REMOVE CODE PERTAINING TO OLD SERVER platform=$(grep "^NAME" /etc/os-release | awk -F'"' '{print $2}') -[ "$platform" == "Ubuntu" ] && authlog=/var/log/auth.log || authlog=/var/log/secure -sudo grep publickey $authlog* | grep -v $USER > $DATADIR/reports/lastlogins.txt +if [ "$platform" == "Ubuntu" ] ; then + authlog=/var/log/auth.log + batchuser=ubuntu +else + authlog=/var/log/secure + batchuser=ec2-user +fi +sudo grep publickey $authlog* | grep -v $batchuser > $DATADIR/reports/lastlogins.txt -echo TEMPORARY HACK REMOVE ME if [ "$platform" == "Ubuntu" ] then ssh ukmonhelper2 "sudo grep publickey /var/log/secure* | grep -v ec2-user" > $DATADIR/reports/lastlogins_old.txt From ac0923ba29a81efd56f7c70f6713706ab93d4b86 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 3 May 2026 14:03:38 +0100 Subject: [PATCH 246/287] removing unused files --- archive/server_setup/{ => unused}/newserver.sh | 0 archive/server_setup/{ => unused}/yum-installs.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename archive/server_setup/{ => unused}/newserver.sh (100%) rename archive/server_setup/{ => unused}/yum-installs.sh (100%) diff --git a/archive/server_setup/newserver.sh b/archive/server_setup/unused/newserver.sh similarity index 100% rename from archive/server_setup/newserver.sh rename to archive/server_setup/unused/newserver.sh diff --git a/archive/server_setup/yum-installs.sh b/archive/server_setup/unused/yum-installs.sh similarity index 100% rename from archive/server_setup/yum-installs.sh rename to archive/server_setup/unused/yum-installs.sh From 44e8481fe7ca47bfd938a43bfb8a58e180060d65 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 3 May 2026 14:17:49 +0100 Subject: [PATCH 247/287] some tidying up --- archive/server_setup/.bash_aliases | 4 +- .../migrateSftpAccts.sh | 0 archive/utils/README.md | 21 ++- archive/utils/checkCalcServerRunTime.sh | 66 -------- archive/utils/getDbPwd.sh | 11 -- .../{updateDb.sh => loadBrightnessCscMDB.sh} | 1 - archive/utils/monthlyCleardown.sh | 157 ------------------ utils/reimageBatchServer.ps1 | 9 + 8 files changed, 26 insertions(+), 243 deletions(-) rename archive/{utils => server_setup}/migrateSftpAccts.sh (100%) delete mode 100644 archive/utils/checkCalcServerRunTime.sh delete mode 100644 archive/utils/getDbPwd.sh rename archive/utils/{updateDb.sh => loadBrightnessCscMDB.sh} (91%) delete mode 100644 archive/utils/monthlyCleardown.sh create mode 100644 utils/reimageBatchServer.ps1 diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index 57c10ccb..e30fb1c8 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -17,8 +17,8 @@ alias stats='if [ "$DATADIR" == "" ] ; then echo select env first; else tail $DA alias matchstatus='if [ "$SRC" == "" ] ; then echo select env first; else grep "Running" $SRC/logs/matchJob.log && grep TRAJ $SRC/logs/matchJob.log | grep SOLVING && echo -n "Completed " && grep Observations: $SRC/logs/matchJob.log | wc -l && grep "nightlyJob" $SRC/logs/nightlyJob.log ; fi ' alias spacecalc='ls -1 | egrep -v "ukmon-shared" | while read i ; do \du -s $i ; done | sort -n' -alias startcalc='$SRC/utils/stopstart-calcengine.sh start' -alias stopcalc='$SRC/utils/stopstart-calcengine.sh stop' +alias startcalc='$SRC/utils/stopstartCalcengine.sh start' +alias stopcalc='$SRC/utils/stopstartCalcengine.sh stop' function dev { source ~/dev/config.ini diff --git a/archive/utils/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh similarity index 100% rename from archive/utils/migrateSftpAccts.sh rename to archive/server_setup/migrateSftpAccts.sh diff --git a/archive/utils/README.md b/archive/utils/README.md index e179ca9f..d1317bbe 100644 --- a/archive/utils/README.md +++ b/archive/utils/README.md @@ -3,27 +3,36 @@ This folder contains some utility functions useful for managing the environment. ## Used by the Batch +* cleanupDeletedTrajs.sh - checks for and removes any logically-deleted trajectories eg duplicates +* getCostmetrics.sh - extract costs data from AWS and push to the website +* loadMatchCsvMDB.sh - loads the match data into MariaDB +* loadSingleCsvMDB.sh - loads the single-station data into MariaDB +* loadBrightCsvMDB.sh - loads brightness data into MariaDB * clearSpace.sh - used by the batch to delete old logs etc -* clearCaches.sh - clears the in-memory cache. Not used much. +* clearCaches.sh - clears the in-memory cache before and after running memory-intensive jobs. + +## Other routine maintenance +* checkAndRollKeys.sh - rolls station AWS Key/Secret pairs periodically to avoid stale keys +* statsToMqtt.sh - posts server space/memory etc statistics to MQ +* userAudit.sh - performs a user audit and emails a report ## User tools to maintain data * deleteOrbit.sh - remove an orbit from the database and website * updateFireballFlag.sh - marks or unmarks a match as a Fireball * updateFireballImage.sh - updates the image shown for a fireball -* stopstart-calcengine.sh - stops/starts the calculation engine server +* stopstartCalcengine.sh - stops/starts the calculation engine server ## Used by the deployment tool * makeConfig.sh - used by the deployment process to make the config file ## Used in case one of the Lambdas fails -* rerunconsolidateFTPdetect.sh - reruns the lambda -* rerunFTPtoUkMONlambra.sh - reruns the lambda -* rerun GetExtraFiles.sh - reruns the lambda +* rerunFTPtoUkMONlambra.sh - reruns FTPtoUKMON lambda +* rerunGetExtraFiles.sh - reruns GetExtraFiles lambda * cftpd_templ.json - template used by the above ## The following are work in progress -* monthlyClearDown.sh - should be run monthly to clear down old data * createTestDataSet.sh - work in progress +* getGmnData.sh - pulls GMN datasets from the GMN server ## Copyright All code Copyright (C) 2018-2023 Mark McIntyre diff --git a/archive/utils/checkCalcServerRunTime.sh b/archive/utils/checkCalcServerRunTime.sh deleted file mode 100644 index 8380d69f..00000000 --- a/archive/utils/checkCalcServerRunTime.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# script to extract the start/stop times of the Calculation Engine server from the logs -# I have a spreadsheet that consumes this and works out the cost for various models of server (c8g.2xlarge, c8g.4xlarge etc) - -cd ~/prod/logs - -csl=/tmp/calcserver_timings.txt -csl2=/tmp/csl.txt -[ -f $csl ] && rm $csl -touch $csl - -ls -1tr nightly*.gz | while read i -do - mfz=matchJob${i:10:50} - gunzip $i - gunzip $mfz - nf=$(basename $i .gz) - mf=$(basename $mfz .gz) - egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl - gzip $nf - gzip $mf -done -ls -1tr nightlyJob* | grep -v gz | while read nf -do - mf=matchJob${nf:10:50} - egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl -done -[ -f $csl2 ] && rm $csl2 -touch $csl2 -cat $csl | while read i -do - parta=$(echo $i | awk -F ">" '{print $2}') - partb=$(echo $parta | awk -F " " '{printf("%s, %s, %s\n",$1,$2,$3)}') - echo $partb >> $csl2 -done -[ -f $csl ] && rm $csl -python << EOD -lis = open('$csl2','r').readlines() -data = [x.strip().split(',') for x in lis] -currdt = None -results = [] -res = [] -col = 0 -with open('/tmp/computeservertimes.csv','w') as outf: - for i in range(len(data)): - mth = data[i][0] - dy = data[i][1] - tim = data[i][2] - dtstr = f'{mth} {dy}' - if currdt is None: - currdt = dtstr - if currdt != dtstr: - outf.write(','.join(res) + '\n') - res = [] - col = 0 - if col == 0: - res.append(dtstr) - res.append(tim) - else: - res.append(tim) - col += 1 - currdt = dtstr - outf.write(','.join(res) + '\n') -EOD -[ -f $csl2 ] && rm $csl2 diff --git a/archive/utils/getDbPwd.sh b/archive/utils/getDbPwd.sh deleted file mode 100644 index cbbc5212..00000000 --- a/archive/utils/getDbPwd.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018- Mark McIntyre - -if [ "$1" == "root" ] -then - passwd=$(aws ssm get-parameters --names prod_rootdbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) -else - passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value) -fi -echo $passwd -unset passwd diff --git a/archive/utils/updateDb.sh b/archive/utils/loadBrightnessCscMDB.sh similarity index 91% rename from archive/utils/updateDb.sh rename to archive/utils/loadBrightnessCscMDB.sh index 98bf4a5a..858c6f62 100644 --- a/archive/utils/updateDb.sh +++ b/archive/utils/loadBrightnessCscMDB.sh @@ -12,7 +12,6 @@ else fi cd ~/prod/data/brightness -# leading space to prevent being inserted into environment passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value| sed 's/"//g') mysql -p$passwd -ubatch -h localhost << EOD use ukmon; diff --git a/archive/utils/monthlyCleardown.sh b/archive/utils/monthlyCleardown.sh deleted file mode 100644 index 89ca8f7b..00000000 --- a/archive/utils/monthlyCleardown.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018-2023 Mark McIntyre -# -# script to backup then clear down orbits from the local disk. - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -source $here/../config.ini >/dev/null 2>&1 -conda activate $HOME/miniconda3/envs/${WMPL_ENV} - -what=$1 -if [ "$what" == "" ] ; then - echo "usage: ./monthlyCleardown.sh single|raw|matched|reports|search|other" - exit -fi - -cyr=$(date +%Y) -cmt=$(date +%m) -lyr=$(date --date='last year' +%Y) -lym=$(date --date='last month' +%Y%m) - -mkdir -p $DATADIR/olddata - -# SINGLE STATION DATA -case "$what" in -single) - logger -s -t monthlyClearDown "single-station data" - cd $DATADIR/single/new/processed - # backup then remove last year's data if still present - if compgen -G "$DATADIR/single/new/processed/ukmon*_${lyr}????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-singlecsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-singlecsv.tar ukmon*_${lyr}????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-singlecsv.tar ukmon*_${lyr}????_*.csv - fi - rm -f $DATADIR/single/new/processed/ukmon*_${lyr}????_*.csv - else - echo "no ukmon-csv files from ${lyr} to archive" - fi - # backup last month's data - if compgen -G "$DATADIR/single/new/processed/ukmon*_${lym}??_*.csv" > /dev/null ; then - if [ $cmt -ne 1 ] ; then - if [ ! -f $DATADIR/olddata/${lym}-singlecsv.tar ] ; then - tar cvf $DATADIR/olddata/${lym}-singlecsv.tar ukmon*_${lym}??_*.csv - else - tar uvf $DATADIR/olddata/${lym}-singlecsv.tar ukmon*_${lym}??_*.csv - fi - fi - rm -f $DATADIR/single/new/processed/ukmon*_${lym}??_*.csv - else - echo "no ukmon-csv files from ${lym} to archive" - fi - ;; -raw) - logger -s -t monthlyClearDown "rawcsv data" - # raw CSV files from cameras - cd $DATADIR/single/rawcsvs - # backup then remove last year's data if still present - if compgen -G "$DATADIR/single/rawcsvs/*${lyr}????_??????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-rawcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lyr}????_??????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lyr}????_??????_*.csv - fi - find . -maxdepth 1 -name "*${lyr}????_??????_*.csv" -print0 | xargs -0 rm -f - else - echo "no rawcsv files from ${lyr} to archive" - fi - # backup last month's data - if compgen -G "$DATADIR/single/rawcsvs/*${lym}??_??????_*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-rawcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lym}??_??????_*.csv - else - tar uvf $DATADIR/olddata/${lyr}-rawcsv.tar *${lym}??_??????_*.csv - fi - rm -f $DATADIR/single/rawcsvs/*${lym}??_??????_*.csv - else - echo "no rawcsv files from ${lym} to archive" - fi - ;; -matched) - logger -s -t monthlyClearDown "match data" - # MATCHES - # last year's fullcsv files - cd $DATADIR/orbits/$lyr/fullcsv/processed - if compgen -G "$DATADIR/orbits/${lyr}/fullcsv/processed/${lyr}*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-matchfullcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-matchfullcsv.tar ${lyr}*.csv - else - tar uvf $DATADIR/olddata/${lyr}-matchfullcsv.tar ${lyr}*.csv - fi - rm -f $DATADIR/orbits/${lyr}/fullcsv/processed/${lyr}*.csv - else - echo "no fullcsv files from ${lyr} to archive" - fi - - # and now last months fullcsv data - cd $DATADIR/orbits/$cyr/fullcsv/processed - if compgen -G "$DATADIR/orbits/${cyr}/fullcsv/processed/${lym}??-*.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lym}-matchfullcsv.tar ] ; then - tar cvf $DATADIR/olddata/${lym}-matchfullcsv.tar ${lym}??-*.csv - else - tar uvf $DATADIR/olddata/${lym}-matchfullcsv.tar ${lym}??-*.csv - fi - rm -f $DATADIR/orbits/${cyr}/fullcsv/processed/${lym}??-*.csv - else - echo "no fullcsv files from ${lym} to archive" - fi - ;; -reports) - logger -s -t monthlyClearDown "reports and so forth" - # SHOWER, MONTHLY AND ANNUAL REPORTS FROM PREVIOUS YEAR - cd $DATADIR/reports/${lyr} - nf=$(ls -1 | wc -l) - if [ $nf -ne 0 ] ; then - if [ ! -f $DATADIR/olddata/${lyr}reports.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-reports.tar * - else - tar uvf $DATADIR/olddata/${lyr}-reports.tar * - fi - rm -Rf * - else - echo "no report files from ${lyr} to archive" - fi - ;; -search) - logger -s -t monthlyClearDown "old search indexes" - # SEARCH INDEXES FOR PRIOR YEAR - cd $DATADIR/searchidx - if compgen -G "$DATADIR/searchidx/${lyr}-allevents.csv" > /dev/null ; then - if [ ! -f $DATADIR/olddata/${lyr}-searchidx.tar ] ; then - tar cvf $DATADIR/olddata/${lyr}-searchidx.tar ${lyr}-allevents.csv - else - tar uvf $DATADIR/olddata/${lyr}-searchidx.tar ${lyr}-allevents.csv - fi - rm -Rf $DATADIR/searchidx/${lyr}-allevents.csv - else - echo "no search indexes from ${lyr} to archive" - fi - ;; -ftpdata) - logger -s -t monthlyClearDown "old ftpdetect and platepars_all files" - # cleardown the raw input data. This is already backed up elsewhere - inputym=$(date --date='3 months ago' +%Y%m) - #python $PYLIB/maintenance/dataMaintenance.py $inputym - ;; -*) - echo missing argument - ;; -esac -# now push it all to the offline backup -logger -s -t monthlyClearDown "syncing to backup bucket" -aws s3 sync $DATADIR/olddata/ s3://ukmda-admin/backups/ --exclude "*" --include "${lyr}*.tar" --quiet -rm -f $DATADIR/olddata/${lyr}*.tar - -aws s3 sync $DATADIR/olddata/ s3://ukmda-admin/backups/ --exclude "*" --include "${lym}*.tar" --quiet -# dont do this in case data arrives late -#rm -f $DATADIR/olddata/${lym}*.tar diff --git a/utils/reimageBatchServer.ps1 b/utils/reimageBatchServer.ps1 new file mode 100644 index 00000000..8f513e4d --- /dev/null +++ b/utils/reimageBatchServer.ps1 @@ -0,0 +1,9 @@ +# script to re-image the UKMON calc server +# copyright Mark McIntyre, 2026- + +$dtstr=(get-date -uformat "%Y%m%d") + +$instdetails=(aws ec2 describe-instances --filters Name="tag:Name",Values="batchserver" --profile ukmonshared --region eu-west-2) +$instanceid= (Write-Output $instdetails| convertfrom-json)[0].reservations.instances.instanceid + +aws ec2 create-image --instance-id $instanceid --name "batchserver_${dtstr}" --description "Latest Batchserver image" --profile ukmonshared --region eu-west-2 \ No newline at end of file From 968b396bee96b6eec075ee82d33f6b7b43709278 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 3 May 2026 14:22:03 +0100 Subject: [PATCH 248/287] motre reorg --- archive/cronjobs/captureBright.sh | 10 +--------- ...loadBrightnessCscMDB.sh => loadBrightnessCsvMDB.sh} | 3 ++- 2 files changed, 3 insertions(+), 10 deletions(-) rename archive/utils/{loadBrightnessCscMDB.sh => loadBrightnessCsvMDB.sh} (95%) diff --git a/archive/cronjobs/captureBright.sh b/archive/cronjobs/captureBright.sh index ff38a264..b06ed51d 100644 --- a/archive/cronjobs/captureBright.sh +++ b/archive/cronjobs/captureBright.sh @@ -8,15 +8,7 @@ cd $DATADIR/brightness rundt=$(date -d "yesterday" +%Y%m%d) python -m utils.compareBrightnessData $rundt -# leading space to prevent being inserted into environment -pwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value | sed 's/"//g') -mysql -p$pwd -ubatch -h localhost << EOD -use ukmon; -select count(*) from ukmon.brightness; -load data local infile '${DATADIR}/brightness/CaptureNight_${rundt}.csv' -into table brightness fields terminated by ','; -select count(*) from ukmon.brightness; -EOD +$SRC/utils/loadBrightnessCsvMDB.sh find ${DATADIR}/brightness -name "CaptureNight*" -mtime +30 -exec rm -f {} \; find ${DATADIR}/brightness -name "matcheddata*" -mtime +30 -exec rm -f {} \; \ No newline at end of file diff --git a/archive/utils/loadBrightnessCscMDB.sh b/archive/utils/loadBrightnessCsvMDB.sh similarity index 95% rename from archive/utils/loadBrightnessCscMDB.sh rename to archive/utils/loadBrightnessCsvMDB.sh index 858c6f62..b38b95c3 100644 --- a/archive/utils/loadBrightnessCscMDB.sh +++ b/archive/utils/loadBrightnessCsvMDB.sh @@ -11,7 +11,8 @@ else rundt=$1 fi -cd ~/prod/data/brightness + +cd $DATADIR/brightness passwd=$(aws ssm get-parameters --names prod_dbpw --with-decryption --region eu-west-1 | jq .Parameters[0].Value| sed 's/"//g') mysql -p$passwd -ubatch -h localhost << EOD use ukmon; From b410ced3e53b61823de6af280ec66b38f0670fca Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 3 May 2026 14:23:42 +0100 Subject: [PATCH 249/287] remove disused template rename file to follow convention --- archive/utils/liveevent.template | 15 --------------- ...start-calcengine.sh => stopstartCalcengine.sh} | 0 2 files changed, 15 deletions(-) delete mode 100644 archive/utils/liveevent.template rename archive/utils/{stopstart-calcengine.sh => stopstartCalcengine.sh} (100%) diff --git a/archive/utils/liveevent.template b/archive/utils/liveevent.template deleted file mode 100644 index 02af78a6..00000000 --- a/archive/utils/liveevent.template +++ /dev/null @@ -1,15 +0,0 @@ -{ - "Records": [ - { - "s3": { - "bucket": { - "name": "ukmda-live", - "arn": "arn:aws:s3:::ukmda-live" - }, - "object": { - "key": "KEYHERE" - } - } - } - ] -} diff --git a/archive/utils/stopstart-calcengine.sh b/archive/utils/stopstartCalcengine.sh similarity index 100% rename from archive/utils/stopstart-calcengine.sh rename to archive/utils/stopstartCalcengine.sh From 94dd4162c940a25a0f8d056dbcfe6a4661ffae95 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 3 May 2026 15:47:44 +0100 Subject: [PATCH 250/287] simplifying logging and removing unused aws profile --- archive/analysis/findAllMatches.sh | 26 ++++------ archive/analysis/runDistrib.sh | 53 +++++++-------------- archive/analysis/updatePlotsAndDetStatus.sh | 35 +++++--------- archive/cronjobs/gatherMonthlyVideos.sh | 4 +- archive/server_setup/.bash_aliases | 4 +- archive/utils/cleanupDeletedTrajs.sh | 9 ++-- archive/utils/makeConfig.sh | 2 +- archive/utils/rerunFTPtoUKMONlambda.sh | 2 +- archive/utils/stopstartCalcengine.sh | 2 +- archive/website/costReport.sh | 10 ++-- 10 files changed, 56 insertions(+), 91 deletions(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index b52fc2ae..05c9c9a8 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -23,12 +23,7 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "start findAllMatches" findAllMatches +logger -s -t findAllMatches "starting" [ -f $DATADIR/rundate.txt ] && rundate=$(cat $DATADIR/rundate.txt) || rundate=$(date +%Y%m%d) @@ -52,15 +47,15 @@ mkdir -p $SRC/logs/distrib > /dev/null 2>&1 startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') enddt=$(date --date="-$MATCHEND days" '+%Y%m%d-080000') -log2cw $NJLOGGRP $NJLOGSTREAM "solving for ${startdt} to ${enddt}" findAllMatches -log2cw $NJLOGGRP $NJLOGSTREAM "start runDistrib" findAllMatches -$SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND +logger -s -t findAllMatches "solving for ${startdt} to ${enddt}" +logger -s -t findAllMatches "start runDistrib" -log2cw $NJLOGGRP $NJLOGSTREAM "clean duplicate/deleted trajs" findAllMatches +$SRC/analysis/runDistrib.sh $MATCHSTART $MATCHEND $SRC/utils/cleanupDeletedTrajs.sh -log2cw $NJLOGGRP $NJLOGSTREAM "start checkForFailures" findAllMatches +logger -s -t findAllMatches "Solving Run Done" + success=$(grep "Total run time:" $SRC/logs/matchJob.log) if [ "$success" == "" ] @@ -68,13 +63,13 @@ then python -c "from utils.sendAnEmail import sendAnEmail ; sendAnEmail('markmcintyre99@googlemail.com','problem with matching','Error in UKMON matching', mailfrom='ukmonhelper@ukmeteors.co.uk')" echo problems with solver fi -log2cw $NJLOGGRP $NJLOGSTREAM "Solving Run Done" findAllMatches -log2cw $NJLOGGRP $NJLOGSTREAM "start rerunFailedLambdas" findAllMatches python -m maintenance.rerunFailedLambdas cd $here -log2cw $NJLOGGRP $NJLOGSTREAM "start reportOfLatestMatches" findAllMatches + +logger -s -t findAllMatches "start reportOfLatestMatches" + matchlog=${SRC}/logs/matchJob.log python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate python -m metrics.getMatchStats $matchlog @@ -84,8 +79,7 @@ if [ "$RUNTIME_ENV" == "PROD" ] ; then aws s3 sync $DATADIR/dailyreports/ $UKMONSHAREDBUCKET/matches/RMSCorrelate/dailyreports/ --quiet fi -log2cw $NJLOGGRP $NJLOGSTREAM "start purgeLogs" findAllMatches find $SRC/logs -name "matches*" -mtime +7 -exec gzip {} \; find $SRC/logs -name "matches*" -mtime +30 -exec rm -f {} \; -log2cw $NJLOGGRP $NJLOGSTREAM "finished findAllMatches" findAllMatches +logger -s -t findAllMatches "finished" diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 536527c4..883f373e 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -20,59 +20,46 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "starting runDistrib" runDistrib - -# set the profile to the UKMDA account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared +logger -s -t runDistrib "starting runDistrib" if [ $# -gt 0 ] ; then if [ "$1" != "" ] ; then - log2cw $NJLOGGRP $NJLOGSTREAM "selecting range" runDistrib MATCHSTART=$1 fi if [ "$2" != "" ] ; then MATCHEND=$2 else - log2cw $NJLOGGRP $NJLOGSTREAM "matchend was not supplied, using 2 days" runDistrib MATCHEND=$(( $MATCHSTART - 2 )) fi fi begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" runDistrib +logger -s -t runDistrib "running phase 1 for dates ${begdate} to ${rundate}" +logger -s -t runDistrib "start correlation server" aws ec2 start-instances --instance-ids $SERVERINSTANCEID stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) while [ "$stat" -ne 16 ]; do sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" runDistrib stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -log2cw $NJLOGGRP $NJLOGSTREAM "running phase 1 for dates ${begdate} to ${rundate}" runDistrib - conda activate $HOME/miniconda3/envs/${WMPL_ENV} -log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" runDistrib +logger -s -t runDistrib "creating the run script" execdist=execdistrib.sh execMatchingsh=/tmp/$execdist python -m traj.createDistribMatchingSh $MATCHSTART $MATCHEND $execMatchingsh $TESTMODE chmod +x $execMatchingsh -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $CALCSERVERIP and run it" runDistrib +logger -s -t runDistrib "deploy the script to the server $CALCSERVERIP and run it" scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" runDistrib scp -i $SERVERSSHKEY $execMatchingsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execdist done # push the python code and ECS templates required @@ -85,7 +72,7 @@ rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/ShowerAssociation.py $SERVERUSE rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/utils/convertSolLon.py $SERVERUSERID@$CALCSERVERIP:src/ukmon_pylib/utils/ # now run the script -log2cw $NJLOGGRP $NJLOGSTREAM "start distributed processing" runDistrib +logger -s -t runDistrib "start distributed processing" ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execdist" rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/candidates/processed/*.tgz $DATADIR/distrib/candidates @@ -93,28 +80,27 @@ rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/ma rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/logs/*${rundate}*.log $SRC/logs/distrib/ ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find ukmon-shared/matches/RMSCorrelate/logs -name '*.log' -mtime +30 -exec rm -f {} \;" -log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" runDistrib +logger -s -t runDistrib "job run, stop the server again" aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -log2cw $NJLOGGRP $NJLOGSTREAM "monitoring and waiting for completion" runDistrib +logger -s -t runDistrib "monitoring and waiting for completion" python -c "from traj.distributeCandidates import monitorProgress as mp; mp('${rundate}', '${TESTMODE}'); " mkdir -p $DATADIR/distrib cd $DATADIR/distrib -log2cw $NJLOGGRP $NJLOGSTREAM "restarting server to consolidate results" runDistrib +logger -s -t runDistrib "restarting server to consolidate results" + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi -log2cw $NJLOGGRP $NJLOGSTREAM "waiting for the server to be ready" runDistrib while [ "$stat" -ne 16 ]; do sleep 30 if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi - log2cw $NJLOGGRP $NJLOGSTREAM "checking - status is ${stat}" runDistrib stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done @@ -123,19 +109,21 @@ execConsolsh=/tmp/$execcons python -c "from traj.createDistribMatchingSh import createExecConsolSh;createExecConsolSh($MATCHSTART, $MATCHEND, '$execConsolsh', '$TESTMODE')" chmod +x $execConsolsh -log2cw $NJLOGGRP $NJLOGSTREAM "running consolidation" runDistrib +logger -s -t runDistrib "running consolidation" + scp -i $SERVERSSHKEY $execConsolsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execcons ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execcons" -log2cw $NJLOGGRP $NJLOGSTREAM "finished consolidation, copying databases" runDistrib -rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib +logger -s -t runDistrib "finished consolidation, copying databases" -# remote temporary files +rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" -log2cw $NJLOGGRP $NJLOGSTREAM "stopping calcserver again" runDistrib +loggers -s -t runDistrib "stopping calcserver again" aws ec2 stop-instances --instance-ids $SERVERINSTANCEID +loggers -s -t runDistrib "copying data to batch server and tidying up" + # grab a copy of the indvidual container dbs so we can get a list of new solutions rm -Rf $DATADIR/latest/contdbs/ mkdir -p $DATADIR/latest/contdbs/ @@ -147,8 +135,6 @@ aws s3 mv $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/${rundate}.pickle $DATAD aws s3 sync $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ $SRC/logs/distrib/ --exclude "*" --include "correl*.log" --exclude "logs/" --quiet aws s3 rm $UKMONSHAREDBUCKET/matches/distrib${TESTSUFF}/ --exclude "*" --include "correl*.log" --exclude "logs/" --recursive --quiet -log2cw $NJLOGGRP $NJLOGSTREAM "compressing the processed data" runDistrib - mkdir -p $DATADIR/trajdb tar czvf $DATADIR/trajdb/databases_${rundate}.tgz $DATADIR/distrib/*.db mkdir -p $DATADIR/distrib/containers @@ -161,7 +147,4 @@ find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \; find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \; rm -f $DATADIR/distrib/${rundate}.pickle - -# and then clear the profile again -unset AWS_PROFILE -log2cw $NJLOGGRP $NJLOGSTREAM "finished runDistrib" runDistrib +loggers -s -t "finished runDistrib" diff --git a/archive/analysis/updatePlotsAndDetStatus.sh b/archive/analysis/updatePlotsAndDetStatus.sh index 36ab4908..1ef92a48 100644 --- a/archive/analysis/updatePlotsAndDetStatus.sh +++ b/archive/analysis/updatePlotsAndDetStatus.sh @@ -17,58 +17,47 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" # load the configuration source $here/../config.ini >/dev/null 2>&1 -# logstream name inherited from parent environment but set it if not -if [ "$NJLOGSTREAM" == "" ]; then - NJLOGSTREAM=$(date +%Y%m%d-%H%M%S) - aws logs create-log-stream --log-group-name $NJLOGGRP --log-stream-name $NJLOGSTREAM --profile ukmonshared -fi -log2cw $NJLOGGRP $NJLOGSTREAM "starting updatePlotsAndDetStatus" updatePlotsAndDetStatus - -# set the profile to the EE account so we can run the server and monitor progress -export AWS_PROFILE=ukmonshared +logger -s -t updatePlotsAndDetStatus "starting updatePlotsAndDetStatus" if [ $# -gt 0 ] ; then if [ "$1" != "" ] ; then - log2cw $NJLOGGRP $NJLOGSTREAM "selecting range" updatePlotsAndDetStatus MATCHSTART=$1 fi if [ "$2" != "" ] ; then MATCHEND=$2 else - log2cw $NJLOGGRP $NJLOGSTREAM "matchend was not supplied, using 2 days" updatePlotsAndDetStatus MATCHEND=$(( $MATCHSTART - 2 )) fi fi begdate=$(date --date="-$MATCHSTART days" '+%Y%m%d') rundate=$(date --date="-$MATCHEND days" '+%Y%m%d') -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "updating plots etc for dates ${begdate} to ${rundate}" +logger -s -t updatePlotsAndDetStatus "start correlation server" + stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) if [ $stat -eq 80 ]; then aws ec2 start-instances --instance-ids $SERVERINSTANCEID fi -log2cw $NJLOGGRP $NJLOGSTREAM "start correlation server" updatePlotsAndDetStatus while [ "$stat" -ne 16 ]; do sleep 5 - log2cw $NJLOGGRP $NJLOGSTREAM "checking server status" updatePlotsAndDetStatus stat=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State.Code --output text) done -log2cw $NJLOGGRP $NJLOGSTREAM "updating plots etc for dates ${begdate} to ${rundate}" updatePlotsAndDetStatus + conda activate $HOME/miniconda3/envs/${WMPL_ENV} -log2cw $NJLOGGRP $NJLOGSTREAM "creating the run script" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "creating the run script" execrerun=execreplot.sh execrerunsh=/tmp/$execrerun python -c "from traj.createDistribMatchingSh import createExecReplotSh;createExecReplotSh($MATCHSTART, $MATCHEND, '$execrerunsh', '$TESTMODE')" chmod +x $execrerunsh -log2cw $NJLOGGRP $NJLOGSTREAM "deploy the script to the server $CALCSERVERIP and run it" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "deploy the script to the server $CALCSERVERIP and run it" scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun while [ $? -ne 0 ] ; do # in case the server isn't responding to ssh sessions yet sleep 10 - log2cw $NJLOGGRP $NJLOGSTREAM "server not responding yet, retrying" updatePlotsAndDetStatus scp -i $SERVERSSHKEY $execrerunsh $SERVERUSERID@$CALCSERVERIP:data/distrib/$execrerun done # push the python and templates required @@ -77,18 +66,18 @@ rsync -avz -e "ssh -i $SERVERSSHKEY" $PYLIB/traj/pickleAnalyser.py $SERVERUSERI # now run the script ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "data/distrib/$execrerun" -log2cw $NJLOGGRP $NJLOGSTREAM "job run, stop the server again" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "job run, stop the server again" + aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -log2cw $NJLOGGRP $NJLOGSTREAM "get a list of uncalibrated data" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "get a list of uncalibrated data" + aws s3 sync $UKMONSHAREDBUCKET/matches/consumed/ $DATADIR/single/used/ --exclude "*" --include "*.txt" --quiet rundate=$(cat $DATADIR/rundate.txt) python -c "from utils.getUncalImages import getUncalibratedImageList;getUncalibratedImageList('$rundate');" -# and then clear the profile again -unset AWS_PROFILE # refresh the website index pages just in case any new data dailyrep=$(ls -1tr $DATADIR/dailyreports/20* | tail -1) $SRC/website/updateIndexPages.sh $dailyrep -log2cw $NJLOGGRP $NJLOGSTREAM "finished updatePlotsAndDetStatus" updatePlotsAndDetStatus +logger -s -t updatePlotsAndDetStatus "finished updatePlotsAndDetStatus" diff --git a/archive/cronjobs/gatherMonthlyVideos.sh b/archive/cronjobs/gatherMonthlyVideos.sh index 52f67048..e8c9c146 100644 --- a/archive/cronjobs/gatherMonthlyVideos.sh +++ b/archive/cronjobs/gatherMonthlyVideos.sh @@ -70,7 +70,7 @@ echo "mkdir \$1 ; cd \$1" >> $outdir/getVideos.ps1 echo "foreach(\$line in Get-Content ../best_\$1.txt) { wget https://archive.ukmeteors.co.uk/\$line }" >> $outdir/getVideos.ps1 echo "cd .." >> $outdir/getVideos.ps1 -aws s3 sync $outdir $UKMONSHAREDBUCKET/videos/ --profile ukmonshared --quiet --exclude "*" --include "getVideos*" --include "README*" +aws s3 sync $outdir $UKMONSHAREDBUCKET/videos/ --quiet --exclude "*" --include "getVideos*" --include "README*" -aws s3 sync $outdir $WEBSITEBUCKET/data/bestvideos/ --profile ukmonshared --quiet --include "*.txt" +aws s3 sync $outdir $WEBSITEBUCKET/data/bestvideos/ --quiet --include "*.txt" diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index e30fb1c8..5ade8d17 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -36,8 +36,8 @@ function prd { } function calcserver { - sts=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State --output text --profile ukmonshared) - ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text --profile ukmonshared) + sts=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].State --output text) + ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) isrunning=$(echo $sts | cut -d " " -f 1) if [ $isrunning -ne 16 ] ; then $HOME/prod/utils/stopstart-calcengine.sh start diff --git a/archive/utils/cleanupDeletedTrajs.sh b/archive/utils/cleanupDeletedTrajs.sh index 6ffe0a6d..9269b647 100644 --- a/archive/utils/cleanupDeletedTrajs.sh +++ b/archive/utils/cleanupDeletedTrajs.sh @@ -14,7 +14,7 @@ cd ${DATADIR}/distrib startdt=$(date --date="-$MATCHSTART days" '+%Y%m%d-080000') jdt_min=$(python -c "from wmpl.Utils.TrajConversions import datetime2JD;import datetime;print(datetime2JD(datetime.datetime.strptime('$startdt', '%Y%m%d-%H%M%S')))") -echo "checking main storage and website" +logger -s -t cleanupDeletedTrajs "starting: checking main storage and website" sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | while read traj ; do # check if there are two trajectories with the same folder - we don't want to delete the active one @@ -42,7 +42,7 @@ sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectorie fi done -echo "checking raw fullcsv data files" +logger -s -t cleanupDeletedTrajs "checking raw fullcsv data files" yr=${startdt:0:4} csvloc=matches/${yr}/fullcsv newloc=matches/duplicates/csvs @@ -57,7 +57,7 @@ if [ $newyr != $yr ] ; then fi -echo "checking consolidated matches" +logger -s -t cleanupDeletedTrajs "checking consolidated matches" lasttraj=$(sqlite3 $DATADIR/distrib/trajectories.db "select traj_file_path from trajectories where status=0 and jdt_ref > ${jdt_min} order by jdt_ref;" | tail -1) trajdir=$(dirname $lasttraj) yr=${trajdir:13:4} @@ -77,4 +77,5 @@ if [ $? == 0 ] ; then fi # no need to check the SQL database in mariadb, as it is populated from the CSV file that -# we cleaned up above \ No newline at end of file +# we cleaned up above +logger -s -t cleanupDeletedTrajs "finished cleanupDeletedTrajs" \ No newline at end of file diff --git a/archive/utils/makeConfig.sh b/archive/utils/makeConfig.sh index 50ffc086..d68f3f9c 100644 --- a/archive/utils/makeConfig.sh +++ b/archive/utils/makeConfig.sh @@ -89,6 +89,6 @@ echo "export TESTMODE TESTSUFF" >> ${CFGFILE} echo "source ~/.condaon" >> ${CFGFILE} # function to log to cloudwatch echo 'function log2cw() { ' >> ${CFGFILE} -echo 'aws logs put-log-events --log-group-name $1 --log-stream-name $2 --log-events "[{\"timestamp\": $(date +%s%3N), \"message\": \"$3\"}]" --profile ukmonshared > /dev/null ' >> ${CFGFILE} +echo 'aws logs put-log-events --log-group-name $1 --log-stream-name $2 --log-events "[{\"timestamp\": $(date +%s%3N), \"message\": \"$3\"}]" > /dev/null ' >> ${CFGFILE} echo 'logger -s -t $4 RUNTIME $SECONDS $3' >> ${CFGFILE} echo '}' >> ${CFGFILE} diff --git a/archive/utils/rerunFTPtoUKMONlambda.sh b/archive/utils/rerunFTPtoUKMONlambda.sh index 9c768d02..38d6b1f6 100644 --- a/archive/utils/rerunFTPtoUKMONlambda.sh +++ b/archive/utils/rerunFTPtoUKMONlambda.sh @@ -22,6 +22,6 @@ cat /tmp/ftp2ukmon.txt | while read i do fullname=${i} cat $SRC/utils/cftpd_templ.json | sed "s|KEYGOESHERE|${fullname}|g" > ./tmp.json - aws lambda invoke --profile ukmonshared --function-name ftpToUkmon --log-type Tail --payload file://./tmp.json --region eu-west-2 --cli-binary-format raw-in-base64-out res.log + aws lambda invoke --function-name ftpToUkmon --log-type Tail --payload file://./tmp.json --region eu-west-2 --cli-binary-format raw-in-base64-out res.log done rm ./tmp.json \ No newline at end of file diff --git a/archive/utils/stopstartCalcengine.sh b/archive/utils/stopstartCalcengine.sh index 85d31f29..366ed507 100644 --- a/archive/utils/stopstartCalcengine.sh +++ b/archive/utils/stopstartCalcengine.sh @@ -5,4 +5,4 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $here/../config.ini >/dev/null 2>&1 -aws ec2 $1-instances --instance-ids $SERVERINSTANCEID --region eu-west-2 --profile ukmonshared +aws ec2 $1-instances --instance-ids $SERVERINSTANCEID --region eu-west-2 diff --git a/archive/website/costReport.sh b/archive/website/costReport.sh index cd60ec06..1ad8efa0 100644 --- a/archive/website/costReport.sh +++ b/archive/website/costReport.sh @@ -27,10 +27,8 @@ if [ $# -gt 1 ] ; then numdays = $1 ; fi mkdir -p $DATADIR/costs > /dev/null -export AWS_PROFILE=ukmonshared logger -s -t costReport "getting data for $numdays for $AWS_PROFILE" python -m metrics.costMetrics $DATADIR/costs eu-west-2 $numdays -unset AWS_PROFILE tod=$(date +%d) if [ "$tod" == "03" ] ; then @@ -74,8 +72,8 @@ done echo "var outer_div = document.getElementById(\"docindex\"); outer_div.appendChild(table); })" >> $frjs logger -s -t costReport "publishing data" -aws s3 sync $DATADIR/costs/ $WEBSITEBUCKET/docs/financial/raw/ --exclude "*" --include "costs-18*.csv" --quiet --profile ukmonshared -aws s3 cp $costfile $WEBSITEBUCKET/reports/ --quiet --profile ukmonshared -aws s3 cp $DATADIR/$imgfile3 $WEBSITEBUCKET/reports/ --quiet --profile ukmonshared -aws s3 cp $frjs $WEBSITEBUCKET/docs/ --quiet --profile ukmonshared +aws s3 sync $DATADIR/costs/ $WEBSITEBUCKET/docs/financial/raw/ --exclude "*" --include "costs-18*.csv" --quiet +aws s3 cp $costfile $WEBSITEBUCKET/reports/ --quiet +aws s3 cp $DATADIR/$imgfile3 $WEBSITEBUCKET/reports/ --quiet +aws s3 cp $frjs $WEBSITEBUCKET/docs/ --quiet logger -s -t costReport "done" From df4aa81c6174e076601895d552b5848e89c4074e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 3 May 2026 16:33:54 +0100 Subject: [PATCH 251/287] removing profile from python --- archive/ukmon_pylib/maintenance/cleanDdbTables.py | 8 ++------ archive/ukmon_pylib/maintenance/dataMaintenance.py | 13 +++---------- .../ukmon_pylib/maintenance/recreateOrbitPages.py | 6 ++---- .../ukmon_pylib/maintenance/rerunFailedLambdas.py | 13 +++++-------- archive/ukmon_pylib/reports/CameraDetails.py | 4 +--- archive/ukmon_pylib/utils/compareBrightnessData.py | 4 +--- 6 files changed, 14 insertions(+), 34 deletions(-) diff --git a/archive/ukmon_pylib/maintenance/cleanDdbTables.py b/archive/ukmon_pylib/maintenance/cleanDdbTables.py index 464d2be7..26e17f80 100644 --- a/archive/ukmon_pylib/maintenance/cleanDdbTables.py +++ b/archive/ukmon_pylib/maintenance/cleanDdbTables.py @@ -9,9 +9,7 @@ def deleteRows(): - archprof = os.getenv('ADM_PROFILE', default='ukmda_admin') - conn = boto3.Session(profile_name=archprof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') tbl = 'uploadtimes' idx = 'uploaddate-stationid-index' table = ddb.Table(tbl) @@ -28,9 +26,7 @@ def deleteRows(): def updateMissingExpiryDate(): - archprof = os.getenv('ADM_PROFILE', default='ukmda_admin') - conn = boto3.Session(profile_name=archprof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') tbl = 'uploadtimes' idx = 'uploaddate-stationid-index' table = ddb.Table(tbl) diff --git a/archive/ukmon_pylib/maintenance/dataMaintenance.py b/archive/ukmon_pylib/maintenance/dataMaintenance.py index 7373c314..32418b25 100644 --- a/archive/ukmon_pylib/maintenance/dataMaintenance.py +++ b/archive/ukmon_pylib/maintenance/dataMaintenance.py @@ -55,18 +55,11 @@ def deleteS3FilesByMonth(flist, archbucket): def deleteFromCalcServerByMonth(outfname): - hname = os.getenv('HOSTNAME', default='none') env = os.getenv('RUNTIME_ENV', default='DEV').lower() - if hname != 'ukmonhelper2': - sess = boto3.Session(profile_name='default') - ssmc = sess.client('ssm', region_name='eu-west-2') - ec2 = boto3.client('ec2', region_name='eu-west-2') - else: - ssmc = boto3.client('ssm', region_name='eu-west-2') - prof = os.getenv('UKMPROFILE',default='ukmonshared') - sess = boto3.Session(profile_name=prof) - ec2 = sess.client('ec2', region_name='eu-west-2') + ssmc = boto3.client('ssm', region_name='eu-west-2') + ec2 = boto3.client('ec2', region_name='eu-west-2') + resp = ssmc.get_parameter(Name=f'{env}_calcinstance') instId = resp['Parameter']['Value'] print('clearing down calcserver') diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py index 4f5f050c..c64c746c 100644 --- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py +++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py @@ -71,8 +71,7 @@ def createExtraJpgtxt(outdir, traj, availableimages): def createExtraJpgHtml(outdir, parentfldr, yr, ym): - mdasess = boto3.Session(profile_name='ukmda_admin') - s3mda = mdasess.client('s3') + s3mda = boto3.client('s3') webfldr = f'img/single/{yr}/{ym}' with open(os.path.join(outdir, 'extrajpgs.html'), 'w') as outf: jpgfldr = os.path.join(parentfldr, 'jpgs') @@ -194,8 +193,7 @@ def recreateOrbitFiles(outdir, pickname, doupload=False): if doupload: files = os.listdir(outdir) - mdasess = boto3.Session(profile_name='ukmda_admin') - s3mda = mdasess.client('s3') + s3mda = boto3.client('s3') if int(yr) > 2020: webfldr = f'reports/{yr}/orbits/{ym}/{ymd}' else: diff --git a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py index 24107d46..3ef031d5 100644 --- a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py +++ b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py @@ -30,8 +30,7 @@ def findOtherBadEvents(): return badevents -def findFailedEvents(prof='ukmonshared'): - #session = boto3.Session(profile_name=prof) +def findFailedEvents(): logcli = boto3.client('logs', region_name='eu-west-2') datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') @@ -98,9 +97,8 @@ def checkFails(fails): return realfails -def rerunFails(fails, prof='ukmonshared'): - session=boto3.Session(profile_name=prof) - lambd = session.client('lambda', region_name='eu-west-2') +def rerunFails(fails): + lambd = boto3.client('lambda', region_name='eu-west-2') datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) lastf = os.path.join(datadir,'orbits', 'lastorbitcheck.txt') @@ -120,10 +118,9 @@ def rerunFails(fails, prof='ukmonshared'): return -def resubmitPicklesForDay(dtstr, prof='ukmonshared'): +def resubmitPicklesForDay(dtstr): s3 = boto3.client('s3') - session = boto3.Session(profile_name=prof) - lambd = session.client('lambda', region_name='eu-west-2') + lambd = boto3.client('lambda', region_name='eu-west-2') pref = f'matches/RMSCorrelate/trajectories/{dtstr[:4]}/{dtstr[:6]}/{dtstr}/' buck = os.getenv('UKMONSHAREDBUCKET', default='s3://ukmda-shared')[5:] res = s3.list_objects_v2(Bucket=buck, Prefix=pref) diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index 44e96241..3557c103 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -41,9 +41,7 @@ def updateCamLocDirFovDB(datadir=None): def loadLocationDetails(table='camdetails', ddb=None, loadall=False): if not ddb: - prof = os.getenv('UKMPROFILE',default='ukmonshared') - conn = boto3.Session(profile_name=prof) - ddb = conn.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table(table) res = table.scan() # strictly, should check for LastEvaluatedKey here, in case there was more than 1MB of data, diff --git a/archive/ukmon_pylib/utils/compareBrightnessData.py b/archive/ukmon_pylib/utils/compareBrightnessData.py index 1ace683c..663fb9af 100644 --- a/archive/ukmon_pylib/utils/compareBrightnessData.py +++ b/archive/ukmon_pylib/utils/compareBrightnessData.py @@ -10,9 +10,7 @@ def getBrightnessData(yyyymmdd, outdir=None): - prof = os.getenv('UKMPROFILE',default='ukmonshared') - sess = boto3.Session(profile_name=prof) - ddb = sess.resource('dynamodb', region_name='eu-west-2') + ddb = boto3.resource('dynamodb', region_name='eu-west-2') table = ddb.Table('LiveBrightness') resp = table.query(KeyConditionExpression=Key('CaptureNight').eq(yyyymmdd)) df = pd.DataFrame(ddbjson.loads(resp['Items'])) From 3e9d8f66ea63fbf12ed01cd12e7cb07b22520292 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 12:13:17 +0000 Subject: [PATCH 252/287] adding executable bit --- archive/server_setup/migrateSftpAccts.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 archive/server_setup/migrateSftpAccts.sh diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh old mode 100644 new mode 100755 From 32353bcfc72fbc2437b2ae03f755a14b21a2a775 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 14:01:43 +0100 Subject: [PATCH 253/287] typo --- archive/analysis/runDistrib.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/analysis/runDistrib.sh b/archive/analysis/runDistrib.sh index 883f373e..1cf23ab3 100644 --- a/archive/analysis/runDistrib.sh +++ b/archive/analysis/runDistrib.sh @@ -119,10 +119,10 @@ logger -s -t runDistrib "finished consolidation, copying databases" rsync -avz -e "ssh -i $SERVERSSHKEY" $SERVERUSERID@$CALCSERVERIP:ukmon-shared/matches/RMSCorrelate/dbs/*.db $DATADIR/distrib ssh -i $SERVERSSHKEY $SERVERUSERID@$CALCSERVERIP "find /tmp -maxdepth 1 -name "*.pickle" -mtime +7 -exec rm -f {} \;" -loggers -s -t runDistrib "stopping calcserver again" +logger -s -t runDistrib "stopping calcserver again" aws ec2 stop-instances --instance-ids $SERVERINSTANCEID -loggers -s -t runDistrib "copying data to batch server and tidying up" +logger -s -t runDistrib "copying data to batch server and tidying up" # grab a copy of the indvidual container dbs so we can get a list of new solutions rm -Rf $DATADIR/latest/contdbs/ @@ -147,4 +147,4 @@ find $DATADIR/distrib/containers/ -name "cont*.tgz" -mtime +30 -exec rm -f {} \; find $DATADIR/distrib/ -maxdepth 1 -name "20*.tgz" -mtime +30 -exec rm -f {} \; rm -f $DATADIR/distrib/${rundate}.pickle -loggers -s -t "finished runDistrib" +logger -s -t "finished runDistrib" From a07908f696b1e8e7ab7af1ed4911c889c9394469 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 14:01:55 +0100 Subject: [PATCH 254/287] bugfix to work around typo --- archive/ukmon_pylib/metrics/getMatchStats.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index e1dcd4f9..f42d1cc2 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -77,7 +77,7 @@ def getMatchStats(logf): nonphys = beglowr + badalti + badvelo + badangl tot = added + uncal + missdf - rtims = [line.strip() for line in loglines if 'runDistrib' in line] + rtims = [line.strip() for line in loglines if 'runDistrib' in line and '/prod/' not in line] stim = rtims[0][11:19] etim = rtims[-1][11:19] d1=datetime.strptime(stim,'%H:%M:%S') @@ -102,11 +102,12 @@ def getMatchStats(logf): return tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime -def updateStats(obscount, candcount, trajcount, runtime): +def updateStats(obscount, candcount, trajcount, runtime, rundate): datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - rundate=os.getenv('rundate') if not rundate: - rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() + rundate=os.getenv('rundate') + if not rundate: + rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() statsdir = os.path.join(datadir, 'dailyreports') reports = glob.glob(os.path.join(statsdir, f'{rundate}*.txt')) @@ -126,7 +127,10 @@ def updateStats(obscount, candcount, trajcount, runtime): if __name__ == '__main__': - tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(sys.argv[1]) + matchfile = sys.argv[1] + tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(matchfile) + if '-' in matchfile: + rundate = matchfile[matchfile.find('-')+1:] if len(sys.argv) < 3: - updateStats(added, cands, trajs, runtime) + updateStats(added, cands, trajs, runtime, rundate=None) print(tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime) From 55bae434012bc69318925f0159d22aba97609411 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 14:02:07 +0100 Subject: [PATCH 255/287] print message when done --- archive/ukmon_pylib/reports/reportOfLatestMatches.py | 1 + 1 file changed, 1 insertion(+) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index 2d873858..d4c3d425 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -212,3 +212,4 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): # update the daily database of paired observations daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) + print('reportOfLatestMatches finished') From a5617a345ae6b7db6be7983501f21675535b8f65 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 15:06:23 +0100 Subject: [PATCH 256/287] bugfix to get trajectories from s3 before trying to check for missing trajectories... --- archive/ukmon_pylib/traj/createDistribMatchingSh.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/traj/createDistribMatchingSh.py b/archive/ukmon_pylib/traj/createDistribMatchingSh.py index 953e7d72..dfe4624a 100644 --- a/archive/ukmon_pylib/traj/createDistribMatchingSh.py +++ b/archive/ukmon_pylib/traj/createDistribMatchingSh.py @@ -147,12 +147,15 @@ def createExecConsolSh(matchstart, matchend, execconsolsh, istest=''): outf.write('logger -s -t execConsol start\n') outf.write(f'aws s3 sync {srcpath}/ ~/data/distrib/canddbs/ --exclude "*" --include "*.db" --exclude "dbs/*" --quiet\n') + # must run this before consolidation as the process updates the database to remove any missing + # trajectories ie not on disk. + outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') + refreshTrajectories(outf, matchstart, matchend, shbucket) + outf.write(f'python -m traj.consolidateDistTraj ~/data/distrib/canddbs/ {calcdir}/dbs/\n') outf.write(f'aws s3 sync {calcdir}/dbs/ {srcpath}/dbs/ --exclude "*" --include "*.db" --quiet\n') - outf.write('logger -s -t execConsol syncing any updated trajectories from shared S3\n') - refreshTrajectories(outf, matchstart, matchend, shbucket) outf.write('logger -s -t execConsol creating density plots\n') createDensityPlots(outf, calcdir, enddt, includeyear) outf.write('logger -s -t execConsol pushing data back to S3\n') From f56b37ecd6722e1f5ccb1eb761027a8a1488bb14 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 15:33:17 +0100 Subject: [PATCH 257/287] improve to support running for past dates --- .../ukmon_pylib/reports/reportOfLatestMatches.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index d4c3d425..125c20f0 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -41,11 +41,11 @@ def processLocalFolder(trajdir, basedir): return outstr -def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): +def getListOfNewMatches(dir_path, db_path, rundate): os.makedirs(db_path, exist_ok=True) - db_name = f'{rundate}_trajectories.db' if rundate else 'trajectories.db' + db_name = f'{rundate}_trajectories.db' dailydb = TrajectoryDatabase(db_path=db_path, db_name=db_name, purge_records=True) - flist = glob.glob(os.path.join(dir_path, 'trajectories_*.db')) + flist = glob.glob(os.path.join(dir_path, f'trajectories_{rundate}_*.db')) flist.sort() for fl in flist: tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') @@ -101,11 +101,11 @@ def getListOfNewMatches(dir_path, db_path='/tmp', rundate=None): return newdirs -def updatePairedDB(dir_path, db_path='/tmp', rundate=None): +def updatePairedDB(dir_path, db_path, rundate): os.makedirs(db_path, exist_ok=True) - db_name = f'{rundate}_observations.db' if rundate else 'observations.db' + db_name = f'{rundate}_observations.db' obsdb = ObservationsDatabase(db_path=db_path, db_name=db_name, purge_records=True) - flist = glob.glob(os.path.join(dir_path, 'observations_*.db')) + flist = glob.glob(os.path.join(dir_path, f'observations_{rundate}_*.db')) flist.sort() for fl in flist: tstamp = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S') @@ -202,6 +202,9 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): cand_db_dir = sys.argv[1] daily_db_dir = sys.argv[2] offset = sys.argv[3] + + if repdtstr != datetime.datetime.now().strftime('%Y%m%d'): + print(f'running for a past date, make sure you untarred the containerdbs to {cand_db_dir}') flist = glob.glob(os.path.join(cand_db_dir, 'trajectories_*.db')) if len(flist) == 0: From 7f20a70415079337f4b16de8c59ca90338b22a76 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 15:35:13 +0100 Subject: [PATCH 258/287] errors in bashrc and logrotate config --- archive/server_setup/.bash_aliases | 2 +- archive/server_setup/logs.conf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/server_setup/.bash_aliases b/archive/server_setup/.bash_aliases index 5ade8d17..fa6eb352 100644 --- a/archive/server_setup/.bash_aliases +++ b/archive/server_setup/.bash_aliases @@ -40,7 +40,7 @@ function calcserver { ipaddr=$(aws ec2 describe-instances --instance-ids $SERVERINSTANCEID --query Reservations[*].Instances[*].PrivateIpAddress --output text) isrunning=$(echo $sts | cut -d " " -f 1) if [ $isrunning -ne 16 ] ; then - $HOME/prod/utils/stopstart-calcengine.sh start + $HOME/prod/utils/stopstartCalcengine.sh start echo "starting server on ${ipaddr}... waiting 10s..." sleep 10 fi diff --git a/archive/server_setup/logs.conf b/archive/server_setup/logs.conf index 9554fd49..910c3bf2 100644 --- a/archive/server_setup/logs.conf +++ b/archive/server_setup/logs.conf @@ -1,7 +1,7 @@ daily rotate 7 notifempty -/home/ec2-user/prod/logs/*.log { +~/prod/logs/*.log { daily rotate 45 compress @@ -12,7 +12,7 @@ notifempty # create } -/home/ec2-user/dev/logs/*.log { +~/dev/logs/*.log { daily rotate 45 compress From 8333f0345c3d84a1b69b75f7f61d0f38d614e8a6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 15:54:59 +0100 Subject: [PATCH 259/287] prevent email being sent for anything other than current days report --- archive/ukmon_pylib/reports/dailyReport.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/reports/dailyReport.py b/archive/ukmon_pylib/reports/dailyReport.py index 69f6eb64..e4333e48 100644 --- a/archive/ukmon_pylib/reports/dailyReport.py +++ b/archive/ukmon_pylib/reports/dailyReport.py @@ -123,5 +123,7 @@ def LookForMatchesRMS(doff, dayfile, statsfile): mailFrom = recs[-1].strip() #mailRecip = 'markmcintyre99@googlemail.com' # TODO for testing only yest = (datetime.date.today()-datetime.timedelta(days=doff)).strftime('%Y-%m-%d') - mailSubj = f'Latest Match Report for {yest}' - sendAnEmail(mailRecip, bodytext, mailSubj, mailFrom, msg_html=body.replace('href="/reports', f'href="{targeturl}/reports')) + # only send the email for the most recent report. + if doff == 1: + mailSubj = f'Latest Match Report for {yest}' + sendAnEmail(mailRecip, bodytext, mailSubj, mailFrom, msg_html=body.replace('href="/reports', f'href="{targeturl}/reports')) From 54454045ee8f61eedf5c0a6cbe85cf02d4f0473a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 16:18:13 +0100 Subject: [PATCH 260/287] bugfix: make sure we use the most recent stats --- archive/ukmon_pylib/reports/dailyReport.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/reports/dailyReport.py b/archive/ukmon_pylib/reports/dailyReport.py index e4333e48..3c5bc897 100644 --- a/archive/ukmon_pylib/reports/dailyReport.py +++ b/archive/ukmon_pylib/reports/dailyReport.py @@ -55,11 +55,7 @@ def AddRowRMS(body, bodytext, ele): return body, bodytext -def LookForMatchesRMS(doff, dayfile, statsfile): - # get stats - with open(statsfile, 'r') as inf: - lis = inf.readlines() - stats = lis[-1].strip().split(' ') +def LookForMatchesRMS(doff, dayfile, stats): bodytext = 'Daily notification of matches\n\n' body = '' @@ -100,7 +96,14 @@ def LookForMatchesRMS(doff, dayfile, statsfile): dailyreps.sort() dailyrep = dailyreps[-1] statfile = os.path.join(reppth, 'stats.txt') - _, _, body, bodytext = LookForMatchesRMS(doff, dailyrep, statfile) + lis = open(statfile, 'r').readlines() + stats = [li for li in lis if repdtstr in li] + if len(stats) > 0: + stats = stats[0].strip().split(' ') + else: + stats = lis[-1].strip().split(' ') + + _, _, body, bodytext = LookForMatchesRMS(doff, dailyrep, stats) #body = body.replace('assets/img/logo.svg', 'latest/dailyreports/dailyreportsidx.html') if doff == 1: From 8c631652b8b6c978de77171d11a9e0c5847e55c9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 16:49:59 +0100 Subject: [PATCH 261/287] improvement in reportOfLatestMatches to handle past dates --- .../reports/reportOfLatestMatches.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index 125c20f0..3f4d888d 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -12,6 +12,7 @@ import tempfile import boto3 import glob +import tarfile from traj.pickleAnalyser import getVMagCodeAndStations from reports.CameraDetails import findSite, loadLocationDetails @@ -195,23 +196,27 @@ def findNewMatches(dir_path, out_path, offset, repdtstr): if __name__ == '__main__': - repdtstr = None - if len(sys.argv) > 4: - repdtstr = sys.argv[4] cand_db_dir = sys.argv[1] daily_db_dir = sys.argv[2] - offset = sys.argv[3] + repdtstr = sys.argv[3] if repdtstr != datetime.datetime.now().strftime('%Y%m%d'): - print(f'running for a past date, make sure you untarred the containerdbs to {cand_db_dir}') + print(f'running for a past date, extracting the container dbs to {cand_db_dir}') + datadir = os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) + tarf = tarfile.open(os.path.join(datadir, 'distrib','containers',f'contdbs_{repdtstr}.tgz'), 'r') + members = [m for m in tarf.getmembers() if '.db' in m.name] + for m in members: + m.name = os.path.basename(m.name) + tarf.extractall(cand_db_dir, filter='fully_trusted') + tarf.close() flist = glob.glob(os.path.join(cand_db_dir, 'trajectories_*.db')) if len(flist) == 0: print('no container databases to process, aborting') else: # arguments dblocation, datadir, days ago, rundate eg 20220524 - findNewMatches(cand_db_dir, daily_db_dir, offset, repdtstr) + findNewMatches(cand_db_dir, daily_db_dir, repdtstr) # update the daily database of paired observations daily_db_dir = os.path.join(os.path.split(cand_db_dir)[0], 'dailydbs') updatePairedDB(cand_db_dir, daily_db_dir, repdtstr) From ab5fd5a37ef347a567827f8403dc5942e2d9766d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 16:52:10 +0100 Subject: [PATCH 262/287] bugfix --- archive/ukmon_pylib/reports/reportOfLatestMatches.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/archive/ukmon_pylib/reports/reportOfLatestMatches.py b/archive/ukmon_pylib/reports/reportOfLatestMatches.py index 3f4d888d..d79ddeff 100644 --- a/archive/ukmon_pylib/reports/reportOfLatestMatches.py +++ b/archive/ukmon_pylib/reports/reportOfLatestMatches.py @@ -122,17 +122,14 @@ def updatePairedDB(dir_path, db_path, rundate): return obscount -def findNewMatches(dir_path, out_path, offset, repdtstr): +def findNewMatches(dir_path, out_path, repdtstr): daily_path = os.path.join(os.path.split(dir_path)[0], 'dailydbs') newdirs = getListOfNewMatches(dir_path, daily_path, rundate=repdtstr) # load camera details caminfo = loadLocationDetails() caminfo = caminfo[caminfo.active==1] - if repdtstr is not None: - repdt = datetime.datetime.strptime(repdtstr, '%Y%m%d') - else: - repdt = datetime.datetime.now() - datetime.timedelta(int(offset)) + repdt = datetime.datetime.strptime(repdtstr, '%Y%m%d') os.makedirs(out_path, exist_ok=True) # create filename. Allow for three reruns in a day From 34299193fac99cdad8471b15a037dd4c06e8e007 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 16:53:59 +0100 Subject: [PATCH 263/287] small tweak to support reportOfLatestMatches --- archive/analysis/findAllMatches.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 05c9c9a8..00f7b981 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -71,7 +71,7 @@ cd $here logger -s -t findAllMatches "start reportOfLatestMatches" matchlog=${SRC}/logs/matchJob.log -python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $MATCHEND $rundate +python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $rundate python -m metrics.getMatchStats $matchlog # copy stats to S3 so the daily report can run From 398ab0b5e87bea62a33bb8ec952794a5547a6de6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 17:01:04 +0100 Subject: [PATCH 264/287] tweak getMatchStats to handle historic data better --- archive/analysis/findAllMatches.sh | 2 +- archive/ukmon_pylib/metrics/getMatchStats.py | 18 ++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/archive/analysis/findAllMatches.sh b/archive/analysis/findAllMatches.sh index 00f7b981..59872560 100644 --- a/archive/analysis/findAllMatches.sh +++ b/archive/analysis/findAllMatches.sh @@ -72,7 +72,7 @@ logger -s -t findAllMatches "start reportOfLatestMatches" matchlog=${SRC}/logs/matchJob.log python -m reports.reportOfLatestMatches $DATADIR/latest/contdbs $DATADIR/dailyreports $rundate -python -m metrics.getMatchStats $matchlog +python -m metrics.getMatchStats $matchlog $rundate # copy stats to S3 so the daily report can run if [ "$RUNTIME_ENV" == "PROD" ] ; then diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index f42d1cc2..46a2f78a 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -36,7 +36,7 @@ def getDailyObsCounts(logf): return -def getMatchStats(logf): +def getMatchStats(logf, rundate): loglines = open(logf).readlines() events = [line.strip() for line in loglines if 'Analysing' in line and 'observations' in line] @@ -92,9 +92,6 @@ def getMatchStats(logf): cstime = str(d2 - d1) datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - rundate=os.getenv('rundate') - if not rundate: - rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() dailydbdir = os.path.join(datadir, 'latest','dailydbs') trajdb = TrajectoryDatabase(dailydbdir, f'{rundate}_trajectories.db') trajs = len(trajdb.getTrajBasics('.',[0,9999999])) @@ -104,10 +101,6 @@ def getMatchStats(logf): def updateStats(obscount, candcount, trajcount, runtime, rundate): datadir=os.getenv('DATADIR', default=os.path.expanduser('~/prod/data')) - if not rundate: - rundate=os.getenv('rundate') - if not rundate: - rundate = open(os.path.join(datadir, 'rundate.txt'),'r').readline().strip() statsdir = os.path.join(datadir, 'dailyreports') reports = glob.glob(os.path.join(statsdir, f'{rundate}*.txt')) @@ -128,9 +121,10 @@ def updateStats(obscount, candcount, trajcount, runtime, rundate): if __name__ == '__main__': matchfile = sys.argv[1] - tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(matchfile) - if '-' in matchfile: - rundate = matchfile[matchfile.find('-')+1:] + rundate = sys.argv[2] + + tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(matchfile, rundate) + if len(sys.argv) < 3: - updateStats(added, cands, trajs, runtime, rundate=None) + updateStats(added, cands, trajs, runtime, rundate) print(tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime) From abd0ed20d47d9310d28064c6a23d626c5c3fc638 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 18:14:58 +0100 Subject: [PATCH 265/287] add mariadb and ping to the sec grp so that the database can be accessed by apis --- archive/terraform/ukmda/secgrp.tf | 40 ++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/archive/terraform/ukmda/secgrp.tf b/archive/terraform/ukmda/secgrp.tf index 1ae17dd0..a6aa5e1d 100644 --- a/archive/terraform/ukmda/secgrp.tf +++ b/archive/terraform/ukmda/secgrp.tf @@ -46,8 +46,8 @@ resource "aws_security_group" "ec2_secgrp" { description = "Security Group for EC2 instances" vpc_id = aws_vpc.ec2_vpc.id ingress = [ -/* { - cidr_blocks = [data.aws_vpc.mjmm_ec2_vpc.cidr_block] + { + cidr_blocks = [aws_vpc.ec2_vpc.cidr_block] description = "SSH for Admin" from_port = 22 protocol = "tcp" @@ -56,9 +56,9 @@ resource "aws_security_group" "ec2_secgrp" { prefix_list_ids = [] security_groups = [] self = false - },*/ + }, { - cidr_blocks = [aws_vpc.ec2_vpc.cidr_block] + cidr_blocks = ["0.0.0.0/0"] description = "SSH for Admin" from_port = 22 protocol = "tcp" @@ -69,16 +69,38 @@ resource "aws_security_group" "ec2_secgrp" { self = false }, { - cidr_blocks = ["0.0.0.0/0"] - description = "SSH for Admin" - from_port = 22 + cidr_blocks = ["194.0.0.0/8"] + description = "MariaDB" + from_port = 3306 protocol = "tcp" - to_port = 22 + to_port = 3306 ipv6_cidr_blocks = [] prefix_list_ids = [] security_groups = [] self = false - } + }, + { + cidr_blocks = ["0.0.0.0/0"] + description = "icmp" + from_port = -1 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "icmp" + security_groups = [] + self = false + to_port = -1 + }, + { + cidr_blocks = ["0.0.0.0/0"] + description = "mariadb" + from_port = 3306 + ipv6_cidr_blocks = [] + prefix_list_ids = [] + protocol = "tcp" + security_groups = [] + self = false + to_port = 3306 + }, ] egress = [ { From 31affc2b3e318b48ef97c09d47c59ef9a94c6f82 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 4 May 2026 18:15:11 +0100 Subject: [PATCH 266/287] update the db servername --- archive/terraform/ukmda/ssm_variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/terraform/ukmda/ssm_variables.tf b/archive/terraform/ukmda/ssm_variables.tf index 690de73b..5661b507 100644 --- a/archive/terraform/ukmda/ssm_variables.tf +++ b/archive/terraform/ukmda/ssm_variables.tf @@ -6,7 +6,7 @@ resource "aws_ssm_parameter" "prod_dbhost" { provider = aws.eu-west-1-prov name = "prod_dbhost" type = "String" - value = "3.11.55.160" + value = "batchserver.ukmeteors.co.uk" tags = { "billingtag" = "ukmda" } From ca9ad30b56149e5aa036446314a47d9ab7848448 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 5 May 2026 11:52:09 +0100 Subject: [PATCH 267/287] bugfix in match stats --- archive/ukmon_pylib/metrics/getMatchStats.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/metrics/getMatchStats.py b/archive/ukmon_pylib/metrics/getMatchStats.py index 46a2f78a..05d098ed 100644 --- a/archive/ukmon_pylib/metrics/getMatchStats.py +++ b/archive/ukmon_pylib/metrics/getMatchStats.py @@ -125,6 +125,5 @@ def updateStats(obscount, candcount, trajcount, runtime, rundate): tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime = getMatchStats(matchfile, rundate) - if len(sys.argv) < 3: - updateStats(added, cands, trajs, runtime, rundate) + updateStats(added, cands, trajs, runtime, rundate) print(tot, added, uncal, missdf, nonphys, cands, trajs, runtime, cstime) From accfc82e96e2a67b01cc6b7b56d4c4db01402261 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 5 May 2026 11:53:51 +0100 Subject: [PATCH 268/287] prevent bogus renaming of matchJob.log --- archive/cronjobs/nightlyJob.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/archive/cronjobs/nightlyJob.sh b/archive/cronjobs/nightlyJob.sh index b955b709..5d63a02a 100644 --- a/archive/cronjobs/nightlyJob.sh +++ b/archive/cronjobs/nightlyJob.sh @@ -48,9 +48,10 @@ $SRC/analysis/createSearchable.sh $yr singles matchlog=matchJob.log # save the existing log, in case the process is being rerun on the same day -if [ -f $SRC/logs/$matchlog ] ; then - suff=$(stat $SRC/logs/$matchjob -c %X) - mv $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff +if [ "$(find $SRC/logs -name $matchlog -mmin +1380 -ls)" != "" ] ; then + dt=$(stat $SRC/logs/$matchlog -c %y) + suff=$(date --date ${dt:0:10} +%Y%m%d) + mv -f $SRC/logs/$matchlog $SRC/logs/$matchlog-$suff fi logger -s -t nightlyJob "start findAllMatches" From 7163bee08cce87cc46e194b3d5ce70e0464665d8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 5 May 2026 12:09:28 +0100 Subject: [PATCH 269/287] bumping version --- archive/server_setup/migratingBatchServer.md | 10 +++++++--- archive/ukmon_pylib/analysis/getUsedUnused.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/archive/server_setup/migratingBatchServer.md b/archive/server_setup/migratingBatchServer.md index 7985e0e1..1bf7db16 100644 --- a/archive/server_setup/migratingBatchServer.md +++ b/archive/server_setup/migratingBatchServer.md @@ -78,7 +78,10 @@ $SRC/utils/loadMatchCsvMDB.sh ``` ## Mariadb database -Sudo to root and run the following to set a root password +Find the mariadb config file (normally in /etc, might ber called `my.cnf`) and ensure that `bind-address` is set to `0.0.0.0`. The default on many installations is to bind to 127.0.0.1 which allows traffic only from localhost. The UKMON APIs access the database externally. After changing the config file, restart the database. + + +Now sudo to root and run the following to set a root password ``` bash mysql -u root -p ALTER USER 'root'@'localhost' IDENTIFIED VIA mysql_native_password USING PASSWORD('ROOTPASSWORD'); @@ -148,6 +151,7 @@ and then reload sshd ``` bash service sshd reload ``` + ### Now move the user accounts and update camera config First, ensure that root on the new server can connect to the old server as the batch user by creating a default SSH keypair for root, and adding its public half to root's authorized_keys file on the old server. @@ -163,6 +167,6 @@ ssh oldserver "sudo ls -1 /var/sftp" > ./move/sftp_accts.txt ``` bash $SRC/utils/migrateSftpAccts.sh oldserverFQDN ./move/sftp_accts.txt ``` -This will also update the `ukmon.ini` file for each station on both server. Upon next connection, the station should download the new ini file and thereafter connect to the new server. +This will also update the `ukmon.ini` file for each station on both server. Upon next connection, the station should update its local ini file and thereafter connect to the new server. -This means that the old server must be kept running for a few days, till all cameras have switched over. +This means that the old server must be kept running for a few days till all cameras have switched over. diff --git a/archive/ukmon_pylib/analysis/getUsedUnused.py b/archive/ukmon_pylib/analysis/getUsedUnused.py index 3d1b6f54..0541ecad 100644 --- a/archive/ukmon_pylib/analysis/getUsedUnused.py +++ b/archive/ukmon_pylib/analysis/getUsedUnused.py @@ -62,4 +62,4 @@ def getUnusedImages(dtstr): allimages = getAllImages(dtstr) uncal = getKnownImages(dtstr, 'uncal') used = getKnownImages(dtstr, 'consumed') - unused = list(set(allimages)-set(used)-set(uncal)) \ No newline at end of file + unused = list(set(allimages)-set(used + uncal)) \ No newline at end of file From c002178b1c025f6074d2c39cfe7200d7e1b3cf54 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 5 May 2026 12:10:02 +0100 Subject: [PATCH 270/287] bump version 2026.04.1 -> 2026.05.0 --- README.md | 2 +- archive/README.md | 2 +- bumpver.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 595de7c5..9f059c11 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2026.04.1 +version: 2026.05.0 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index 108c7fbf..c37b478a 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,7 +1,7 @@ Data Processing and Flows ========================== -version: 2026.04.1 +version: 2026.05.0 This diagram shows the overall flow of data from Cameras to websites and out to the public. diff --git a/bumpver.toml b/bumpver.toml index f8ef5f0f..17d501fc 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2026.04.1" +current_version = "2026.05.0" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true From 86d7e549e8bf5aea3f92ee66f0d73314d6831df6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 5 May 2026 21:09:18 +0100 Subject: [PATCH 271/287] update matchapi to use indexed column --- archive/lambdas/matchDataApi/matchDataApi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/lambdas/matchDataApi/matchDataApi.py b/archive/lambdas/matchDataApi/matchDataApi.py index 73c393db..3f4d1f17 100644 --- a/archive/lambdas/matchDataApi/matchDataApi.py +++ b/archive/lambdas/matchDataApi/matchDataApi.py @@ -67,7 +67,7 @@ def getStationData(statid, dtstr, period=None): statfrag = f"and s.stations like '%{statid}%' " if statid is not None else "" perfrag = periodToSqlFragment(period) if period is not None else "" with connection.cursor() as cursor: - sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' {statfrag} {perfrag}" + sql = f"SELECT s.orbname from matches s where s.orbname like '{dtstr}%' {statfrag} {perfrag}" cursor.execute(sql) result = cursor.fetchall() finally: @@ -85,7 +85,7 @@ def getSummaryData(dtstr, period=None): fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' - expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%' {perfrag}" + expr = f"SELECT {fieldlist} from matches s where s.orbname like '{dtstr}%' {perfrag}" result=[] try: with connection.cursor() as cursor: @@ -171,7 +171,7 @@ def lambda_handler(event, context): res = '{"points": "unavailable"}' else: - res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}' + res = '{"invalid request type - must be one of \'matches\', \'detail\', \'station\',\'summary\'"}' print(res) return { From c7eec3a85cb0dfb1d2e738a0b11f5c10ecb044ba Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 5 May 2026 22:27:02 +0100 Subject: [PATCH 272/287] add api readme --- api_examples/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 api_examples/README.md diff --git a/api_examples/README.md b/api_examples/README.md new file mode 100644 index 00000000..4de90ef8 --- /dev/null +++ b/api_examples/README.md @@ -0,0 +1,6 @@ +#Available APIs and Examples + +We have a number of APIs which can be used to retrieve our data programatically, either using commandline tools like curl and wget, or from programming languages such as Python. + +Data are returned in JSON format which can readily be converted into a python list, array or Pandas DataFrame. Simple examples are shown on our main website (here)[https://ukmeteornetwork.org/our-data-apis/], and a set of Python examples are available here. A Data Dictionary in Excel format can be downloaded from (here)[https://archive.ukmeteors.co.uk/browse/datadictionary.xlsx]. + From 59e51789640e0f23f7ac4057a12bc92672354848 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 12:49:49 +0100 Subject: [PATCH 273/287] tweak to cammetrics to enhance stationlogins report --- archive/ukmon_pylib/metrics/camMetrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 36aeaf98..7b3107a6 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -206,15 +206,15 @@ def backPopulate(stationid): df['uploadtime']=df.uploadtime.astype("str").str.pad(6,fillchar="0") df['lastupload']=df.upddate.astype('str') + '_' +df.uploadtime df.lastupload = [datetime.datetime.strptime(x, '%Y%m%d_%H%M%S') for x in df.lastupload] - df = df.drop(columns=['stationid','manual','rundate', 'upddate','uploadtime']) + df = df.drop(columns=['manual','rundate', 'upddate','uploadtime']) df = df.sort_values(by=['lastupload']) outfile=os.path.join(datadir, 'reports', 'stationlogins.txt') zerodate = datetime.datetime(1970,1,1,0,0,0) with open(outfile,'w') as outf: - outf.write('Last Upload, StationID, Last Login, Via\n') + outf.write(f'{"Last Upload":19s} {"Last Connect":19s} {"StationID":20s} {"GMN ID":10s} Via\n') for _,rw in df.iterrows(): lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') via = '' if pd.isnull(rw.host) else rw.host - outf.write(f'{lastup} , {rw.location:20s}, {lastlo}, {via}\n') + outf.write(f'{lastup} {lastlo} {rw.location:20s} {rw.stationid:10s} {via}\n') From 4f9bed760f961cdcb1cd2f524f408dd280337bf3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 13:01:03 +0100 Subject: [PATCH 274/287] slight tweak to formatting --- archive/ukmon_pylib/metrics/camMetrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/ukmon_pylib/metrics/camMetrics.py b/archive/ukmon_pylib/metrics/camMetrics.py index 7b3107a6..6824db9f 100644 --- a/archive/ukmon_pylib/metrics/camMetrics.py +++ b/archive/ukmon_pylib/metrics/camMetrics.py @@ -214,7 +214,7 @@ def backPopulate(stationid): with open(outfile,'w') as outf: outf.write(f'{"Last Upload":19s} {"Last Connect":19s} {"StationID":20s} {"GMN ID":10s} Via\n') for _,rw in df.iterrows(): - lastup = '> 1 month' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') - lastlo = '> 1 month' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') + lastup = '> 1 month ago' if pd.isnull(rw.lastupload) else rw.lastupload.strftime('%Y-%m-%dT%H:%M:%S') + lastlo = '> 1 month ago' if pd.isnull(rw.lastseen) else rw.lastseen.strftime('%Y-%m-%dT%H:%M:%S') via = '' if pd.isnull(rw.host) else rw.host - outf.write(f'{lastup} {lastlo} {rw.location:20s} {rw.stationid:10s} {via}\n') + outf.write(f'{lastup:19s} {lastlo:19s} {rw.location:20s} {rw.stationid:10s} {via}\n') From e8d0c0f6e2a848e1564e6eabf271a26afe6a723b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 15:45:31 +0100 Subject: [PATCH 275/287] minor tweak to avoid unexpectedly changing directory --- install_or_update.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index 83713e34..fed959ad 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -10,8 +10,8 @@ envname=$(echo $RUNTIME_ENV | tr '[:upper:]' '[:lower:]') here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +pushd $here/archive echo "Updating codebase..." -cd $here/archive git pull echo "Updating code..." @@ -69,4 +69,5 @@ else echo skipping config and bashrc fi echo "" +popd echo "$msg complete" From 4fa95d5872036706009067e60d015c008286b221 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 16:06:23 +0100 Subject: [PATCH 276/287] add support script for during server migrations --- archive/server_setup/checkSftpAccounts.py | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 archive/server_setup/checkSftpAccounts.py diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py new file mode 100644 index 00000000..830f375d --- /dev/null +++ b/archive/server_setup/checkSftpAccounts.py @@ -0,0 +1,58 @@ +import pandas as pd +import datetime + + +data=open('/home/ubuntu/prod/data/reports/stationlogins.txt').readlines() +camlist = [] +not_live = [] +still_upl = [] +not_upl = [] + +livedate = datetime.datetime.now() - datetime.timedelta(days=10) +for li in data: + lastup = li[:19] + lastlo = li[21:40] + loc = li[42:61] + gmnid = li[64:73] + via = li[76:].strip() + camlist.append([lastup, lastlo, loc, gmnid, via]) + + if ">" in lastup and ">" in lastlo: + not_live.append([loc, gmnid, lastup, lastlo, via]) + + elif ">" not in lastup: + lastupdt = datetime.datetime.strptime(lastup, '%Y-%m-%dT%H:%M:%S') + if lastupdt >= livedate: + still_upl.append([loc, gmnid, lastup, lastlo, via]) + else: + if '>' in lastlo: + not_live.append([loc, gmnid, lastup, lastlo, via]) + else: + lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S') + if lastlodt >= livedate: + not_upl.append([loc, gmnid, lastup, lastlo, via]) + else: + not_live.append([loc, gmnid, lastup, lastlo, via]) + else: # '>' not in lastlo + lastlodt = datetime.datetime.strptime(lastlo, '%Y-%m-%dT%H:%M:%S') + if lastlodt >= livedate: + not_upl.append([loc, gmnid, lastup, lastlo, via]) + else: + not_live.append([loc, gmnid, lastup, lastlo, via]) + + + + + +with open('still-live.txt','w') as outf: + for cam in still_upl: + outf.write(','.join(cam) + '\n') + +with open('not_uploading.txt','w') as outf: + for cam in not_upl: + outf.write(','.join(cam) + '\n') + +with open('not_live.txt','w') as outf: + for cam in not_live: + outf.write(','.join(cam) + '\n') + \ No newline at end of file From a0fad5a7addde6837e254d8ac84795666a186975 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 16:08:02 +0100 Subject: [PATCH 277/287] skip 1st row --- archive/server_setup/checkSftpAccounts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py index 830f375d..0a9812f5 100644 --- a/archive/server_setup/checkSftpAccounts.py +++ b/archive/server_setup/checkSftpAccounts.py @@ -10,6 +10,8 @@ livedate = datetime.datetime.now() - datetime.timedelta(days=10) for li in data: + if 'Last Upload' in li: + continue lastup = li[:19] lastlo = li[21:40] loc = li[42:61] From 0c6ed696297f7bc30a854bc9a9786770e33d2ed0 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 16:18:09 +0100 Subject: [PATCH 278/287] add check for todos --- archive/server_setup/checkSftpAccounts.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py index 0a9812f5..878fcf48 100644 --- a/archive/server_setup/checkSftpAccounts.py +++ b/archive/server_setup/checkSftpAccounts.py @@ -7,6 +7,7 @@ not_live = [] still_upl = [] not_upl = [] +livenames = [] livedate = datetime.datetime.now() - datetime.timedelta(days=10) for li in data: @@ -26,6 +27,7 @@ lastupdt = datetime.datetime.strptime(lastup, '%Y-%m-%dT%H:%M:%S') if lastupdt >= livedate: still_upl.append([loc, gmnid, lastup, lastlo, via]) + livenames.append(loc) else: if '>' in lastlo: not_live.append([loc, gmnid, lastup, lastlo, via]) @@ -42,10 +44,6 @@ else: not_live.append([loc, gmnid, lastup, lastlo, via]) - - - - with open('still-live.txt','w') as outf: for cam in still_upl: outf.write(','.join(cam) + '\n') @@ -57,4 +55,10 @@ with open('not_live.txt','w') as outf: for cam in not_live: outf.write(','.join(cam) + '\n') - \ No newline at end of file + +donelist = open('done.txt', 'r').readlines() +with open('todo.txt', 'w') as outf: + for nam in livenames: + if nam not in donelist: + print('done list is missing', nam) + outf.write(f'{nam}\n') \ No newline at end of file From 2125490b847b08e47db3fcdedc368e585e0afdae Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 16:19:49 +0100 Subject: [PATCH 279/287] oops strip whitespace --- archive/server_setup/checkSftpAccounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py index 878fcf48..6ddeb504 100644 --- a/archive/server_setup/checkSftpAccounts.py +++ b/archive/server_setup/checkSftpAccounts.py @@ -27,7 +27,7 @@ lastupdt = datetime.datetime.strptime(lastup, '%Y-%m-%dT%H:%M:%S') if lastupdt >= livedate: still_upl.append([loc, gmnid, lastup, lastlo, via]) - livenames.append(loc) + livenames.append(loc.strip()) else: if '>' in lastlo: not_live.append([loc, gmnid, lastup, lastlo, via]) @@ -59,6 +59,6 @@ donelist = open('done.txt', 'r').readlines() with open('todo.txt', 'w') as outf: for nam in livenames: - if nam not in donelist: + if nam.strip() not in donelist: print('done list is missing', nam) outf.write(f'{nam}\n') \ No newline at end of file From 26de8156cb788f7ac49e75bd463856538167b0f8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 16:23:58 +0100 Subject: [PATCH 280/287] darn whitespace --- archive/server_setup/checkSftpAccounts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/archive/server_setup/checkSftpAccounts.py b/archive/server_setup/checkSftpAccounts.py index 6ddeb504..8149c018 100644 --- a/archive/server_setup/checkSftpAccounts.py +++ b/archive/server_setup/checkSftpAccounts.py @@ -57,6 +57,7 @@ outf.write(','.join(cam) + '\n') donelist = open('done.txt', 'r').readlines() +donelist = [x.strip() for x in donelist] with open('todo.txt', 'w') as outf: for nam in livenames: if nam.strip() not in donelist: From 48fa13c5130b67a49ecbd555f771dc05fecceb5b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 18:40:11 +0100 Subject: [PATCH 281/287] working on migration tasks --- archive/server_setup/acctStatusChecks.sh | 13 +++++++++++ archive/server_setup/backupSftpAccts.sh | 29 ++++++++++++++++++++++++ archive/server_setup/copyTestData.sh | 14 ------------ archive/server_setup/migrateSftpAccts.sh | 2 ++ install_or_update.sh | 4 ++++ 5 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 archive/server_setup/acctStatusChecks.sh create mode 100644 archive/server_setup/backupSftpAccts.sh delete mode 100644 archive/server_setup/copyTestData.sh diff --git a/archive/server_setup/acctStatusChecks.sh b/archive/server_setup/acctStatusChecks.sh new file mode 100644 index 00000000..6c346387 --- /dev/null +++ b/archive/server_setup/acctStatusChecks.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +cd $here + +sudo ls -1 /var/sftp > done.txt +grep ukmonh /home/ubuntu/prod/data/reports/stationlogins.txt |awk '{print $3}'| sort > pending.txt + +ssh ukmonhelper2 "~/prod/server_setup/get-nbd.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > not-being-done.txt +ssh ukmonhelper2 "~/prod/server_setup/get-all.sh" | while read i ; do echo $i | awk -F"/" '{print $4}' ; done > all-accounts.txt + +python ~/src/ukmda-dataprocessing/archive/server_setup/checkSftpAccounts.py diff --git a/archive/server_setup/backupSftpAccts.sh b/archive/server_setup/backupSftpAccts.sh new file mode 100644 index 00000000..64f4d582 --- /dev/null +++ b/archive/server_setup/backupSftpAccts.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +cd $here + +bkpOneUser() { + userid=$1 + srchost=$2 + mkdir -p ./backup/$userid + sudo rsync -av $srchost:/var/sftp/$userid/ ./backup/$userid + cat ./backup/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini + mv -f /tmp/$userid.ini ./backup/$userid/ukmon.ini +} + +if [ $# -lt 2 ] ; then + echo "Usage: ./backupSftpAccounts.sh oldservername userfile" + exit +fi + +echo "Warning: this must only be run on the new server" +read -p "press ctrl-c to quit or enter to continue" + +oldserver=$1 +srcfile=$2 + +cat $srcfile | while read stn +do + bkpOneUser $stn $oldserver +done \ No newline at end of file diff --git a/archive/server_setup/copyTestData.sh b/archive/server_setup/copyTestData.sh deleted file mode 100644 index e2c041c5..00000000 --- a/archive/server_setup/copyTestData.sh +++ /dev/null @@ -1,14 +0,0 @@ -# prep some test data -$yr=$(date +%Y) - -here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -source $here/../config.ini >/dev/null 2>&1 - -rsync -avz ~/prod/data/single/ $DATADIR/single/ -rsync -avz ~/prod/data/matched/*${yr}* $DATADIR/matched/ -rsync -avz ~/prod/data/consolidated/*${yr}* $DATADIR/consolidated/ -rsync -avz ~/prod/data/dailyreports/stats.txt $DATADIR/dailyreports/ -rsync -avz ~/prod/data/dailyreports/${yr}*.txt $DATADIR/dailyreports/ -rsync -avz ~/prod/logs/*${yr}*.txt $SRC/logs/ -rsync -avz ~/prod/data/*${yr}*.jpg $DATADIR/ -rsync -avz ~/prod/data/*.png $DATADIR/ \ No newline at end of file diff --git a/archive/server_setup/migrateSftpAccts.sh b/archive/server_setup/migrateSftpAccts.sh index 52d5add0..82344495 100755 --- a/archive/server_setup/migrateSftpAccts.sh +++ b/archive/server_setup/migrateSftpAccts.sh @@ -1,5 +1,7 @@ #!/bin/bash +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +cd $here addOneUser() { userid=$1 diff --git a/install_or_update.sh b/install_or_update.sh index fed959ad..af2814bb 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -32,6 +32,10 @@ mkdir -p $DATADIR/browse/{annual,monthly,daily,showers} mkdir -p ~/$envname/logs mkdir -p ~/.logrotate mkdir -p ~/.aws +mkdir -p ~/server_setup + +rsync -a server_setup/*.sh ~/server_setup +rsync -a server_setup/*.py ~/server_setup echo "Checking conda environment..." if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] From 6a3af380f9af636ecf85ed309d8180182ae887ed Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 18:44:16 +0100 Subject: [PATCH 282/287] minor tweaks --- archive/server_setup/backupSftpAccts.sh | 1 + install_or_update.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/archive/server_setup/backupSftpAccts.sh b/archive/server_setup/backupSftpAccts.sh index 64f4d582..df09ff64 100644 --- a/archive/server_setup/backupSftpAccts.sh +++ b/archive/server_setup/backupSftpAccts.sh @@ -8,6 +8,7 @@ bkpOneUser() { srchost=$2 mkdir -p ./backup/$userid sudo rsync -av $srchost:/var/sftp/$userid/ ./backup/$userid + sudo chown -R $USER:$USER ./backup/$userid cat ./backup/$userid/ukmon.ini | sed 's/3.11.55.160/batchserver.ukmeteors.co.uk/g' > /tmp/$userid.ini mv -f /tmp/$userid.ini ./backup/$userid/ukmon.ini } diff --git a/install_or_update.sh b/install_or_update.sh index af2814bb..f7988fe5 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -35,7 +35,8 @@ mkdir -p ~/.aws mkdir -p ~/server_setup rsync -a server_setup/*.sh ~/server_setup -rsync -a server_setup/*.py ~/server_setup +rsync -a server_setup/*. ~/server_setup +chmod +x ~/server_setup/*.sh echo "Checking conda environment..." if [[ -f ~/.condaon && -d ~/miniconda3/envs/wmpl ]] From 7c8307849ccb1540640bc596b838df17d4811822 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 6 May 2026 18:45:52 +0100 Subject: [PATCH 283/287] daft typo --- install_or_update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_or_update.sh b/install_or_update.sh index f7988fe5..c903adce 100755 --- a/install_or_update.sh +++ b/install_or_update.sh @@ -35,7 +35,7 @@ mkdir -p ~/.aws mkdir -p ~/server_setup rsync -a server_setup/*.sh ~/server_setup -rsync -a server_setup/*. ~/server_setup +rsync -a server_setup/*.py ~/server_setup chmod +x ~/server_setup/*.sh echo "Checking conda environment..." From fdb17b5d4eb9d2aca8b70df6a0977589f54a3e60 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 7 May 2026 00:50:56 +0100 Subject: [PATCH 284/287] bump version 2026.05.0 -> 2026.05.1 --- README.md | 2 +- archive/README.md | 2 +- bumpver.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9f059c11..90d61f03 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2026.05.0 +version: 2026.05.1 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index c37b478a..2eda151a 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,7 +1,7 @@ Data Processing and Flows ========================== -version: 2026.05.0 +version: 2026.05.1 This diagram shows the overall flow of data from Cameras to websites and out to the public. diff --git a/bumpver.toml b/bumpver.toml index 17d501fc..016cbab2 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2026.05.0" +current_version = "2026.05.1" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true From e99637a2ed6a8ed592da9f8f358df4f041cc564c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 7 May 2026 00:51:16 +0100 Subject: [PATCH 285/287] bump version 2026.05.1 -> 2026.05.2 --- README.md | 2 +- archive/README.md | 2 +- bumpver.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 90d61f03..2e9d9281 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # UK Meteor Data Analysis Shared code and libraries -version: 2026.05.1 +version: 2026.05.2 This repository contains the code behind the UK Meteors data archive and data processing pipeline. diff --git a/archive/README.md b/archive/README.md index 2eda151a..e8cdc90f 100644 --- a/archive/README.md +++ b/archive/README.md @@ -1,7 +1,7 @@ Data Processing and Flows ========================== -version: 2026.05.1 +version: 2026.05.2 This diagram shows the overall flow of data from Cameras to websites and out to the public. diff --git a/bumpver.toml b/bumpver.toml index 016cbab2..cc67f302 100644 --- a/bumpver.toml +++ b/bumpver.toml @@ -1,5 +1,5 @@ [bumpver] -current_version = "2026.05.1" +current_version = "2026.05.2" version_pattern = "YYYY.0M.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true From 5446e2b3b01bb537c10cd31b0e24546a0262fb47 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 7 May 2026 00:56:26 +0100 Subject: [PATCH 286/287] update CodeQL to v3 --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 77e99e8a..6be6a0fb 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From 6749e8da0d6c2f2454556ea8fda36ba77c32b7da Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 7 May 2026 01:00:37 +0100 Subject: [PATCH 287/287] bump codeQL to v4 --- .github/workflows/codeql-analysis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6be6a0fb..bc121630 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,11 +39,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4