From d8dee37c525a59d4d69d668f80241e6420f7aff7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 1 Aug 2025 14:13:19 +0100 Subject: [PATCH 01/83] bugfix to handle older pickles safely --- .../ukmon_pylib/maintenance/recreateOrbitPages.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py index cb74b016..4b2f8fdf 100644 --- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py +++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py @@ -149,8 +149,12 @@ def recreateOrbitFiles(outdir, pickname, doupload=False): print('created additional output') basename = pickname[:15] repname = basename + '_report.txt' - traj.saveReport(outdir, repname, None, False) - traj.savePlots(outdir, basename, show_plots=False, ret_figs=False) + try: + traj.saveReport(outdir, repname, None, False) + traj.savePlots(outdir, basename, show_plots=False, ret_figs=False) + except: + print('unable to process pickle properly') + pass print('created reports and figures') orbparent, orbfldr = os.path.split(outdir) yr = orbfldr[:4] @@ -161,7 +165,12 @@ def recreateOrbitFiles(outdir, pickname, doupload=False): availableimages = getImgList(outdir, traj) createExtraJpgtxt(outdir, traj, availableimages) createExtraJpgHtml(outdir, orbparent, yr, ym) - fixupTrajComments(traj, availableimages, outdir, pickname) + try: + fixupTrajComments(traj, availableimages, outdir, pickname) + except: + print('unable to fixup traj comments') + pass + if doupload: files = os.listdir(outdir) From 9f5f7f85063e34b9ea98769d70d6d70c7dd51e0d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 9 Aug 2025 17:13:16 +0100 Subject: [PATCH 02/83] bugfix in getliveimages, parsing date wrongly --- archive/samfunctions/getLiveImages/getLiveImages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/samfunctions/getLiveImages/getLiveImages.py b/archive/samfunctions/getLiveImages/getLiveImages.py index 16202cd1..164ecc60 100644 --- a/archive/samfunctions/getLiveImages/getLiveImages.py +++ b/archive/samfunctions/getLiveImages/getLiveImages.py @@ -53,7 +53,7 @@ def getTrueImgs(dtstr, dtstr2, statid, ddb=None): if d1.hour < 13: capnight=(d1 + datetime.timedelta(days=-1)).strftime('%Y%m%d') else: - capnight = dtstr[:8] + capnight = d1.strftime('%Y%m%d') lv=Decimal(f'{d1.timestamp():.0f}') hv=Decimal(f'{d2.timestamp():.0f}') resp = table.query(KeyConditionExpression=Key('CaptureNight').eq(int(capnight)) & Key('Timestamp').between(lv, hv), From 82ff88db7cf3f6429553c3a534efe81f885ff622 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 9 Aug 2025 17:13:41 +0100 Subject: [PATCH 03/83] bugfix in cameradetails - needs to only look at first 6 chars --- archive/ukmon_pylib/reports/CameraDetails.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py index 49e5680e..74ce7ac8 100644 --- a/archive/ukmon_pylib/reports/CameraDetails.py +++ b/archive/ukmon_pylib/reports/CameraDetails.py @@ -126,7 +126,7 @@ def createStatOptsHtml(sitedets, datadir, active=False, locs=False): def findSite(stationid, camdets=None, ddb=None): if camdets is None: camdets = loadLocationDetails(ddb=ddb) - cc = camdets[camdets.stationid==stationid] + cc = camdets[camdets.stationid==stationid[:6]] if len(cc) > 0: res = cc.iloc[0]['site'] else: @@ -137,7 +137,7 @@ def findSite(stationid, camdets=None, ddb=None): def findEmail(stationid, camdets=None, ddb=None): if camdets is None: camdets = loadLocationDetails(ddb=ddb) - cc = camdets[camdets.stationid==stationid] + cc = camdets[camdets.stationid==stationid[:6]] if len(cc) > 0: res = cc.iloc[0]['eMail'] else: From 0dc3923b2b5693169b952577e51fb5d216153984 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 9 Aug 2025 17:14:02 +0100 Subject: [PATCH 04/83] moved from MeteorTools --- utils/addVidorImg.ps1 | 49 ++++++++++++++++++++++++++++++ utils/uploadOrbit.ps1 | 70 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 utils/addVidorImg.ps1 create mode 100644 utils/uploadOrbit.ps1 diff --git a/utils/addVidorImg.ps1 b/utils/addVidorImg.ps1 new file mode 100644 index 00000000..a3bc5757 --- /dev/null +++ b/utils/addVidorImg.ps1 @@ -0,0 +1,49 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +# powershell script to upload an extra video or image of an event, given a trajectory ID + +$loc = get-location +if ($args.count -lt 1) { + write-output "Usage: addVideoorImg.ps1 trajpath" + write-output " First create a folder for the trajectory eg 20241113_192754.345_UK" + write-output " Copy any additional mp4, mov or jpgs into it, then run this script" + exit +}else { + $pth = $args[0] +} +set-location $PSScriptRoot +# load the helper functions +. .\helperfunctions.ps1 +$ini=get-inicontent analysis.ini +$fbfolder = $ini['localdata']['fbfolder'] +set-location $fbfolder +$traj = (split-path -leaf $pth) + +$yr=$traj.Substring(0,4) +$ym=$traj.Substring(0,6) +$ymd=$traj.Substring(0,8) +$pick=$traj.substring(0,15) + '_trajectory.pickle' +$zipf=$traj.substring(0,15) + '.zip' + +wget https://archive.ukmeteors.co.uk/reports/${yr}/orbits/${ym}/${ymd}/${traj}/${pick} -O ${pth}\${pick} + +if ((test-path $pth\jpgs) -eq $false) { mkdir $pth\jpgs } +if ((test-path $pth\mp4s) -eq $false) { mkdir $pth\mp4s } + +move-item $pth\*.jpg $pth\jpgs +move-item $pth\*.mp4 $pth\mp4s + +compress-archive -path $pth\* -DestinationPath $pth\$zipf -force + +$apikey=(Get-Content ~/.ssh/fbuploadkey.txt) +$headers = @{ + 'Content-type' = 'application/zip' + 'Slug' = ${zipf} + 'apikey'= ${apikey} +} +$url = "https://api.ukmeteors.co.uk/fireballfiles?orbitfile=${zipf}" + +invoke-webrequest -uri $url -infile $pth\$zipf -Method PUT -Headers $headers + +Write-Output "uploaded $zipf at $(get-date)" +set-location $loc diff --git a/utils/uploadOrbit.ps1 b/utils/uploadOrbit.ps1 new file mode 100644 index 00000000..e470d096 --- /dev/null +++ b/utils/uploadOrbit.ps1 @@ -0,0 +1,70 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +# powershell script to create a zip file of a solution and upload it + +$loc = get-location +if ($args.count -lt 1) { + $curdir = get-location +}else { + $curdir = resolve-path $args[0] +} +set-location $PSScriptRoot +# load the helper functions +. .\helperfunctions.ps1 +$ini=get-inicontent analysis.ini +Set-Location $Loc + +Add-Type -AssemblyName System.Windows.Forms + +conda activate $ini['wmpl']['wmpl_env'] +$wmplloc = $ini['wmpl']['wmpl_loc'] +$env:pythonpath="$wmplloc;$env:pythonpath" + +write-output "Find the pickle" +$picklefile= (Get-ChildItem $curdir\*.pickle -r).fullname +if ($picklefile -is [Array] -or $picklefile.length -eq 0) { + $FileBrowser = New-Object System.Windows.Forms.OpenFileDialog -Property @{ InitialDirectory = $curdir; Filter = 'pickles (*.pickle)|*.pickle'; Title='Select orbit pickle' } + $null = $FileBrowser.ShowDialog() + $picklefile = $filebrowser.filename + if ( $picklefile -eq "" ) { + Write-Output "Cancelled" + set-location $loc + exit + } +} +write-output "find the orbit full name" +$srcdir = ((get-item $picklefile).directoryname).replace('\','/') +$pickname = (get-item $picklefile).name +$origdir = (python -c "from wmpl.Utils.Pickling import loadPickle;pick = loadPickle('${srcdir}','${pickname}');print(pick.output_dir)") +$orbname = split-path $origdir -leaf +if ((test-path $curdir\$orbname) -eq 0 ) +{ + mkdir -force $curdir\$orbname + move-item $picklefile $curdir\$orbname\ +} +$picklefile = (Get-ChildItem $picklefile -r).fullname + +write-output "look for jpgs and mp4s" +mkdir -force $curdir\jpgs +$jpgdir = resolve-path $curdir\jpgs +Move-Item $curdir\*.jpg $jpgdir +mkdir -force $curdir\mp4s +$mp4dir = resolve-path $curdir\mp4s +Move-Item $curdir\*.mp4 $mp4dir + +write-output "Creating zip file" +if ((test-path $env:temp\$orbname\mp4s) -eq 0 ) {mkdir $env:temp\$orbname\mp4s | out-null} +if ((test-path $env:temp\$orbname\jpgs) -eq 0 ) {mkdir $env:temp\$orbname\jpgs | out-null} +copy-item $picklefile $env:temp\$orbname +copy-item $jpgdir\*.jpg $env:temp\$orbname\jpgs +copy-item $mp4dir\*.mp4 $env:temp\$orbname\mp4s +if ((test-path $env:temp\$orbname.zip) -eq 1 ) { remove-item $env:temp\$orbname.zip} +compress-archive -destinationpath "$env:temp\$orbname.zip" -path "$env:temp\$orbname\*" +remove-item $env:temp\$orbname\* -Recurse + +write-output "Uploading $orbname.zip" +curl -X PUT -H "Content-Type:application/zip" --data-binary "@$env:temp\$orbname.zip" "https://api.ukmeteors.co.uk/fireballfiles?orbitfile=$orbname.zip" +move-item $env:temp\$orbname.zip $curdir -Force +Write-Output "uploaded at $(get-date)" +set-location $loc +conda deactivate From 7b74face48fcb6cb5126e3c9a15201578312226e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 9 Aug 2025 17:14:25 +0100 Subject: [PATCH 05/83] add scripts to get brightes N matches or singles for social media --- archive/ukmon_pylib/reports/findBestMp4s.py | 72 +++++++++++++++++++++ utils/getBrightest.ps1 | 33 ++++++++++ 2 files changed, 105 insertions(+) create mode 100644 utils/getBrightest.ps1 diff --git a/archive/ukmon_pylib/reports/findBestMp4s.py b/archive/ukmon_pylib/reports/findBestMp4s.py index bd0af136..a07611dd 100644 --- a/archive/ukmon_pylib/reports/findBestMp4s.py +++ b/archive/ukmon_pylib/reports/findBestMp4s.py @@ -5,9 +5,81 @@ import shutil import sys import os +import datetime +import requests from traj.pickleAnalyser import getAllMp4s +def getBestNMatches(reqdate=None, numtoget=10): + if reqdate is None: + tod = datetime.datetime.now() + tod = tod.replace(hour=12, minute=0, second=0, microsecond=0) + reqdate = tod + datetime.timedelta(days=-1) + else: + reqdate = reqdate.replace(hour=12, minute=0, second=0, microsecond=0) + tod = reqdate + datetime.timedelta(days=1) + + yr = reqdate.year + datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') + mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') + + # select only the columns we need + cols=['_localtime', '_mag','url'] + matches = pd.read_parquet(mf, columns=cols) + matches['dt'] = [datetime.datetime.strptime(x,'_%Y%m%d_%H%M%S') for x in matches._localtime] + + matches = matches[matches.dt >= reqdate] + matches = matches[matches.dt <= tod] + sepdata = matches.sort_values(by=['_mag']) + sorteddata = sepdata.head(numtoget) + sorteddata.drop_columns(['_localtime'], inplace=True) + + +def getBestNSingles(reqdate=None, numtoget=20, shwr=None, outdir=None): + if reqdate is None: + tod = datetime.datetime.now() + tod = tod.replace(hour=12, minute=0, second=0, microsecond=0) + reqdate = tod + datetime.timedelta(days=-1) + else: + print(reqdate) + reqdate = datetime.datetime.strptime(reqdate, '%Y%m%d') + reqdate = reqdate.replace(hour=12, minute=0, second=0, microsecond=0) + tod = reqdate + datetime.timedelta(days=1) + yr = reqdate.year + url = f'https://archive.ukmeteors.co.uk/browse/parquet/singles-{yr}.parquet.snap' + + # select only the columns we need + cols=['Mag','Shwr','Filename','Dtstamp'] + matches = pd.read_parquet(url, columns=cols) + + matches = matches[matches.Dtstamp >= reqdate.timestamp()] + matches = matches[matches.Dtstamp <= tod.timestamp()] + sepdata = matches.sort_values(by=['Mag']) + sepdata['url'] = [getUrlFromFilename(x) for x in sepdata.Filename] + if shwr: + sepdata = sepdata[sepdata.Shwr==shwr] + sorteddata = sepdata.head(numtoget) + if outdir: + outdir = os.path.join(outdir, reqdate.strftime('%Y%m%d')) + os.makedirs(outdir, exist_ok=True) + for _, rw in sorteddata.iterrows(): + fname = (f'{rw.Shwr}_{rw.Mag}_{rw.Filename}').replace('.fits','.jpg') + print(fname) + res = requests.get(rw.url) + if res.status_code == 200: + open(os.path.join(outdir, fname),'wb').write(res.content) + + filtereddata = sorteddata.drop(columns=['Dtstamp','Filename']) + return filtereddata + + +def getUrlFromFilename(fname): + ymd = fname.split('_')[2] + jpgname = fname.replace('.fits','.jpg') + url = f'https://archive.ukmeteors.co.uk/img/single/{ymd[:4]}/{ymd[:6]}/{jpgname}' + return url + + def getBestNMp4s(yr, mth, numtoget): datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap') diff --git a/utils/getBrightest.ps1 b/utils/getBrightest.ps1 new file mode 100644 index 00000000..a40442b2 --- /dev/null +++ b/utils/getBrightest.ps1 @@ -0,0 +1,33 @@ +# powershell script to get the 30 best detections from the previous night +# for sharing on social media +# note that you will have to go through and delete non-meteors from the images + +# requirements - you must clone WesternMeteorPyLib and ukmda-dataprocessing from github +# to your local development space. I use onedrive\dev for my code - set this location in $codeloc + +# you also need to specify the output folder, $outdir + +# copyright (c) Mark McIntyre, 2025- + +$codeloc ="$env:userprofile\onedrive\dev" +$outdirw = "$env:userprofile\pictures\_ToBeProcessed\brightest" + +$repdir = "$codeloc\ukmda-dataprocessing\archive\ukmon_pylib" +$outdir = $outdirw.replace('\','/') + +$env:pythonpath="$codeloc\WesternMeteorPyLib" +Push-Location $repdir + +conda activate ukmon-shared + +Write-Output "working..." +if ($args.count -eq 0) { + python -c "from reports.findBestMp4s import getBestNSingles;getBestNSingles(numtoget=30,outdir='$outdir')" +}else{ + $reqdate = $args[0] + python -c "from reports.findBestMp4s import getBestNSingles;getBestNSingles(numtoget=30,outdir='$outdir', reqdate='$reqdate')" +} + +explorer "$outdirw" + +Pop-Location \ No newline at end of file From 020da9fbd3bd14193bf1ac78e187ca3218ca4933 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 11 Aug 2025 17:17:15 +0100 Subject: [PATCH 06/83] Adding fbcollector to ukmda repo --- fbCollector/download_events.sh | 67 +++ fbCollector/fbCollector.ps1 | 24 + fbCollector/fbcollector.yml | 22 + fbCollector/fireballCollector.py | 985 +++++++++++++++++++++++++++++++ fbCollector/helperfunctions.ps1 | 107 ++++ fbCollector/noimage.jpg | Bin 0 -> 70608 bytes fbCollector/requirements.txt | 8 + fbCollector/ukmda.ico | Bin 0 -> 16958 bytes fbCollector/vars.yml | 2 + 9 files changed, 1215 insertions(+) create mode 100644 fbCollector/download_events.sh create mode 100644 fbCollector/fbCollector.ps1 create mode 100644 fbCollector/fbcollector.yml create mode 100644 fbCollector/fireballCollector.py create mode 100644 fbCollector/helperfunctions.ps1 create mode 100644 fbCollector/noimage.jpg create mode 100644 fbCollector/requirements.txt create mode 100644 fbCollector/ukmda.ico create mode 100644 fbCollector/vars.yml diff --git a/fbCollector/download_events.sh b/fbCollector/download_events.sh new file mode 100644 index 00000000..7c47e3f4 --- /dev/null +++ b/fbCollector/download_events.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +dt=$1 +basedir=$(grep basedir $here/config.ini | awk -F= '{print $2}'|tr -d '\r' | sed 's|f:|/mnt/f|g' | sed 's|c:|/mnt/c|g') +echo $basedir +sleep 30 +dtns=$dt +mkdir -p $basedir/$dtns +cd $basedir/$dtns + +rsync -avz gmn.uwo.ca:/home/uk*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns +rsync -avz gmn.uwo.ca:/home/be*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns +rsync -avz gmn.uwo.ca:/home/ie*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns +rsync -avz gmn.uwo.ca:/home/nl*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns +if [ "$2" == "all" ] ; then + rsync -avz gmn.uwo.ca:/home/fr*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns + rsync -avz gmn.uwo.ca:/home/de*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns + rsync -avz gmn.uwo.ca:/home/es*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns + rsync -avz gmn.uwo.ca:/home/ch*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns + rsync -avz gmn.uwo.ca:/home/it*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns + rsync -avz gmn.uwo.ca:/home/cz*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns + rsync -avz gmn.uwo.ca:/home/hr*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns + rsync -avz gmn.uwo.ca:/home/sk*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns +fi + +for f in *.bz2 ; do tar -xvf $f ; done +if [ -d UKMON ] ; then + sites=$(ls -1 UKMON) + for site in $sites ; do + cams=$(ls -1 UKMON/$site) + for cam in $cams ; do + if [ -d ./$cam ] ; then rm -Rf $cam ; fi + mv -f UKMON/$site/$cam . + done + done + rm -Rf UKMON +fi +if [ -d NEMETODE ] ; then + sites=$(ls -1 NEMETODE) + for site in $sites ; do + cams=$(ls -1 NEMETODE/$site) + for cam in $cams ; do + if [ -d ./$cam ] ; then rm -Rf $cam ; fi + mv -f NEMETODE/$site/$cam . + done + done + rm -Rf NEMETODE +fi +if [ -d "UKMON,NEMETODE" ] ; then + sites=$(ls -1 "UKMON,NEMETODE") + for site in $sites ; do + cams=$(ls -1 "UKMON,NEMETODE/$site") + for cam in $cams ; do + if [ -d ./$cam ] ; then rm -Rf $cam ; fi + mv -f "UKMON,NEMETODE/$site/$cam" . + done + done + rm -Rf "UKMON,NEMETODE" +fi +mkdir -p ./stacks +mkdir -p ./jpgs +mkdir -p ./mp4s +cp -f */*.jpg ./jpgs +mv -f ./jpgs/*captured_stack* ./stacks +cp -f */*.mp4 ./mp4s diff --git a/fbCollector/fbCollector.ps1 b/fbCollector/fbCollector.ps1 new file mode 100644 index 00000000..1220f741 --- /dev/null +++ b/fbCollector/fbCollector.ps1 @@ -0,0 +1,24 @@ +# +# powershell script to launch the fireball data collector tool +# Copyright (C) 2018-2023 Mark McIntyre +# +push-location $PSScriptRoot + +. .\helperfunctions.ps1 +$ini=get-inicontent .\config.ini + +conda activate $ini['solver']['wmpl_env'] +$wmplloc = $ini['solver']['wmpl_loc'] +$wmplloc = $wmplloc.replace('$HOME',$env:userprofile) + +$env:pythonpath="$wmplloc" + +if ($args.count -lt 1) { + python fireballCollector.py +}else { + python fireballCollector.py -d $args[0] +} + +Pop-Location + + diff --git a/fbCollector/fbcollector.yml b/fbCollector/fbcollector.yml new file mode 100644 index 00000000..11707f86 --- /dev/null +++ b/fbCollector/fbcollector.yml @@ -0,0 +1,22 @@ +--- +- hosts: ukmonhelper2 + gather_facts: no + tasks: + - name: import variables + include_vars: ./vars.yml + tags: [dev,prod] + - name: Ensures {{destdir}} exists + file: path={{destdir}} state=directory + delegate_to: localhost + tags: [dev,prod] + - name: Copy files + copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} + tags: [dev,prod] + delegate_to: localhost + with_items: + - {src: '{{srcdir}}/fbCollector.ps1', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/fireballCollector.py', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/download_events.sh', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/config.ini', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/noimage.jpg', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/ukmda.ico', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py new file mode 100644 index 00000000..649b247c --- /dev/null +++ b/fbCollector/fireballCollector.py @@ -0,0 +1,985 @@ +# +# UI to manage fireball data collection +# Copyright (C) 2018-2023 Mark McIntyre +# +import os +import sys +import argparse +import logging +import logging.handlers +import datetime +import configparser +import shutil +import platform +import subprocess +import glob +import xmltodict +from PIL import Image +import requests + +import boto3 +import paramiko +from scp import SCPClient + +from meteortools.ukmondb import getECSVs as getecsv +from wmpl.Formats.ECSV import loadECSVs +from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric + +import tkinter as tk +import tkinter.filedialog as tkFileDialog +import tkinter.messagebox as tkMessageBox +from tkinter.simpledialog import askstring +from tkinter import StringVar, Frame, ACTIVE, END, Listbox, Menu, Entry, Button +from tkinter.ttk import Label, Style, LabelFrame, Scrollbar + +from PIL import Image as img +from PIL import ImageTk + +from meteortools.ukmondb import getLiveJpgs, createTxtFile + + +config_file = '' +noimg_file = '' +global_bg = "Black" +global_fg = "Gray" + + +def quitApp(): + # Cleanly exits the app + root.quit() + root.destroy() + + +def log_timestamp(): + """ Returns timestamp for logging. + """ + return datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + + +class StyledButton(Button): + """ Button with style. + """ + def __init__(self, *args, **kwargs): + Button.__init__(self, *args, **kwargs) + + self.configure(foreground = global_fg, background = global_bg, borderwidth = 3) + + +class StyledEntry(Entry): + """ Entry box with style. + """ + def __init__(self, *args, **kwargs): + Entry.__init__(self, *args, **kwargs) + + self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg, disabledbackground = global_bg, disabledforeground = "DimGray") + + +class ConstrainedEntry(StyledEntry): + """ Entry box with constrained values which can be input (e.g. 0-255). + """ + def __init__(self, *args, **kwargs): + StyledEntry.__init__(self, *args, **kwargs) + self.maxvalue = 255 + vcmd = (self.register(self.on_validate), "%P") + self.configure(validate="key", validatecommand=vcmd) + # self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg) + + def disallow(self): + """ Pings a bell on values which are out of bound. + """ + self.bell() + + def update_value(self, maxvalue): + """ Updates values in the entry box. + """ + self.maxvalue = maxvalue + vcmd = (self.register(self.on_validate), "%P") + self.configure(validate="key", validatecommand=vcmd) + + def on_validate(self, new_value): + """ Checks if entered value is within bounds. + """ + try: + if new_value.strip() == "": + return True + value = int(new_value) + if value < 0 or value > self.maxvalue: + self.disallow() + return False + except ValueError: + self.disallow() + return False + + return True + + +class fbCollector(Frame): + + def __init__(self, parent, patt=None): + Frame.__init__(self, parent, bg = global_bg) + parent.configure(bg = global_bg) # Set backgound color + parent.grid_columnconfigure(0, weight=1) + parent.grid_rowconfigure(0, weight=1) + + self.grid(sticky="NSEW") # Expand frame to all directions + self.parent = parent + + self.fb_dir = '' + self.upload_bucket = '' + self.upload_folder = '' + self.live_bucket = '' + self.gmn_key = '' + self.gmn_user = '' + self.gmn_server = '' + self.wmpl_loc = '' + self.wmpl_env = '' + self.rms_loc = '' + self.rms_env = '' + self.selected = {} + self.evtMonTriggered = None + self.review_stack = False + self.soln_outputdir = None + self.log_files_to_keep = 30 + + self.readConfig() + + self.patt = patt + if patt is None: + self.dir_path = self.fb_dir.strip() + else: + self.dir_path = os.path.join(self.fb_dir, patt) + log.info(f"Fireball folder is {self.fb_dir}") + + self.initUI() + if self.dir_path != self.fb_dir: + bin_list = self.get_bin_list() + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + + # Update UI changes + parent.update_idletasks() + parent.update() + + return + + def readConfig(self): + localcfg = configparser.ConfigParser() + localcfg.read(config_file) + self.fb_dir = os.path.expanduser(localcfg['Fireballs']['basedir'].replace('$HOME','~')).replace('\\','/') + self.upload_bucket = localcfg['Fireballs']['uploadbucket'] + self.upload_folder = localcfg['Fireballs']['uploadfolder'] + self.live_bucket = localcfg['Fireballs']['livebucket'] + os.makedirs(self.fb_dir, exist_ok=True) + + try: + self.gmn_key = localcfg['gmnconnection']['gmnkey'] + self.gmn_user = localcfg['gmnconnection']['gmnuser'] + self.gmn_server = localcfg['gmnconnection']['gmnserver'] + except: + pass + + self.wmpl_loc = os.path.expanduser(localcfg['solver']['wmpl_loc'].replace('$HOME','~')).replace('\\','/') + self.wmpl_env= localcfg['solver']['wmpl_env'] + self.rms_loc = os.path.expanduser(localcfg['reduction']['rms_loc'].replace('$HOME','~')).replace('\\','/') + self.rms_env = localcfg['reduction']['rms_env'] + + self.shareloc = os.path.expanduser(localcfg['sharing']['shrfldr'].replace('$HOME','~')).replace('\\','/') + return + + def quitApplication(self): + print('quitting') + logdir = os.path.join(os.getenv('TMP'), 'fbcollector') + logfiles = os.listdir(logdir) + numtokeep = self.log_files_to_keep + if len(logfiles) > numtokeep: + logfiles.sort() + for i in range(len(logfiles)-numtokeep): + try: + os.remove(os.path.join(logdir, logfiles[i])) + except: + pass + quitApp() + + def initUI(self): + """ Initialize GUI elements. + """ + + self.parent.title("Fireball Downloader") + + # Configure the style of each element + s = Style() + s.configure("TButton", padding=(0, 5, 0, 5), font='serif 10') + s.configure('TLabelframe.Label', foreground=global_fg, background=global_bg) + s.configure('TLabelframe', foreground =global_fg, background=global_bg, padding=(3, 3, 3, 3)) + s.configure("TRadiobutton", foreground = global_fg, background = global_bg) + s.configure("TLabel", foreground = global_fg, background = global_bg) + s.configure("TCheckbutton", foreground = global_fg, background = global_bg) + s.configure("Vertical.TScrollbar", background=global_bg, troughcolor = global_bg) + + self.columnconfigure(0, pad=3) + self.columnconfigure(1, pad=3) + self.columnconfigure(2, pad=3) + self.columnconfigure(3, pad=3) + self.columnconfigure(4, pad=3) + self.columnconfigure(5, pad=3) + self.columnconfigure(6, pad=3) + self.columnconfigure(7, pad=3) + self.columnconfigure(8, pad=3) + + self.rowconfigure(0, pad=3) + self.rowconfigure(1, pad=3) + self.rowconfigure(2, pad=3) + self.rowconfigure(3, pad=3) + self.rowconfigure(4, pad=3) + self.rowconfigure(5, pad=3) + self.rowconfigure(6, pad=3) + self.rowconfigure(7, pad=3) + self.rowconfigure(8, pad=3) + self.rowconfigure(9, pad=3) + self.rowconfigure(10, pad=3) + + # Make menu + self.menuBar = Menu(self.parent) + self.parent.config(menu=self.menuBar) + + # File menu + fileMenu = Menu(self.menuBar, tearoff=0) + fileMenu.add_command(label="Load Folder", command=self.loadFolder) + fileMenu.add_command(label="Archive This Folder", command=self.archiveFolder) + fileMenu.add_command(label="Open in Explorer", command=self.openFolder) + fileMenu.add_separator() + fileMenu.add_command(label="Delete This Folder", command=self.delFolder) + fileMenu.add_separator() + fileMenu.add_command(label="Configuration", command=self.showConfig) + fileMenu.add_command(label="View Logs", command=self.viewLogs) + fileMenu.add_separator() + fileMenu.add_command(label="Exit", command=self.quitApplication) + self.menuBar.add_cascade(label="File", underline=0, menu=fileMenu) + + rawMenu = Menu(self.menuBar, tearoff=0) + rawMenu.add_command(label="Get Live Images", command=self.get_data) + rawMenu.add_command(label="Get Videos", command=self.get_vids) + rawMenu.add_command(label="Get Traj Pickle", command=self.get_trajpickle) + rawMenu.add_separator() + rawMenu.add_command(label="Get GMN Raw Data", command=self.getGMNData) + rawMenu.add_separator() + rawMenu.add_command(label="Get ECSVs", command=self.getECSVs) + self.menuBar.add_cascade(label="Raw", underline=0, menu=rawMenu) + + watchMenu = Menu(self.menuBar, tearoff=0) + watchMenu.add_command(label="Get Watchlist", command=self.getWatchlist) + watchMenu.add_command(label="View Watchlist", command=self.viewWatchlist) + watchMenu.add_command(label="Upload Watchlist", command=self.putWatchlist) + watchMenu.add_separator() + watchMenu.add_command(label="Fetch Event Data", command=self.getEventData) + self.menuBar.add_cascade(label="Watchlist", underline=0, menu=watchMenu) + + revMenu = Menu(self.menuBar, tearoff=0) + revMenu.add_command(label="Review Stacks", command=self.checkStacks) + revMenu.add_command(label="Clean Folder", command=self.clean_folder) + self.menuBar.add_cascade(label="Review", underline=0, menu=revMenu) + + solveMenu = Menu(self.menuBar, tearoff=0) + solveMenu.add_command(label="Reduce Data", command=self.reduceCamera) + solveMenu.add_command(label="Toggle Ignore", command=self.ignoreCamera) + solveMenu.add_separator() + solveMenu.add_command(label="View Raw Data", command=self.viewData) + solveMenu.add_command(label="Upload Raw Data", command=self.uploadRaw) + solveMenu.add_separator() + solveMenu.add_command(label="Solve", command=self.solveOrbit) + solveMenu.add_separator() + solveMenu.add_command(label="View Solution", command=self.viewSolution) + solveMenu.add_command(label="Delete Solution", command=self.removeSolution) + solveMenu.add_separator() + solveMenu.add_command(label="Upload Orbit", command=self.uploadOrbit) + self.menuBar.add_cascade(label="Solve", underline=0, menu=solveMenu) + # buttons + self.save_panel = LabelFrame(self, text=' Image Selection ') + self.save_panel.grid(row = 1, columnspan = 2, sticky='WE') + + self.newpatt = StringVar() + self.newpatt.set(self.patt) + + self.patt_entry = StyledEntry(self.save_panel, textvariable = self.newpatt, width = 20) + self.patt_entry.grid(row = 1, column = 1, columnspan = 2, sticky = "W") + save_bmp = StyledButton(self.save_panel, text="Get Images", width = 8, command = lambda: self.get_data()) + save_bmp.grid(row = 1, column = 3) + + save_bmp = StyledButton(self.save_panel, text="Remove", width = 8, command = lambda: self.remove_image()) + save_bmp.grid(row = 1, column = 4) + + + # Listbox + self.scrollbar = Scrollbar(self) + self.listbox = Listbox(self, width = 47, yscrollcommand=self.scrollbar.set, exportselection=0, activestyle = "none", bg = global_bg, fg = global_fg) + self.listbox.config(height = 25) # Listbox size + self.listbox.grid(row = 4, column = 0, rowspan = 7, columnspan = 2, sticky = "NS") # Listbox position + self.scrollbar.grid(row = 4, column = 2, rowspan = 7, sticky = "NS") # Scrollbar size + + self.listbox.bind('<>', self.update_image) + self.scrollbar.config(command = self.listbox.yview) + + # IMAGE + try: + # Show the TV test card image on program start + #noimage_data = img.open('noimage.bin').resize((640,360)) + noimgdata = img.open(noimg_file).resize((640,360)) + noimage = ImageTk.PhotoImage(noimgdata) + except: + noimage = None + + self.imagelabel = Label(self, image = noimage) + self.imagelabel.image = noimage + self.imagelabel.grid(row=3, column=3, rowspan = 4, columnspan = 3) + + # Timestamp label + self.timestamp_label = Label(self, text = "CCNNNN YYYY-MM-DD HH:MM.SS.mms", font=("Courier", 12)) + self.timestamp_label.grid(row = 7, column = 3, sticky = "E") + + def reduceCamera(self): + current_image = self.listbox.get(ACTIVE) + if current_image == '': + return + camid = current_image[3:9] + print('selected camera is', camid) + dirname = os.path.join(self.dir_path, camid) + tmpscr = os.path.join(os.getenv('TEMP'), 'reduce.ps1') + with open(tmpscr, 'w') as outf: + outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {dirname} -c {dirname}/.config\n') + _ = subprocess.run(['powershell.exe', tmpscr]) + frs = glob.glob(os.path.join(dirname, 'FR*.bin')) + if len(frs) > 0: + if not tkMessageBox.askyesno("Rerun", f'{len(frs)} FR files detected - rerun?'): + return + for fr in frs: + with open(tmpscr, 'w') as outf: + outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {fr} -c {dirname}/.config\n') + _ = subprocess.run(['powershell.exe', tmpscr]) + try: + os.remove(tmpscr) + except: + pass + return + + def ignoreCamera(self): + current_image = self.listbox.get(ACTIVE) + if current_image == '': + return + camid = current_image[3:9] + log.info(f'selected camera is {camid}') + dirname = os.path.join(self.dir_path, camid) + allecsvs = glob.glob(os.path.join(dirname, '*.ecsv')) + ecsvs = [e for e in allecsvs if 'REJECT' not in e.upper()] + rejs = [e for e in allecsvs if 'REJECT' in e.upper()] + if len(ecsvs) == 0 and len(rejs) == 0: + log.info('no files to reject or include') + return + elif len(ecsvs) == 1 and len(rejs) == 0: + os.rename(ecsvs[0], ecsvs[0].replace('.ecsv','_REJECT.ecsv')) + jpgname = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) + os.rename(jpgname[0], jpgname[0].replace('.jpg','_REJECT.jpg')) + elif len(ecsvs) == 0 and len(rejs) == 1: + os.rename(rejs[0], rejs[0].replace('_REJECT.ecsv','.ecsv')) + jpgname = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) + os.rename(jpgname[0], jpgname[0].replace('_REJECT.jpg','.jpg')) + else: + # more than one ECSV or REJ file to handle, urk + log.info('urk') + return + bin_list = [line for line in os.listdir(os.path.join(self.dir_path, 'jpgs')) if self.correct_datafile_name(line)] + #print(bin_list) + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + return + + def solveOrbit(self): + log.info('Using ECSV files:') + ecsv_names = [] + ecsv_paths = [] + for entry in sorted(os.walk(self.dir_path), key=lambda x: x[0]): + dir_name, _, file_names = entry + for fn in file_names: + if fn.lower().endswith(".ecsv") and 'REJECT' not in dir_name.upper() and 'REJECT' not in fn.upper(): + + # Add ECSV file, but skip duplicates + if fn not in ecsv_names: + ecsv_paths.append(os.path.join(dir_name, fn)) + ecsv_names.append(fn) + log.info(fn) + if len(ecsv_paths) < 2: + tkMessageBox.showinfo('Warning', 'Need at least two ECSV files') + return + jdt_ref, meteor_list = loadECSVs(ecsv_paths) + mcruns = 20 + max_toffset = 15.0 + velpart = None + vinitht = None + plotallspatial = True + uncertgeom = False + jacchia = False + traj = solveTrajectoryGeneric(jdt_ref, meteor_list, self.dir_path, + max_toffset=max_toffset, monte_carlo=True, mc_runs=mcruns, + geometric_uncert=uncertgeom, plot_all_spatial_residuals=plotallspatial, + show_plots=False, v_init_part=velpart, v_init_ht=vinitht, + show_jacchia=jacchia, enable_OSM_plot=True, mc_cores=8) + tkMessageBox.showinfo('Info', 'Solver Finished') + self.soln_outputdir = traj.output_dir + return + + def viewSolution(self): + self.review_stack = False + if not self.soln_outputdir: + solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') + if len(solndir) == 0: + return + solndir = os.path.join(self.dir_path, solndir[0]) + self.soln_outputdir = solndir + bin_list = [line for line in os.listdir(self.soln_outputdir) if self.correct_datafile_name(line)] + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + return + + def removeSolution(self): + if not self.soln_outputdir: + solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') + if len(solndir) == 0: + return + solndir = os.path.join(self.dir_path, solndir[0]) + self.soln_outputdir = solndir + if not tkMessageBox.askyesno("Delete file", f"delete {self.soln_outputdir}?"): + return + shutil.rmtree(os.path.join(self.dir_path, self.soln_outputdir)) + return + + def uploadOrbit(self): + pickles=[] + for path, _, files in os.walk(self.dir_path): + for name in files: + if '.pickle' in name and '_mc_' not in name: + pickles.append(os.path.join(path, name)) + + if len(pickles) == 0: + return + elif len(pickles) == 1: + pickfile = pickles[0] + else: + pickfile = tkFileDialog.askopenfilename(title='Select Orbit Pickle', defaultextension='*.pickle', + initialdir=self.dir_path, initialfile='*.pickle', + filetypes=[('pickles','*.pickle')]) + orbname = os.path.split(pickfile)[1] + tmpdir = os.path.join(self.dir_path, 'tmpzip') + os.makedirs(tmpdir, exist_ok=True) + shutil.copyfile(pickfile, os.path.join(tmpdir, orbname)) + if os.path.isdir(os.path.join(self.dir_path, 'jpgs')): + shutil.copytree(os.path.join(self.dir_path, 'jpgs'), os.path.join(tmpdir, 'jpgs')) + if os.path.isdir(os.path.join(self.dir_path, 'mp4s')): + shutil.copytree(os.path.join(self.dir_path, 'mp4s'), os.path.join(tmpdir, 'mp4s')) + for path, _, files in os.walk(self.dir_path): + for name in files: + if '_dyn_mass_fit' in name: + im = Image.open(os.path.join(path, name)).convert("RGB") + im.save(os.path.join(tmpdir,'jpgs', name[:-4] + '.jpg')) + break + zfname = os.path.join(self.dir_path, orbname[:15]) + shutil.make_archive(zfname,'zip',tmpdir) + try: + shutil.rmtree(tmpdir) + except Exception: + pass + apikey = open(os.path.expanduser('~/.ssh/fbuploadkey.txt')).readlines()[0].strip() + headers = {'Content-type': 'application/zip', 'Slug': orbname[:15], 'apikey': apikey} + url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbname[:15]}.zip' + r = requests.put(url, data=open(zfname+'.zip', 'rb'), headers=headers) #, auth=('username', 'pass')) + #print(r.text) + if r.status_code != 200: + tkMessageBox.showinfo('Warning', 'Problem with upload') + else: + tkMessageBox.showinfo('Info', 'Orbit Uploaded') + return + + def uploadRaw(self): + zfname = os.path.join(os.getenv('TMP'), os.path.basename(self.dir_path)) + log.info(f'zfname is {zfname}') + shutil.make_archive(zfname,'zip',self.dir_path) + try: + targname = os.path.join(self.shareloc, os.path.basename(zfname)+'.zip') + log.info(f'targname is {targname}') + shutil.copyfile(zfname+'.zip', targname) + tkMessageBox.showinfo('Info', 'Raw Data Uploaded to Dropbox') + subprocess.Popen(f'explorer "{self.shareloc}"') + except Exception: + tkMessageBox.showinfo('Warning', 'Problem with upload') + return + + def viewData(self): + dirpath = self.dir_path.replace('/', '\\') + log.info(f'self-dir-path {dirpath}') + subprocess.Popen(f'explorer "{dirpath}"') + return + + def getECSVs(self): + notgotlist=[] + img_list = self.get_bin_list() + for current_image in img_list: + if not self.getOneEcsv(current_image): + notgotlist.append(current_image) + if len(notgotlist) > 0: + tkMessageBox.showinfo('Info', f'No ECSVs for {notgotlist}') + return + + def getOneEcsv(self, current_image): + if current_image[:1] == 'M': + datestr = current_image[1:16] + statid = current_image[-11:-5] + elif current_image[:2] == 'FF': + statid = current_image[3:9] + datestr = current_image[10:29] + else: + return + #dtval = datetime.datetime.strptime(datestr, '%Y%m%d_%H%M%S') + #datestr = dtval.strftime('%Y-%m-%dT%H:%M:%S') + try: + lis = getecsv(statid, datestr, savefiles=True, outdir=os.path.join(self.dir_path, statid)) + for li in lis: + if 'issue getting data' in li: + return False + return True + except Exception: + return False + + def get_bin_list(self): + """ Get a list of image files in a given directory. + """ + if self.dir_path is None: + dirname = tkFileDialog.askdirectory(parent=root,initialdir=self.fb_dir, + title='Please select a directory') + _, thispatt = os.path.split(dirname) + self.dir_path = dirname + self.patt = thispatt + self.newpatt.set(self.patt) + + if self.review_stack is False: + targdir = os.path.join(self.dir_path, 'jpgs') + else: + targdir = os.path.join(self.dir_path, 'stacks') + if os.path.isdir(targdir): + bin_list = [line for line in os.listdir(targdir) if self.correct_datafile_name(line)] + else: + log.info('no jpgs available') + bin_list = [] + return bin_list + + def update_listbox(self, bin_list): + """ Updates the listbox with the current entries. + """ + self.listbox.delete(0, END) + for line in sorted(bin_list): + self.listbox.insert(END, line) + if line not in self.selected: + self.selected[line] = (0, '') + if self.selected[line][0] == 1: + self.listbox.itemconfig(END, fg = 'green') + + def checkStacks(self): + self.review_stack = True + bin_list = self.get_bin_list() + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + + + def loadFolder(self): + self.review_stack = False + self.dir_path = None + self.soln_outputdir = None + bin_list = self.get_bin_list() + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + + def openFolder(self): + dir_path = self.dir_path.replace("/","\\") + os.system(f'explorer.exe {dir_path}') + + def viewLogs(self): + logdir = os.path.join(os.getenv('TMP'), 'fbcollector') + os.system(f'explorer.exe {logdir}') + + def archiveFolder(self): + noimgdata = img.open(noimg_file).resize((640,360)) + noimage = ImageTk.PhotoImage(noimgdata) + self.imagelabel.configure(image = noimage) + self.imagelabel.image = noimage + if self.dir_path == self.fb_dir.strip(): + self.loadFolder() + if self.dir_path is not None and self.dir_path != self.fb_dir: + try: + _, fldr = os.path.split(os.path.normpath(self.dir_path)) + yr = fldr[:4] + archdir = os.path.join(self.fb_dir, yr) + os.makedirs(archdir, exist_ok=True) + zfname = os.path.join(archdir, fldr) + shutil.make_archive(zfname,'zip',self.fb_dir, fldr) + shutil.rmtree(self.dir_path) + self.dir_path = self.fb_dir + except Exception as e: + log.warning(f'unable to archive {self.dir_path}') + log.warning(e) + + def delFolder(self): + noimgdata = img.open(noimg_file).resize((640,360)) + noimage = ImageTk.PhotoImage(noimgdata) + self.imagelabel.configure(image = noimage) + self.imagelabel.image = noimage + if self.dir_path is not None and self.dir_path != self.fb_dir: + try: + shutil.rmtree(self.dir_path) + self.dir_path = self.fb_dir + except Exception as e: + log.warning(f'unable to remove {self.dir_path}') + log.warning(e) + + def correct_datafile_name(self, line): + if ('.jpg' in line or '.png' in line) and 'noimage' not in line: + return True + return False + + def removeRelated(self, imgname): + camid = imgname[:6] + mp4s = os.listdir(os.path.join(self.dir_path, 'mp4s')) + badones = [x for x in mp4s if camid in x] + for ba in badones: + os.remove(os.path.join(self.dir_path, 'mp4s', ba)) + jpgs = os.listdir(os.path.join(self.dir_path, 'jpgs')) + badones = [x for x in jpgs if camid in x] + for ba in badones: + os.remove(os.path.join(self.dir_path, 'jpgs', ba)) + fldrs = os.listdir(self.dir_path) + badones = [x for x in fldrs if camid in x] + for ba in badones: + try: + shutil.rmtree(os.path.join(self.dir_path, ba)) + except Exception: + os.remove(os.path.join(self.dir_path, ba)) + os.remove(os.path.join(self.dir_path, 'stacks', imgname)) + + def remove_image(self): + """ Remove the selected image from disk + """ + current_image = self.listbox.get(ACTIVE) + if current_image == '': + return + if not tkMessageBox.askyesno("Delete file", f"delete {current_image}?"): + return + log.info(f'removing {current_image}') + if self.review_stack: + self.removeRelated(current_image) + else: + try: + os.remove(os.path.join(self.dir_path, 'jpgs', current_image)) + except Exception: + pass + xmlf = current_image.replace('P.jpg', '.xml') + try: + os.remove(os.path.join(self.dir_path, 'jpgs', xmlf)) + except Exception: + pass + self.selected[current_image] = (0,'') + self.update_listbox(self.get_bin_list()) + + def update_image(self, thing): + """ When selected, load a new image + """ + try: + # Check if the list is empty. If it is, do nothing. + if self.review_stack: + self.current_image = os.path.join(self.dir_path, 'stacks', self.listbox.get(self.listbox.curselection()[0])) + elif self.soln_outputdir: + self.current_image = os.path.join(self.soln_outputdir, self.listbox.get(self.listbox.curselection()[0])) + else: + self.current_image = os.path.join(self.dir_path, 'jpgs', self.listbox.get(self.listbox.curselection()[0])) + except: + return 0 + + with img.open(self.current_image).resize((640,360)) as imgdata: + thisimage = ImageTk.PhotoImage(imgdata) + self.imagelabel.configure(image = thisimage) + self.imagelabel.image = thisimage + + self.timestamp_label.configure(text = os.path.split(self.current_image)[1]) + return + + def save_image(self): + """ Marks the image as of interest + """ + current_image = self.listbox.get(ACTIVE) + if current_image == '': + return + log.info(f'marking {current_image}') + srcfile = createTxtFile(current_image, self.dir_path) + _, targfile = os.path.split(srcfile) + s3 = boto3.client('s3') + s3.upload_file(srcfile, self.upload_bucket, f'{self.upload_folder}/{targfile}') + cur_index = int(self.listbox.curselection()[0]) + self.listbox.itemconfig(cur_index, fg = 'green') + self.selected[current_image] = (1, srcfile) + + def clean_folder(self): + stacklist = os.listdir(os.path.join(self.dir_path, 'stacks')) + camlist = [x[:6] for x in stacklist if 'stack.jpg' in x] + datalist = os.listdir(self.dir_path) + datalist = [x for x in datalist if 'jpgs' not in x and 'mp4s' not in x and 'stacks' not in x] + for d in datalist: + keep = False + for c in camlist: + if c in d: + keep = True + if keep is False: + try: + os.remove(d) + except Exception: + pass + return + + def get_data(self): + thispatt = self.newpatt.get().strip() + self.patt = thispatt.ljust(15,'0') + self.dir_path = os.path.join(self.fb_dir, self.newpatt.get().strip()) + log.info(f'getting data matching {thispatt}') + os.makedirs(os.path.join(self.dir_path, 'jpgs'), exist_ok=True) + reqdate = datetime.datetime.strptime(self.patt, '%Y%m%d_%H%M%S') + reqdate = reqdate + datetime.timedelta(seconds=-30) + getLiveJpgs(reqdate.strftime('%Y%m%d_%H%M%S'), outdir=os.path.join(self.dir_path, 'jpgs')) + self.renameImages(self.dir_path) + self.update_listbox(self.get_bin_list()) + + def get_trajpickle(self): + basepatt = os.path.split(self.dir_path)[1] + ymd = basepatt[:8] + fullpatt = askstring('Trajectory Name', 'eg 20240101_010203.345_UK', initialvalue=basepatt) + trajpick = f'{basepatt}_trajectory.pickle' + url = f'https://archive.ukmeteors.co.uk/reports/{ymd[:4]}/orbits/{ymd[:6]}/{ymd}/{fullpatt}/{trajpick}' + log.info(f'{url}') + get_response = requests.get(url, stream=True) + if get_response.status_code == 200: + log.info(f'retrieved {trajpick}') + with open(os.path.join(self.dir_path, trajpick), 'wb') as f: + for chunk in get_response.iter_content(chunk_size=4096): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + else: + log.info(f'unable to retrieve {trajpick}, {get_response.status_code}') + return + + def get_vids(self): + jpglist = glob.glob1(os.path.join(self.dir_path,'jpgs'), 'FF*.jpg') + os.makedirs(os.path.join(self.dir_path, 'mp4s'), exist_ok=True) + count = 0 + for jpg in jpglist: + mp4 = jpg.replace('.jpg','.mp4') + ym = mp4[10:16] + url = f'https://archive.ukmeteors.co.uk/img/mp4/{ym[:4]}/{ym}/{mp4}' + get_response = requests.get(url, stream=True) + if get_response.status_code == 200: + log.info(f'retrieved {mp4}') + count += 1 + with open(os.path.join(self.dir_path, 'mp4s', mp4), 'wb') as f: + for chunk in get_response.iter_content(chunk_size=4096): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + log.info(f'retrieved {count} videos') + tkMessageBox.showinfo("Info", f'Retrieved {count} videos') + return + + def renameImages(self, dir_path): + xmllist = glob.glob(os.path.join(dir_path,'jpgs', 'M*.xml')) + for xmlf in xmllist: + jpgf = xmlf.replace('.xml','P.jpg') + if os.path.isfile(jpgf): + xmld = xmltodict.parse(open(xmlf).read()) + realfname = xmld['ufocapture_record']['@cap'].replace('.fits','.jpg') + if not os.path.isfile(os.path.join(dir_path, 'jpgs', realfname)): + os.rename(jpgf, os.path.join(dir_path, 'jpgs', realfname)) + else: + os.remove(jpgf) + os.remove(xmlf) + return + + def viewWatchlist(self): + evtfile = os.path.join(self.fb_dir,'event_watchlist.txt') + if platform.system() == 'Darwin': # macOS + procid = subprocess.Popen(('open', evtfile)) + elif platform.system() == 'Windows': # Windows + procid = subprocess.Popen(('cmd','/c',evtfile)) + else: # linux variants + procid = subprocess.Popen(('xdg-open', evtfile)) + procid.wait() + if not tkMessageBox.askyesno("Upload File", "Upload event watchlist?"): + return + else: + self.putWatchlist() + + def showConfig(self): + if platform.system() == 'Darwin': # macOS + procid = subprocess.Popen(('open', config_file)) + elif platform.system() == 'Windows': # Windows + procid = subprocess.Popen(('cmd','/c',config_file)) + else: # linux variants + procid = subprocess.Popen(('xdg-open', config_file)) + procid.wait() + + def getWatchlist(self): + k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) + c = paramiko.SSHClient() + server=self.gmn_server + user=self.gmn_user + log.info(f'trying {user}@{server} with {self.gmn_key}') + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(hostname = server, username = user, pkey = k) + scpcli = SCPClient(c.get_transport()) + log.info('getting Watchlist') + scpcli.get('./event_watchlist.txt', self.fb_dir) + self.viewWatchlist() + + def putWatchlist(self): + k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) + c = paramiko.SSHClient() + server=self.gmn_server + user=self.gmn_user + log.info(f'trying {user}@{server} with {self.gmn_key}') + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(hostname = server, username = user, pkey = k) + scpcli = SCPClient(c.get_transport()) + log.info('Uploading watchlist') + evtfile = os.path.join(self.fb_dir,'event_watchlist.txt') + scpcli.put(evtfile, 'event_watchlist.txt') + self.evtMonTriggered = datetime.datetime.now() + datetime.timedelta(minutes=10) + return + + def getEventData(self): + evtdate = self.newpatt.get().strip() + if len(evtdate) < 15: + tkMessageBox.showinfo("Warning", f'Need seconds in the event date field {evtdate}') + return + cmd = os.path.join(os.path.dirname(__file__), 'download_events.sh') + f' {evtdate} 1' + if ':' in cmd: + drv = cmd[0].lower() + cmd = '/mnt/' + drv + cmd[2:] + cmd = cmd.replace('\\','/') + log.warning(f'executing {cmd}') + if self.evtMonTriggered is None: + tkMessageBox.showinfo("Warning", 'Event Monitor has not been triggered') + return + if datetime.datetime.now() < self.evtMonTriggered: + tkMessageBox.showinfo("Warning", f'Wait till at least {self.evtMonTriggered.strftime("%H:%M:%S")}') + return + os.chdir(self.fb_dir) + log.info(f'getting data for {evtdate}') + procid = subprocess.Popen(('bash','-c', cmd)) + procid.wait() + tkMessageBox.showinfo("Info", 'Done') + return + + def getGMNData(self): + camlist = [line for line in os.listdir(self.dir_path) if self.correct_datafile_name(line)] + dts=[] + camids=[] + for cam in camlist: + camid,_ = os.path.splitext(cam) + spls = camid.split('_') + if camid[:2] == 'FF': + camids.append(spls[1]) + dts.append(camid[10:25]) + else: + camids.append(spls[-1][:6]) + dts.append(camid[1:16]) + dts.sort() + dtstr = f'{dts[0]}.000000' + stationlist = ','.join(map(str, camids)) + server=self.gmn_server + user=self.gmn_user + log.info('getting data from GMN') + k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) + c = paramiko.SSHClient() + log.info(f'trying {user}@{server} with {self.gmn_key}') + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(hostname = server, username = user, pkey = k) + command = f'source ~/anaconda3/etc/profile.d/conda.sh && conda activate wmpl && python scripts/extract_fireball.py {dtstr} {stationlist}' + log.info(f'running {command}') + + _, stdout, stderr = c.exec_command(command, timeout=900) + for line in iter(stdout.readline, ""): + log.info(line) + for line in iter(stderr.readline, ""): + print(line, end="") + scpcli = SCPClient(c.get_transport()) + log.info('done, collecting output') + indir = os.path.join(f'event_extract/{dtstr}/') + scpcli.get(indir, self.dir_path, recursive=True) + command = f'rm -Rf event_extract/{dtstr}' + log.info(f'running {command}') + _, stdout, stderr = c.exec_command(command, timeout=120) + for line in iter(stdout.readline, ""): + log.info(line) + for line in iter(stderr.readline, ""): + log.info(line) + dirs = os.listdir(os.path.join(self.dir_path, dtstr)) + for d in dirs: + srcdir = os.path.join(self.dir_path, dtstr, d) + targ = os.path.join(self.dir_path, d) + os.makedirs(targ, exist_ok=True) + for f in os.listdir(srcdir): + shutil.copy(os.path.join(srcdir, f), targ) + shutil.rmtree(os.path.join(self.dir_path, dtstr)) + tkMessageBox.showinfo("Data Collected", 'data collected from GMN') + self.update_listbox(self.get_bin_list()) + return + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--datepatt", type=str, help="date pattern to retrieve") + args = parser.parse_args() + + dir_ = os.path.dirname(os.path.realpath(__file__)) + config_file = os.path.join(dir_, 'config.ini') + noimg_file = os.path.join(dir_, 'noimage.jpg') + + log = logging.getLogger(__name__) + log.setLevel(logging.INFO) + + logdir = os.path.join(os.getenv('TMP'), 'fbcollector') + os.makedirs(logdir, exist_ok=True) + log_file = os.path.join(logdir, log_timestamp() + '.log') + handler = logging.handlers.TimedRotatingFileHandler(log_file, when='D', interval=1) # Log to a different file each day + handler.setLevel(logging.INFO) + + formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', datefmt='%Y/%m/%d %H:%M:%S') + handler.setFormatter(formatter) + log.addHandler(handler) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s: %(message)s', datefmt='%Y/%m/%d %H:%M:%S') + ch.setFormatter(formatter) + log.addHandler(ch) + + # Log program start + log.info("Program start") + log.info(f'config file is {config_file}') + + + # Initialize main window + root = tk.Tk() + root.geometry('+0+0') + targdir = args.datepatt + log.info(f'patt is {targdir}') + + app = fbCollector(root, patt=targdir) + root.iconbitmap(os.path.join(dir_,'ukmda.ico')) + root.protocol('WM_DELETE_WINDOW', app.quitApplication) + + root.mainloop() diff --git a/fbCollector/helperfunctions.ps1 b/fbCollector/helperfunctions.ps1 new file mode 100644 index 00000000..08b14d92 --- /dev/null +++ b/fbCollector/helperfunctions.ps1 @@ -0,0 +1,107 @@ +# helper functions + +Function Get-IniContent { + <# + .Synopsis + Gets the content of an INI file + + .Description + Gets the content of an INI file and returns it as a hashtable + + .Notes + Author : Oliver Lipkau + Blog : http://oliver.lipkau.net/blog/ + Source : https://github.com/lipkau/PsIni + http://gallery.technet.microsoft.com/scriptcenter/ea40c1ef-c856-434b-b8fb-ebd7a76e8d91 + Version : 1.0 - 2010/03/12 - Initial release + 1.1 - 2014/12/11 - Typo (Thx SLDR) + Typo (Thx Dave Stiff) + + #Requires -Version 2.0 + + .Inputs + System.String + + .Outputs + System.Collections.Hashtable + + .Parameter FilePath + Specifies the path to the input file. + + .Example + $FileContent = Get-IniContent "C:\myinifile.ini" + ----------- + Description + Saves the content of the c:\myinifile.ini in a hashtable called $FileContent + + .Example + $inifilepath | $FileContent = Get-IniContent + ----------- + Description + Gets the content of the ini file passed through the pipe into a hashtable called $FileContent + + .Example + C:\PS>$FileContent = Get-IniContent "c:\settings.ini" + C:\PS>$FileContent["Section"]["Key"] + ----------- + Description + Returns the key "Key" of the section "Section" from the C:\settings.ini file + + .Link + Out-IniFile + #> + + [CmdletBinding()] + Param( + [ValidateNotNullOrEmpty()] + [ValidateScript({(Test-Path $_) -and ((Get-Item $_).Extension -eq ".ini")})] + [Parameter(ValueFromPipeline=$True,Mandatory=$True)] + [string]$FilePath + ) + + Begin + {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function started"} + + Process + { + Write-Verbose "$($MyInvocation.MyCommand.Name):: Processing file: $Filepath" + + $ini = @{} + switch -regex -file $FilePath + { + "^\[(.+)\]$" # Section + { + $section = $matches[1] + $ini[$section] = @{} + $CommentCount = 0 + } + "^(;.*)$" # Comment + { + if (!($section)) + { + $section = "No-Section" + $ini[$section] = @{} + } + $value = $matches[1] + $CommentCount = $CommentCount + 1 + $name = "Comment" + $CommentCount + $ini[$section][$name] = $value + } + "(.+?)\s*=\s*(.*)" # Key + { + if (!($section)) + { + $section = "No-Section" + $ini[$section] = @{} + } + $name,$value = $matches[1..2] + $ini[$section][$name] = $value + } + } + Write-Verbose "$($MyInvocation.MyCommand.Name):: Finished Processing file: $FilePath" + Return $ini + } + + End + {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function ended"} +} \ No newline at end of file diff --git a/fbCollector/noimage.jpg b/fbCollector/noimage.jpg new file mode 100644 index 0000000000000000000000000000000000000000..80e1f661b5026d1638879487ba8e3cc998297c6e GIT binary patch literal 70608 zcmeFa2|QH$|37|^rc|_$LZ;Fpl7y5nS|oK#+7vN~EJ?Cu2}i4&7Nz7yHKLNlRJO8B zQKS;GOp(TxZODvu%yQ=c8r@qny7zwW{d~WV-~a!A{Oa?V>ztYQIp_U;y>vFwDlKj>1*lDgachIeH~ro&%T3r24aVJBCg0x zII>5~kb{Uf;t$^dB7vyKxS7u0-iP!xH4l0%vU70XyKj-byPKw;-62h_MT<2N1A^Zn zJA2oC-ZS^^b8NZJ)Q@Oh2~+2fg(D3}+ADT_27GUu(`bGVpY8)ZemZ?e8Pt%y9PabMf`{ zUF561$lcRPQ)~J1<(i9^Xf9cz0V6cL{11BD`Dq;VQu+M`Yxa5Bd%7I*c5y#AQ*cAO zz3x8VhO>QqTpaWr?HqOYIymTR?6uQ&(9qIyu+y+}(ALs$(AIX))74tG#8GSS?B5^n zVE^suhkQH_3_sh!UUT1peQx^>dV2viv@~aHex3CH)-!=b-yZs>tYiVfdZTn-}7inuRHqaa~&OlQ@455ks@6P-mQOn4{!Cv29Fn4#)uaDol z&*Oi#+Xq~XL_qb|Wx}^#58(d5d}GIsHP94(^pCU{{swptuKUKRaOKSZ1OI+Lt9jeUk(1rT|de7uPE@Z2LI%)|DTcTJKEiM5FoBEU|D`GvKqljN{*Hsg&Qq7 zdJGOXMrzVHz!*|;G84y7nkJ_(eVW`fc}3; zBxErKiOk>siHS>$k{pd2GgfLG3`n1dh>J-`h>wzxlpF;qOtB#N9T_!Aa`G&#m7}Lv z*x_b-$Syg0@#YwnRS%!ZZ7FBW(cbHMY^>B&`DxP?=FXe1x5zjgag z!rjDsk5bdpGcq4P$;vNy{-UtxW$~+b6_r)h)b}518XB9LKex2DwRbRkdi(kZ23bRF z!Mel{iLcB0wz3iHngr_-A2mv16i%=%F>zn`NK6_fIZJEw-u(8i5Na|(xX0{JtdI9M~*!vu%dg-!#_=66rN**v3pG!Ef?*) zQ=d*fVm&SH(D;yv)3+(R*3wIY2`*NrQei}?>4PVfESGw9$@?1W7v_w*aBpm5h|#>b zEOVBnvK}9~srMYuQYZ0|)A5*G(K$ZSwvma$q+0n%qq9WJKO0twI^t#8YOsfmqp03z zCTyH_Y>L9C+Bwh0Iu5}hL6;z$YArrVZLJo^DW>bEcN3s;gpE5 ze>{$5+BD(hovTdW=TigR?oU1Z;BDM8jiyO)G8&u2&r5o2rxT_H$~2akH+KK3sY4q_ z*SnIs>CW=5z99!wWvjEN#vR{n$yEX zG3oLb$X-5@|CnLQ8=HscBhPl@xf_3E9GkhaVkQ4@Tq7Qzqu6jxu|clCCA^_zc7D54 z$ijgPtu^)2k(o>G8c>UU21<*%)HhcLPa^JW8<5u<+-M-VC;jAY&ru-}aw`oii-{xG zZLCx$_YTc}%%s3HX9}jtN50Neggh|p2)VPMfu~5UHZYF%`Phsj5~pX`r$y(?j>(f~W~kxID6+SQWJVrBk!g|2IYdIRm) zt@6%K=D4Xk*CJ2FqPXMvYhgRJP8!Q zF+$r$1|Pws>Oldx-irIPabz79ubwvEnKp(@)a~PBh!^U^z_ih|DdueW3#KdN8hY3#|^TJ}K-AB^G`3E)JuKG`h$(#sY{NBoiBpED#t!)Da`DttXHkKl(S7`hCam>R${(Op2g%m z%bQl7H;HKLj48caxaf+Xc&V$s>?W}b;u31dO9G>a2e`rS(kD29db$yA=Y@J#Jl!#F zr;r$k18 zzzKFiL|=Pf-i!ld``?VxWs(aO$8FRG%e1hscIUz_yX$KXSr@9TQMS4!sU*e8%F|^P z)T2}PqrhJNcUZ(-IqA_pTBYs@uV*josdQMpTLJ0$Ulcc^vUhniSZW)UOPU$M$KICB za&%iV*{o(cC;bPRadln4HrzDTOg8EMnCUzB98dGuaraN~j6m##Eq&JdpQz{WQ4cF- zN5-+z!aTL~)Kkke8mNn!!c;G)t&LkHeisTCpNfY?5&1~O(MCSPnWl;Mxx`T%iV4j& zre(eRxd)mxa)O!^SL{qaz}=Oy(p_pyW0=PJed{X6OKw&~ki|$Mp~0R-!$LJd4ZlM% zwI^BX%+z=K=B%BV0584acX#yWnAT-RSqHVf&0SIp8Bj+3~jFlr9O*<;xDd|`(>f`mf_Sg;Y zHyrblm&J{b!&zoNH1>r31QvkkCL5>x?jZoR)CYXzLsCfW1cJ!VrrbGM-MrE1c&-%6 zy#Gp7!W^4yb9^CyUPChDqg~XOnn7c=ni- z;1HdWzvCGZbV{E>CCXU9M+#F_M^4&?@;u1e;vx2uBgd=Cyn1lMkoJ;;1kL%l=w7RC zOqGwknS$kpW3P>sNW#mD7g^dqSur5(f82}!%Yh8o8mS{)6!$ESWsO$rPzgoZQ7e0S zii+_4sJr-x&T8xvI&k~YKfCjf+uVgI?XNjEI6$d&|B@1zeQ4|3t0b49C({iqqp2t( z8?#6II|&2(k>-C(upf4r%s7~xROeT)=~V0X-01liPMzNny3hS`|ETgN4PVQ2=BfAz z8A(URo6Q`j>BLsOaqilXp9C zSyj4XcNdyLtJ^(vxIW>!woyf|=Muk>-1B{V>qiLTpAoELq}q5=U*~I?#XNO=!rP=H zI_9$`C`C!XqAlnco3biP24wSP_Z^f9x+69kq5Rbr-xpO4mVrJ30f*hTF~yFllSWG6 z4@@PZI;ieU{Eo^4=p&;SsE=EVbp-ayo0#y|#m`!DszLvS5qd}Q5wblWc~pj*KY}Ix zd!@)r1|6AygX3QNC9i7WtUu=EUUc>B!MCozjwDz$S~8K3WRiG^|1B4VDrCe5@yG+U zoFvkskFvdYoN9s(t~HlyNvCnah5L=hJ0io24p}Zr)xW3OpY1iBtC-R;XuG)L=94OS z$J5QT;v*V9SbsJI8GLP1)fH%Wx4`Y*r-I4r#EWJW+#|ozNA`yvMVoyq2=qiSA{&+U z1%eBEp8|SprL5QnVYT&V4;;x{faOHZ(VMssL-j_Z=YWu+5#L8!vR`mT~Jcjv|ertEnItA-8Cxc#BVAW z@f4Nu9|TeKu0L3Uwmh+_e)ZRe-gO=FTYpQ|Tf6Uij9meUscsL6yATybJw8L!L&2Pn zys)e!NTW@!pA-5&^Az&*W2G-)Qwf5wktc+WCIBxt0uP=H2M1m$2ORk0X5hfzYiWy> zrWilD(NojW(c?1zarEcrE4y~5zI-me>vVI(CNrIF!Sa0M!fsMO(6mk6B)B*NeVG-0 zv!#ml;gyAn9P2~7WRMAij%_?l!zd1Xq`WZChI}_~23nj_3)a1p#9mf5p_WKzy7t=M zF_E=csa#Dc!E-P1ktj)73*#d@fmUdbIkwxEkBp-vb|-~Y=l*UB@mC8I+41VJ3{5cBPCOq1du&SZNM&Lmsv}a47ZH0_bfEo}Xxr>3 zY(zCK3{j`BfbkZavj+$qjYD!YF6`%T^La<`yelEgq*UWV1fLarWGD_l(91{C)k6UL zV9ueu96r*uzFrlH3sXhls|H^#crx_&XlK<^OJQ?ZFL;XJGXjzbp^W&al%5ssr2a}E z?!53B=*w;p4Ju3|HVkQP8@@clyL+r_&Are^?^+$yX4o35?7-a|{~pJPm5yl)=RPCQ zVPiVepIWB-k)FvkxZad*(b=fD!vEl0mD$TKq~f1i=GFKSS3O*JGUJlt9hJ+6PiP;%*6LAQn&-MB z#6X;yZ!|0Wn#t5~zx!*w7OvIhBY80r{V9Xkq#%1t&S`)E6RF_xF!Si^daWrQ6`yjY z!Vu>BJXw0rs`I@&pT7~)%1MqKShj^Q+XaHEiRrWaU_ z#78LM5ElX}NX3BVBjv8ZFajT$WJ@tY{0EQbs3)~;+5h@(?Y*&aJ&isWcdlC-u&3E) zhH;8Q?=z!054XFdojEoBkioU}=)@CEOmaY6h>=ImyZ#A=+sT~=-Bw>)aAWF5+Am>A z#9y|r$i@Fci7CmeNM7VnSl4Tu^1iQFR;>5BwN2-NeLzB74W+TIVnGq0xh z6*$;4?pYBFcBY||tUGoDT*}#O^!eD%Q;k~3qi{5WFV|TjL};f|g0sUF{Wk25c^*}B zGVi*^i?l}z)`m)(zapFnq}}#dvX=3x{MFeA6*-73-trlBSO=*Qu%Iw(+${+*B8#?+ z1sN&gBl}{d#luu%1s`c)##E>9k?2``#GSy|@>uE*RtG|AQP=}`j;?^!7k#$_-&kE= z)Ee!0kl|?xWctsgY2ybQjFc3{sp&U9 zBx_+}ni<*)D_p=wdW}I#2%pZN7LZsyOc|a(x|V!CsgoE=V191IvwN^}#2MQY7pUJM z-KLmrM4L-LKY&YrktXN!k)Cmvuxva}(m7uEg#2NN`H;LtpFs^part`nE7{0gWXFd$-xE`o*fqYjxcpXv~(VmbZ>!?G%v#Ag98UJx=1R3cdP< zy0V@)G^4MS-GTN^*^1_a$Zw&BQWz3|tP*E3o2ygrSJ#X_KK`!fBU6X%QEjU?<7;FM zUge2^0=ZJ^c0%~Ptm}84l<7>r?R6z$!|XS-Emoknh1Y3I8gNv3#3{j2e`#?}iNU6r zv#;bB@{tfcFA^_8AMDNZ!$iLzN0=YVG{yFi`a979D^TCUx3dWi2kX&E>7=kCU7~Jy zOH$r-<&<1sX1i9>%QC;Rpq7+)TmNAq6bZd+2wkmwWWS}DZ88re zcWNqU14kX)8DeH9#gSJ;D?IE(4o(MY5Fuf#^ncoikmb?cm^}K%9m@^HN_L9yNzwzx zR0=PVm&ivZCod^24$))J&5WXuCltSom01v%rb9z5$Wltp&0g#myO{pS3rj+8dp`WwWW`WBcK)>svMETxo-yHbMLxFE#|(@1EIS zy~My2v(_ud@5cYe@Mv2Sl=S3X-sH8hMwxfDGo<&XPeWS6uZOD4$Oumk*y!)xCDt^< z_nETTqsknupaI1vKFjL;4K80&-5dF9LE?<<{)K+A(wtuyZ_mX~tT}vr{FViC)~_9$ zK5dy}aZ!frmb|%qqgWy7XO?+xv~Q+6&_X(r?vYqf3ag9?Uz_hnTbE8cbIODNdq1=4?|mdv=JC6CgJs{M7EN zKQGe%P(@w)6|Xn%>*i#%y*ada$-2J#MEMUL8sn$9SS^aYw@_`hsqg4P3+(C3X-x^X zzsZU<^jU7zJ99hpYQtkj#&OxZhhDNhc5H21d2j(qMDIvk{d=LBK>BVM$vZF_A33v& zcbgY^2PM}I5L1Jjx|>#LQz=$IC}-S%(lAOTqQLsTuxxLC?>CN!Rbl`?Pw%+u)cjPD zH@WsTje4rNc{A>Exs4uo9;si$?+92%0!NJ?qV_lc$45_O$wMqg8_Oc`Abvw0n$b^S zcM$p}S$~pcn%CxRw|C>6jH~rOZyMw7^pN`5P}(p4MeqWSOc*(t@*vHcr17-(;KIS| z!1%;c-Q4j8^w(ELdnHa&e_8*xIzB0tqdnx#H<9?9e6FX>VD#;!80!HRnFe?qO+DqtKShd zB&H(dd(9z^tK$`bTIM4yzoNCJSmlV#K-~c}-N)WxuPO-CE{^60ipL=H`c*nD**)}) zDwU+6=a^14FJxw7qjEPZ{|D;>0Blzj7bCK?07b1+Dv_%YRpjv^>%j|+RUirbk>n>v z%C73kANNlzzSVx^*M}+HkF5QJ_slt~H-B~pU zWH4(ZhW%JF5GxbL)#M{HpAhJP7|V8q(W;VTQ<>zjps$K~XfEnPT2Z2ecUTs5J+6*K1}G%*v|&3JHA6G%e@ zJ`y+sEU|Fp8uW;;2e{eq$*m{XY`Q7d1N0ICZvi(0+{oiazx$JUo&Bo>m>q!d-=O7! zl7Z0Dz7-E*QbXdx)Ss@%{~tqqwd(74Y&)oDT<_H8K7XT~j$7GbZoIt1waP%3@%Htm zT94$;yeC~#3ErfmoFaOW;fgYxWCKld|EJmNbF_Fj-rP92Z`u~^<7qzj)C46@lK@gE zcfu`avAH|*s{9_$Ydka-7oMfBsdAp%ulo+t9MiG7-+dd&z%z%YXHEfc9Z4w*JO#KC)*Q+7~F-pB{LLqlOl5 zt)z60)5OA$mhLBrHCN98qatYf!_hO^G`YcR>X0@DzunKrS#8Zx>?tVT}q*-#Kgm5$f zzOwa?wM4(groa#6BYy-NZL2pf+P%RSz*AOAPI)FKiQk3;#kBbF~|6BfT~H z6%!l$x>D?SHeERDPLD7&F0~~kD@(K7v82sRCA7=b0bK?80`u07mbdTSo4IZCy?8_{ zL=v$m@7pjz>ZMitgN0_?jaAp!55J%Z2ZbzU$6L^W98Zr!by%m_n8l=1w`?V?t{%{SXNge5Xl?< zZ1bJUsI$(I*P2tltYA*q^qohJZ-K%E-_to-1MbMj_Gr1B2(?%9)~)>b^kA{XlOj7R z-FC^!QO6dWZgy=CXg08^S2v$HI{V=V*Sq#ng_GR}L%K04aW^wWW@_Ah;!Ycvs7F@B zr)B2T3shu!FMe63Ybyw?{1nQ6FO-!A^EvBTgp~~2)3n+{ExwMZ_15>9ErVyd}JPx zo6~PD4%)>(AaejWa0SB5;o4n!wpwqsGICPDM2p6sLHxZu+-O0*UA*?-1Aiv^{pK@J zxS9C?9~r(INwd?|R;$n}Yxr)$20t{WONP|uYp3U^$K6cnUknOMiqLox!Y-kJfh@I6 z{BZsi-_{k6sS?@0p*@mbI-7Ax=nK#oQmNQp(g$0Vr`!68(>I!rvH~Y)W$HaC)dv+V!KVqMegJMT%S)Vjb*Lx z-FAA+)ImM=vBJ9Lb_&l|UE^E|(RYCIo2jfkbkgo=d+H{faO*Lxg8-5{qic5+5~RXM zCU`-B#~j;E;Xnx_mCz4*{IfM5Nu!ojZpOVM2!B;BGCHu%1PQ8TBsFSs7t*9Y(g`2X z4>wHcR{jL%-b}YOLA4p=LFU=RDK$A82DS+>T_XB^Kan>{@`xL_ZIL$2YY=OGL$L&4MBJ;`PTpC zm(O8sXJ}$_QDG@QB-T1g8kaLKttYskx$9#aC)58N?~VNodUe~^Y>ZS1Q<=<1#(-%0 z*f_o9m=5?{rUq}~-Eg{ca^TCeLXWfc)sbH4BoL=7eq`)vCW$?(P2xG)kb-uY!RBQ0 z5f=zEv|_hN+$3}$dMGOlVVeqlK@c8Ej)AQSVB)$|?y#=Z*;g7!JuBm0GD!-dca9BV zcDg2%r;|hVzud^&vBd9WsOjqFgJ;>3h?+Z2ok{-WtbeR-XP%7d>OlwP6Pt0B1SZ(z z$^uBGsXt)Ib5$rz9qhTl@7`?Coj!#6lCm=?w&{&E(iE4*8OwHm$e8A_a1FOS^X5RG zA-yv1o24?IM*DUrRd6EYu{(I~(cLAoVt=77f*Qv_C5XEuoBJSg?tQ)Y&?rrfn_fb$R@(p|**`;&MW06@+F}>H; zgO9w6x+kIdhnB;CTcP^ZxV&t7Drj^NR0DMAmUPUY4hl~x#o*q}OhDcj46UDPvkp@^ zTbkA=T&4`W2QSg&wBMoenQU{3Ck?&v32veof)t9cOAc zCHRZc$r)6jF1U(t;+f&cx- z2Uksdl)#NB*?<*n0c`)Pdadho3ZTll(NVlpfCi;n zIRyyf1EtU2;;(<&j5{Lqq(7ecAOG&AJx8H2L|xDr(u(~W(5@cC3KyX07wCW_q>9UB zjPc)aF5q1;)TH`gpN5&{LeltU-h{&@UnT@?XKb)er}owD{Zc~t{9(<-OTBApdZY8! zro=aZkEKfp#Z(l|UkiLWZ}o)FNlFWu%JCmUH`j&VoikC!KJH$7&AE)Egctt%=CSro z$35=Hm4&4b!}A%qzp1zI`}5$OZQfR{OF;15<`>*Cf($f!b;<9s3(EKs0*%f`&J$w{ zqfzlb_&NtNrhk=s%)KAI`|*`-H@vlvjeop5SIX$FAYVyN`R+pkZ7upK>?3?6lECSd zrZ<#w{ry6*97?=p(e?pN;Y8ov*49BXx`M`Rg`CB0!j-3xB}9BgYZH{^|GHAamJ%0S zVyi_>n?SHktxVI&tkkg2I=%14o{p58A$w-4EHkK&1{ZXEB9uo+3A$m#=R=XAE|fPY z&m~#}LRRXM4OkEI2ST4miaIuZD?o=nIR-3#lsN>eUEdq5U~o1w$j_E>weIiNcd)eh zn0C&i_0h9amkudjU0K>-Y+D189S!-t)-*v>atx98OP(OV2e&8;XMuI-*asOt96$yg zTxL`+fjbuRmc7GqW04=xd|2jIO-Q8?_{hUp!yF2fI@ZG^qGAX_afKA|+^tWu@B?cC zQqlb`nVAM58#GRM5u++J})XjeTvHTD5 zW=?MR-c6SExN%74wuD)x!`8U0L@rMEe%FMzl-L3_B^^)O$bIZ%P@Xvp?O(!z9h88I zRt1hs3NY+-W$MGk_rsaEi4^wLK!_L|1qb^ao&f;2@-^ftWx(TiW`DsCl^s#Ie%>)# zBpOz&CwkLPgCnt!AVUG7J*WPX+OtO;i}L#1-TsQqG@4!yOAR0S{*$_ z@6^1C^fSfQuPyfZB@{PB&CNK!Vtl{KS;rjzJQ)`AeQMjutLM%1E=)G(4H_$%$kn3x z*O@dPLh138!4_S_Ci=DLI0eZ_#n0iTNRtNZ5RYxZtg;qXRx+XK=2P|)_mpb{S9v`s z{yY>$QZ3xH^XSLTquoZ_CY-f3Zv|K?d2G{4hzghC%J3sKAF!Yk&xj65uH1@1bNTlN zKUX$iSNS;S)wH6k`JjqrH?+Q?cGdrxZZQ?g7o$=9TN~s z77a9&<#R}bQrKF~tXYDy0;uMsv9*#;u@ac1r$#C)Vsg8}qTGQP`rYkzXF9#^H07G_ zFvcPMI)MvWrS*8}K+STUbFQ`=-TwOvQ%wez(_B6k-Mfi9rmuhu?pH6NK;~>3gmg;) zyvq}~2ea4ZY`kSIwL$Jv^TpK*dOkWRo9;)d;!@FuS?~BrYbzg_mC3Szq_T7rA0ZHi zCQ$N2bnd30yO`8a1H!G6#Qp5>4_&irTg;l*)p|ZNmAcihlI}@kl%eC%HURF$<4BmE zvGOGo|4gn9z5vgjPU%?%cRg1Qeg}?Ggq?EdUU&l1?Jd@>W2xTIeL8qD(16&onq%bB2<`BzCI`^`)nHRfOR?4NjMxck>U!AZ8Jbuy+M|e`#4|3mqrJVLaca4$ z66?cyYiZ|c(mGwM8ejE354L%cmOMUjK=F8bW&PP-8U8cw36fo>2*b&)Qb=~i%9zFF z0Je~62p`UPd4Uq?sE%P4HGCo`u1skEG^;ly1@B_!8VVMl#3A|+Cu{qrs*?minUkc1Y+m0<_BkZQ=RHEr*rOl42D`EbZ0W$ z1d^QgUz41>A#*mIPj?{f9}8K<6EwX<>cViVMDcAsal_XmW8t7rd>qh&VLbTs)_=kfM|~V?QRw937Lu z3VoZMQ?Y2zF~Y;$iw6C!ZgVI^id7;!jPWyvgGg4Nlo@#s86r7059OD6lD&)Yvw ze%afdM=PlyKvKR%Xdn3c#V-l0ENGxisl{Zw&jW;z1kHW{WqOOqaLBS~H7xyRhtZ6x zIiL#A)Ve}3=fR}c%JbXyW5W|zYG*c*zKy5p=D*9)qyPEMRud^ST`yLdX=S3G3JtKr0v zCU%Dig&Sad>`g^AAQJl5k>O8_gp+v%jY*26uDYqMuRQ~kj+A_%Eojp!QC4SULY6Ja zi?4%$zZD;en$~p3>YfvqyM-kTXAm$(Bw*X$#xh9|xfYYS34AOwc|0 z^fZLDN)Z2UFp|yk^Ld8}Y^id~1Kx>!iX~guj%pXyr*-96#;Eb&fXWDlh7b-1#!fq2OL4!hXb{V7~ z1SOY3iF+uy{9ed|)T^c3zPs|aONDJ+#`~K#MU`x^FXnRla}pXge}!O_u4ZSpWl4H` zFyr_jun^sw-4bG0(cpM{(Zl+6y5mBU=JdWDf1Ljf0#0^K9d_!E5oMiVQ2Su zJ0o`wg4HNQ*)6nJcZ7~ZjMN45BAc-$@sYXHDfCywmh_gYc7iyeIbM(FzkypFVsvqM z$TiFeEnbxYzJfQD;YRlD3v6ZfV(aBHdB@|yL^j9DXbq1`?IRCK>_VwID0>X~ta>L; z9W$;b_IKm4W2)kH5Or4~6oS{IkXbFmQ;O{A3Nm3{!pznt_aEmYrgHq=_B1r3{b4&-9~Wrri_K_nW{-u1(*&gLV#W4rbzb zC%&M4t~=%;aur;#6cLd(QEClU%Z5o`?S#(+8Oo`D6sjb{|P+8yy#keNqSl8C>553shI&&-s z9f+fr=?stRl`kX@saf)o{#SV3)bwYTN0>?2`bYqyo}qliB>B)bWZS?mmYXy8%~>3| zZf0LgY~p>`Vzvysjq*0cD5| zOxvR4xxjMEYKiw{qX+-Jw`Huv`#psM8br8WawL;KcSO6rhJx3Lg9o+juj?oW9=Uin zTIZ$(xmEeJoTD&Q*|CE9PEdhp-yk1RLkHhOrpL6JkFaMjin5vm?+xwo%p< ze3i`xL@-^F-Pk!D%g(Fwxo7UOmEDNvNIL_U3BtBQsrV<9k%_$~r(1Hx+zpWXo85$E z6+ttb6n%uIl9lXA?qUT)?EfxPAJS&*BAx-*b8tHZb=DGed>??Z`LLwpBI-z_*!8iL zjJ!^i_LaimzYi2+u>gK)NpJEEu@tm#+s%3PiVerjlS0>@TAuxy~D#eg!O=Vb)$GkiUd{ato*#v zca-J{;GEeqLCMSx(I^M;EZ{o8o?9dZ?EyKh)_gA-LpZY)uCwOLF^BY6qysxn+P*f2z{k(6f6Cdr_=U?0=yHS`w*XHvfr z_}&3c5DzZH3Hk{T%PQJ9O7m}Rq<-+o+cpwasiI#P&)5f&I$e1o@TjIWDDXf&!q!M? zdc~1>LF$=HM+am`piw2$J}! zeYFDj1Ep>Q>f3!2Xyv+F&X=M-C`e{4Yc(txS}{Gt zhobw{dT1vIz+D1rs`8PmJnWmICHi;sr!d7TdxeR+uP^ zP*ZZ8d1hcuI*dYNESsFF@gvYt>1%hqU*X$ob!);@*9+G9$CkyN*>ZJ@u7udO(NB@_ zh}w^hwg9L&*pu*IcSOnO$=n$3rC2g+(({wQaX%f)oBw0G4SU7Y04&v?v+P-7T?e&- z7L&QvSL1csE&s=bJzi%$szWG?D9IGXq#%0bYp5j?H2%dd0v!?(j>)3=xnQ~REx_M$gcs3d$W!hJ7Z>>vo4i0DS^#@3mj~qoj6zadx-&OR_F`JSL^yIXp4{`otyHqj{?-(*& zC20p`SEB3x$h?xBRKqH;I90nn_tcyVlIK^1ZYN&$Mym0`-L-%k|7VmGCEyO!xyz=H zIH7%(#M~E^6Gk~rtIhvH*i6`>mMQ$yl&0qjm-frI>0Mc;va`^9Yxr*%`qDD$YKzwE zwtMGnG9Ae7y^AKc9yVGKm-j3#&zA=lG=VUr2dcrX)8MVU2=a7Q2qRQe*bpPQgbv*7 zG$@JwuHciFi2ha@ewX~f^;#Chjf*MtcvOF|$KW-`>Q=++*RRN6$JhAz7ZNy&t~tFf zdDE3kSsPd&Yz$m0Rmq9$@Ii_UYyMj_8SQI@oCF{&^ObQq>@cHvUUvaDM&x_)2uR!R zZPAbK4#qQ93oR+I#3V@D1aE;2Ht>;A-`A4xrfMJ<;Mh!8P4*zTBP->eSAlNrABPS0 z>ApT2-+#a$$(EaBR}_}g)rt1FW0ieb>c!6FOV~#s7aGo$K>IIY*U@GgL;6h9d~U=! zYdQ`)zkcX&UPxScS1H?O953uTMqo(rz(yqyhxEAOnjh$xNf!~z zi{c~3WFmR)pIvMfmvbeIkV$B#f!RO^)#hwS*jz%N8n_lxdI)~Uv3y?wTQ_NPZg4AR z)WS!CDA!GhaTdk7x0jWYyl>QDP92rY(Aq8M%v}yT+^C}_Hm3^%YbO&IUML?iC&;ko z(K?3ftcuZklCcGmP;NSa=NWJb^`Y!c7=iW>S@X+Am^HO90-%BR)?ory4F}aD;9Wlk zSHk-5sbU3!^2JatV7nc|&<~K-s2(eJHdHQNnlS}?jOUF_dP#=ptR{z6Rvi>r?SRGq zIzX6u#)k5yMoHrhxr@?6#NM+V!-k^6dh>F5v?5Fl?I7`t0D_?$`!>y~=*y^N0E9*; zZ)z3ubHD`Ipd%bgxS%ZsaAx};29K;&4l?=k9~AK;;C#z|RrcT@if4&op!F-dxv(=C zMvxA4O!1^+1mMY(b6k4c5`*~z>1G8nmm-OjA!TKcNOWJt*`c8)*E2FZ04bd#_H2Vy z(JvF&8m`BZqt!dmDgrwQ3@#Xnq7n|OqHqU%JOnE6) zJR|#2<}3Y)>a8-;bYCMy=G;@Mdp!&<)78#49l}K#J$JzGb!{{BDwMA|^>FT&i%CRN zd(Vp{;&TgBw>#{RK3h>H>ad#o+uLAZ3 z&p78w>63(45=;UGDcW8t($Y(wSVs6SFO8MXY>bX{liNAbBwKt;`=Dznz07FAL&x%6 zr{~l@oHNKtZ|xUF0%Vhnw8y!uMYB>fuQYD;xTv&lS9rIfq<>X_Mta?gsAnF!*Pmzf zoH;vow_6Q&ON{?9Sk)@MQc;YQbUb`kG72mqi=o%HVYoBMS%EYmb^CQfb z&nYK9av=3Aif{8z4_@jC`JLdU0V_x5vB*)b9h#xU9#@D*^mFE8=XJijST7+AA#)$C zDCLr%x(K|^Sfhsm<tQ8|FqX49cR74AGzt^&CT5x6;HF`Mu1rxxXmlO7>s02}u zYjhfV2ET2jQ$-bZx^UHkP{P!OZdlb3C1U<2>-JAsU((MsZ~_-bjq(|iA0o9|Zy%ye zYRt{h{Gf3&p`;rlsysUDrU;i2bq>HRAPWNy!CM=&sF~JZPPy*w^ZhvZ=req`93`Kk z-MFlPC7VlZHh}`z4%3!~-;uDu_Z8dj{=1f}x|f7vH_=bnshs;wynOP_+l14yf@J=+ zFTW>Dzt;YOZ?&9GVsT=41DKE81~B4yFEo$#HjdmDIG>YCFd4#E`QZoG6B~#TaADS_ z2Z9LD?;r7W@G>AEc!aG`nmq0g%Sa+dd*C^oK2tZmpsaSNVvo=zh_Hf;7e48e#_|BvVLudb2&$bQGx4Od?6cwM`6E8!!d-fJkV_g)R-#!cS! z#ubO{e{C_^z*9`1AGclWb&_2z}$RHUq$Ka)f=@5 zgHO1WIY3d%Yhu7Z-^$e>oF`l(^29^qQ;ktZM;0W+S+YT!l^}XSwE3Ph77tU6{~p@- z+uqPWElBPV?Kv>e6z)I}DMr2wE)JA1N{x7NPGI&(srueWu+{{)^?Q9Wu%1vn^_g%J z?qktnRNR^94Od@$;`CDv=2Sp&4C%FCgAER)h*XFR0;Iqa-(Pj38_(Uzjuk2n7qrET z6#2Ba8KtSl-EU_Mu}`S_D?SQ79{nnP-Ma*}h!69yhTFdOw;vwwgh1Tci8;I@RS;?_ z?AR8$D=VTdo7Wl_843G(|0ysWBKW8aVaztV&+1t7~H8c zyd)pk0ygaH=N^rZ+$9cDZYFI9NWz1Vd@gT-|E?m=N#g4D=wwxRx4_WykAH~lAAh~5 z1m>U~r;FR6c0~wfm9fkR1A@==5biMuUoL|P&w&fPuM6CcVWEY;lCIyR$4VZO?ee_t zHdiBxP9tG(uEpR_5-U`%VSdan`wEP8Z9 z6}Dy@XxKZ!2mhF+toX`NhfG&PAfz)(cW~g!IalAx;%OS;-O@506KqmeH+=4nxwCuM z!+5lsh8>&YCr%R`mMEW*vuXrDDThQPK{m)$T+?uL=ZeT zt^9CLjbw6IZVgFErDNN?K5o2KQ@rp|`-9Y%g>rL$i!7dVEy5LY|0CYAYNUh~FJvU} zjvj!B@G46)LH~6L*dWmtyIC@Itk9DNlV&@!mtLK5-*Ztd;nCgW3qM{zbM!|44_OZ) zy%#!v3|d8-qJ(?+lCH*u3@|YWbRE^iZ2z|J8p`o-nc=;f+_7j+ZOimt)yT zdceBJuH}q_jxf@;lyk69NhH8|wq!roSOU9jfzcl@g?#q z$7>$fyk~@R!xiLtC)|Mb_EKwuw#x7kBkITS^5A$d(p=eo` z!rSF%QM%D#jZ0C*ohzu(z^4}X(b86Il{tgFftP?blfyZ<)!a`x0l9KvWVwBM?mGe= z7Vh19*?xqrJ`2|9(s{u15#MXbr@x46Z#_%9h+S8*F30cJHCM z@4!VOm1WZ?(}|aN_^``XqjJL7^8loi^7;x*DEbeU(J1|(ad6<{q|Y{{xzY|_)Jt4}M1M4NeT`mnNi8|x_utmsHu~?n zVxe?D$hSJ4ONR$74AHx*;f#h2Zy@MM3DFS{{A>i!pi=eHbG~Dx1U|m zcAQ|<9B_zWIqk8TfHf@huh062_+irT_(A0?;TwLq zC4?UY`tv(1vbvY~2V!~kmhe%Ht+17KaTUmG@!R@8WbOxLXQ>SgG9)`6Smd-IpVsk= zkK|PJn;s-$gXY`~y#`k@3}yj*ECjt}r-<^VrH5N9gIT}W1X%3BISO{xiqq8B^bU2Q z4XHO_&&v3S9$A@eC!(@K^hpu`4@zj9Gu)EHx&{YxqYgar&0QL31IcYt|@iJh}GIpRv>R-tigOw?%CqG*Vmj%sLnGFThMEF zb89M>%9RL^`Sh?%VAtBK|K9{)eeNT2A7l;DG5}vuLn~a(}JwkWOwyP@(O{WVS(}sIoupY)_ zcpIq!o*MuS)+)8GBg#u!X(KA`ZS4JC7kn~c+3<4gDjVLlkQ_@M-fxIxR(LI1V)(tk zE8(&Q6ea!G8nqd}E1U}vnPL)@Y<}&y|Gfvo^xJ3&y4G+Hm^QQ^wUmm4Cdk_ungh6~ zByi-ga#>pQqUz(qxMeA~ zPOG>T;pVP19BZf!5_+;9+N)@oIXaNV;&K)h+N&3Co8=wlou2*WZP?bcI}#q0+Zw_f z)acl43ik#??0r6QH!(z@W4MAunD4I(&UP?HNNbJ{uLZm@H;u_DvEdhd}XVEu?_gCB~F!3AYYLeMk zfW?l$d(p~;AM5??wxQ{vUhkb#+_A|t)*}kzfzb=tF(cFW1~j%ys*xi>$t`6ZCQs#U zK!;%BHEr0lH~gT%~cw zDgJgXC^Z3FP9)|(RvQZ1AMv~>QYZOdP+&?Rd9t0Wed5;hGrV-5|HPm(V7pukk?|Pq zfC$JXly}mXGTsNWd0;MyPp}f~b;cB4_llCYU-TClb=w9|Kk@d5qi2)hvOUf6Dw+?G_`7%v0KfpG?u1T) z1otL9@X{O_WEF8sH^x1|kMiq=HZ_IzJUsT6BVT)K!9Y0S&4-g3*R$6K1cVv5H&Ixt z16>pSP|k`jirt0zH3mS-;B89}cP*oLIJ2vi;dNf$2!GvY`yZOs)*tpJ6pKX#iKX}3 zytvlAP2+f{GjYo~w&=k6`ZxDheTAr66duKuk~?U4rFqthVxd!4ujkL zAYS-{T#7zol8Ra6UVX|#4|umkHBASs7`-VU#eM)xw|y(NhUoaoI%J8O%WPTf9D zw-2F=G_*qoNKfRQ#DDNBZRT16#TF5Oxa!c=l za;@T`7hB0=U+K@wf(1PA-?a3`>?zlDHu?}-{E`p&t;S5xjl=T6H)d@GbpKm131FArp9yCK)TJK7hezooeHchw zYCzI{!}%E0aQ;Sr3GlTSK=E#FxS%w>;xmh@t7KreWk*0?5KrdOt6qb7_^}E;PLAaA z!9M*Obx2yAXvJ>QhG3l!;q@IPb|~76Vuj%1r!PiJS%e-#Lf3s9m&?Tu&2W<|MuqNP zJ$Ou)!1^V}^$rVtC15cvUcB$sR4MBfhqNmyEqIB?G4vS}cA##12(7L4(5yr;+04vK zfN;6{IUh}#A)*^+{4qk>!Jfg-tDO!?>$9U_xX^SsTX9@?sU*bDu$rf6X({}A@g4D- z&rwJ!3=Xsleg>@9n+mT0t3p}p(e_e^Ktfn1siF6F|9$I3)9w3s`@JhlJ=+T3CNyaF z=qsp$Dm#&YT?0nczrPrGvJeOVXI)WYEVmtJ-mep1Cksmbd zHZjOcH`_~8bFn59)k*5lA9I)LpIg@O`AvmydCCo!sdw)YYw&%qp)gC}ogZNoJtZ<8 zg2kaU?&HPqq7KaGBNnaxPwI+3?GE~s(@)OD{vY<gxcQBe^iAUL8D6cH35B2`*Mgn)=NArv9>-a-kf$8QDa9tk7P z{oL>KeZS{>-}etoIN_Xq&e?mfYh7!tYcX@t=4N=RTFT|R(e%PYv{7uC`vT$h#~9XGNt5w!Fk2TT zCF|k8{$NUMJed~QVX^mNh-&PGeo>vx-wMEcJom?Z&ls&yEpSum>|154wn6JtrTqnZ zZfOcHO=%@i?v&hKZ9tulZ)KU^butXcI_cdPK7#ZMM>O88giVT7V47Caw9IByWkOBk zUAv`wDs`f&h!4^J&^GrzeNkrj-vA1)J1S$SUa?nHuQ&>d(Mra1zv5uegm)aM-J%81 zd{|yy%HY3WmOV_N zO9JL)sRdR&$C9NibfIEi`ad~@s{!A7h_(Oea8=X8aUy?`3b5gU>Q`JM8T&%?AOaf5 z^9QF+2TI%Fjv`Fn5qdSuN$kl9kg(=;WFWd0&sAKd#{abvvaT|rvPH~AO!Z{La`8<= zIu!7IPOLWE&u+E-&k;oR)p!a5sM5eP=b|m~Q8LN-mkGW4DR9Fi*sq^dxZ}OSCg`W^ zfFHm}pOhx9?-qX?H7$SM3yh{|H=r2U*E{@cO1hEQZb1%(ErSYu-8E+Y8IQn^e#_sV zitrCw6P)GMS+rw7bfOqJ!6%e0qbm&hej*w1k%!cya351p#s_#daUo)NC4MYeMkbkh z)z;f`K+`&1@CWLBCayxT1YsRCB=gSM3Z)lGmWbxk&*WvI#{6{WvInsCFSW@yzg_pH zRCR{5+eA=TSp%o7QzG#eoZSH??8pD&{$WLG!8?Y8lQ2kUxm4w{PK^3b%jCe*P&A$k z6o^_&39Dqb02}CTcE&UY5RvrP7*QDT-<4m5z##~9jZlte2dV9| z#!+1nYC5J{kPQ=6T{vh4xp6dXAMDc-{4UttBv3)H7_AOBEmjrjQT+Dc!6vp(axpj< z74)yT9i(M&DorG!5ZgL9N1R~)TZnCC}J_}ptUmr7TM~7t$i72K<+nxTAg8rte<^Bb2(`VBLyp_^({op;AOFQ;_AUq7gMez zi>oeDSA%H!hzqrh99o~{BFQ#g&e5Qya0Eczl?1*4rcAN!O=Gal)4Cre$gj3;rSI(3 z6+bsvxzkjv4&mDwOG81PsQiw#VW|8rUQ~XEW^5F|Zu$5Dq&6?1ajG2Srb)CSJq74g z+P6ABTWpPSLXT>lnO6!pSL4?R<_9KJx@DYv4I53!R>iChT<&jX!R>)uo786Z32?mf zvGg1ubPJ>aA^+E}r32-H@4JV}1>pkTj0e!pq_Zcc9wDdbur_ZgQjc`*T)pPtiafcp znmrjNiZ9%6Yqyw-swC;vvpJ8yRY_XOAN7YS$unAo(Ykt@bmtGKzZ0ch`IgkucwAfI z^v^D~qEB_YJT6H%JyybA{2l-B(HEir_zJb2a@tudGpk|8ykdq;PBrKt*}S%}K?gb#dD`4fKH z#V@DJ6T%c!8s5el=M{4BR0fv7iMd$&O*^D|4 zY~9GEGD0Udr2@nuC?l3dLakXAN^l_8UAytrN`J88w9PZqK(*+%53Xft9+sGWseo~X z;Rvz?yNK5S+&9wt4faW0VCjk`;yTu5gQ8VN&K;$zV^lI=BD#j02zq%XXh)kBxQF5! zN+@U@>>`2;LZc{;Tk-197gE-uK_O00{@scOO~y0&oHZ3uqNgmImL}pG}L|I_T1wSRP}$cRkT}6Hp8g1#+4ejTdHLI09H3nyP1P`}SL5uU)y^+Tt7OYB2mlbgh`pSd zpY5S3Wyr|AAO--|LiOIyTi|C~L0p@A+F*fou z1`hplR%)|OCUOhY>S@lDw%uNMp*!YC#m9(AB$G>Q@0(7qr#LD?}yrSQbm>{!Xq9rfOF3+LUo-k2v{Xjj>z@+auHcg`K zJkarAT8F98{~#^F1S+$J4fc0Wc8N{o*(jcvtZ6h=lC_f+Ax1KaHni7Z(RV zT06fhUVi}Vp!MHtc`Hg$;C8dgjVK4`x2Am%qa=U-w1GNY0iIY!8!le@QaKMr(Oqt{ z??SxRzfUxZ=aLaUG{I#6)=^;nH|j4E`=rX=?T2ms?22Da&H7ZlMHrZ30`2A8sAkX2 zEo{T}=@+KtYNVGKZN|Obvh9pxy1Mheo)=y0vpCiafuE^1&(XD1F`0z+l*3m zg%zsdUHaP+`^t}H>`0XU7~I@L)f11EO-~D;)pL#l`st&HD@KTmLWp$G%7D3~3M2p| z`+*C3W%T0WCi7s;TeRj1NCyF{xma^_)rctZ7~DSRqlSiu{38?K@kR^oKoO1NTjIu= z@2c;jDygx+yD$NLe{0S(QxBV#Iy1G7HqkXXlgL0Sla?g>QNtz5Jy3Al|JmdwypZ7| z-Nz9N#`SL1y_q(RkI%@h%a^VMcQ|F@K;-ak*!H&&8*Y>|~%U ztl=8wU)zr(MhY&l502$8Cq%0AV0n{t4Wm? z=O!_*6rUREdI!&Qya6YhuHG-#Aq1d3zW)UjFhQgx zp~zk9Y@x>TmkzgUHiqcCKHFrMkhcYfc|FSS0B1?Pa9)0={9@N0C#C5H>(rBb zbg*r*UvX|}ejGB8ZeW9>P&f<0Yj<)Wzr!vwseWyF;ssu_Y(M&zAPI%r)3p|+FksxT z)6o;rl{AWc_i@@S)hX+5CsOgnNQ0AzYAQ|p)WZ~9ugBg1t$^d094?4cn9|)*5lZt} zZI|Gb(Dw9Z%PPWEtyQd5^NjG@_y;CG6uy`S<9w(`&LnZ=<*_6pP4Fwwjw)!fHUSt{ z9BZ{lWTnNd_y=s~_K@o*`s%IeZ7;P2dzQZG;lIb^OCrzbS?%VlH&$z0#|o!*=kGmy z9nuNRIH^Fw#(;7~0orm{jgIH4YRQ3TfbPWAXvo$+GN`3y;U|6JR21`9On>;wcKO2W zMQ`n9ntZ-Vy6rZzq>d94PRg+fVvL-3^B1Sm%OOv1o-4O?@JKx?Q*jQ*xelGU)ofwvGgp5# zJN7yDb&8vlYw22nopB|9BTCP|f^<^)LxsU;!^~Pd-%Pkj@;x{to@vTGsgFWSlb^W; z$bWH^9{RO2FSiRUE_8dQE$aF7zyghIrI^O0(0!CEMAv&4t;C{QbNKL5$yaEib8#7Ig7(c>H!M)&k(&Kl=o{j#mXth}H9 z(s8BhlZ90j_Td|Z>r`6~+|YdI48j)duFEk%toYjeC;OzXx1lnEgCQIzrs3!FH08Z- z=>AB>JnQn--fO=jlcCMSE0(8V{xWm4oWNR<7|%D{=lYBg_`BJ~PQH%9jr=%gGf-S! zL&Kvfl%jYk!Na9jzv50XupDiII*{Qrsqoi^rO9kOCm{0zTZi(AeB+#Vvb{|^bH*U` zD5II2L5MMMP%^ZA&AD-ESFtH@0B!Y|tBZ`jgP-OZ<4HBytdo;8LfAV8=Suk1#BA0 z1YQ`n(@bRWe7+>!gc{wS~Qmwmcn#!>|jssbm?LL0B%{P^N z54oyIPk)+pkQ}=;v#^y=4TDYVPIbOUgQ#DS3dA?MGPn~Ax;Y|=ypE7`!Bw2Y8IEJ! zHJ^PJ=t@hSosylCQF_5wT!5%XO_870zZo9R)YPP{pRpNxf&R4{C`H+p5w%S2FQBRab{3=8 z3CD`%xYjw2nO3}F{`9M8(Y)qPQXAsMiRRd6E*Ffg+um7iR%|zG^SKQKEv+e4?A>mI zJqa_~oKK~u(S?V}d{HO+0&hLw_U=eCg_i`oAZ_S``;1k&S;7Z8mmuX}|0CnIV*_dj z8?>CHx#b^8tL;sqn{Ton*Vs;$pX>mZ;fws1RD3fX3k~h{k51O0&45+fly=n%(=n%Y zSQgXzORetfawivKnHxyL_0UfM$so&y7FWjm-#Fmxb&3?b_5B^?`zp=$J=AqA&L@kf z!n11l_2BO8JNjU$wtG?H&~Qwxj$X@@?Y1ZNt~5f+oZln7O)6K(ya>Xe#$uojX5m;; zJ#sgc5*#Ue-djk2>~P_5cG;IN=QH1ZF?rq@$VM*BGHv$gPT6Hqx7LlW75u8TG$(OK zlc9#L%CmiUR$Q6=P^LV88b6kmse8EvL8E z6o18?v7XY8Pzbz$dmr?&I(h{rR2P%LS6^{58hZD1Z7TE@?C4r@->pz?igwV+h)2wK zDb0^N{fwh1y-H!in-^)k^HJB@v29jVKHk9-?r|&5rRqfsgEzmFWqHOCf6iLr^Jb^` zmZwJ9U#8x=O{eIG7|}Bk?!*=Xduc1gB5~dBLBJa^@>ZzoyiTpj@pFb4jM*E#93kkvL%o&i)0jT3dNmr2KpXxi} z$Jfz~?p3;R&KbE!v$IW3&#wa1`8#zl=deuGntc!WeJXC75xzRkd4$l{2vx0>TX9bK z%){jiZ@rO@uex@qfskC3!b=vD^;llF&CT3bqRd3xg#O4C8T?%}y_K#}rNtN0 zwX8os`0d2rZ*3U=N?FeOL|i>nm2^0i$_x!f1!H;3mj7JsRBK9gJzGzmyUc4==E?E@FEAj)?p%@Z9jne%`8YQUp4|_|^!iml6H?@V=^+ zdXR}yWYfu_te4vDdgvyXo#sWwl`e71jI6FS8MC-bs$h90?jTUPxCnic;2PBFwAVdHEi>Abf609pQ}wc>mY?~iH2f(*zR01SYg<ISA8@&^uOl(CV!gn8xff=0FkP-5MRirGvr>TwR(PnsXXrLgxuQ*Dx|QDLBD+i$apfXcM)gTw*$ACy?at%zdqfnr3KdtsZ|@g?glhx zHRSQf1dsg@+T72I!L|=x)4CyC#7PG*O30vW2_CPt06<8Xqh~-owLU*WoKCw^gW-0v&(%=ARQ3Ap?WO{VFc-RZrG!c9HlT45j2L#hJ zg>ca$Pwd)2zdV|A5q1wRCVznow+7gvfqU4yiv*rLqX1CHg32ryl^WV{S0YnQ;6niG zz$U=(+wJKzK%^{PVbw~&v3gVBl9M?oq-;XbOi+Rgpda`e*z&4kq~8>`G7ClP)(FuS z@PC(V$V6GJ`h5P(eriK3b0)Zu*O_qtA%Li2yJcYl)5%DOSrN^*U`#t)f?G%BD|R4v zLen71)2WMn&>&ycpTGFrn$FDxr^ z!&ls%Q*Z79b)kTC9E6!t9D2SG&F+*Fdh6R`fqCa(D$iSwmW9F73EPQlv7t4bN`g@h zUPt?{9G*xR&_lW$4noO+x|^7_-#6^y+OpzTj}*K3tDhZoUnf4aHN(J>YS4GcMcjM= zX1h8==$W$lEAHt$cMX#|ir{KLo-cPEX3rE=wlKloOgm5SEJ(fn%qT~O*HW_uxY z$m^XjNSaU~B)Sc8Br5+k*c|U0+H^JvHR%6 zU%lD#ri=wvZAm2B-lHT4=)LjV!93+RQsIb)n++1-jixH*Szwpc{j*R5KVHoMj7T2N93sq9OIXM)7;p4T?~suikv=9(X$ zFPJ$Ws5`V1=427^qsUO7{7(!uElYno^+LV$HY;ltwt0}Y-uBKa|Fr!DmyV!frRg3G zw?43F*Y!|SmUo84)x<4#-HsqH0tZnsc@kqYmPdw@|CRL@!<@s647MGO{OG5hJwB^( zrV6I%Ds9|fj?y;97aO7vYfG9X4e4c z1cNL0f^?hEtLLHy(iGTToD1xFJ}}Erk+~M2@WJmLBKrSm2lh$)kd<1Dn5%KP-%LF% z7BBG7m*z z@+ANew7d_-%SMuaMYCCOmTgQhs!L2p*m?+viw6tY1W5AP2`Q(q&Les4^l-@+J*dDb z7p(9GyimHmWi{MtXFbZ710=%jO9z|Tt|q+Ff%LDT23KOc61-tp@ERc>=q8Cf55dDn z1me`C-DI>L)C?*7Bs1*R%LQ;2r5?y@%>cuvX~P-B5DY9$@-wvM1J3Lg2KvXgJb*1= zGWG3mLIk~@PGPTX1?DnsB_H}D)7l|J0oneVHV-W*Itn z7rY@t1x0w*0lD*Zv;)CoeoU7fp@B3WfAJE*bIA7-!bpX8RcQ_85*h$a2wD@tWzIc- z?b%V&Jpc?}$C?PZ&Ch8wN4NVpU9jbOYkBS9Vjk}47F>3La~zYl;%e&mkOal#s2F%i zhe_agWh=j>v2mp|zk;z|!_2FGAEvzqWlO>Q@V^;jg1w*%p$S~ewg_W1dt5j67K2Ax z6s;{EJ*x3;1xMefLXAHwwp*wk*mMxv0;Vx~FFcvOcO?p{(Cj?0p~t(y(zO2iVmi|H z68+K>hF!4~G6c66B(@?8kKMWX`1_UJRmCoCF#O8LM?ZuUhZN$7f~YY%f8a2ZANV_A z0ESn4>e0%$#{TI!gU@_S2nXzhRxN7_O}zJylhwvKNf>YJVQxugXjogx`afmtv57p> zI>|J9;@h4QO+Qw&>`uk9E#cIBPT1?SrB7JmJcSny$fqKA^UrmPs- zDhdP)2G6{!M-dml8ZHac0XBJ_Xj|nM$m9>l5fORVbTFb6=kPe8cV4(dfv9`K~?M*mL%u@|YZ;DV%CGB>PBJE|h+oxrg z_D*sAgH4^Et8&kFE>^Tzp7OAM_64fn-;_S6PcWJk7ukexc3fccq%9E+0G)a<$(WZP z7O9ozb4i#akbTlW|03QDf$%JBY-co8Yd<{?x19KLiEtpVVAq*Lx+}Gqsl^LZL~<_T z9hEvvZ-lNYiEp(P#v{C)Tv5?Y`#)9f`Pbm_dPqa!~|U08O))AjXJXMtYY~`2+nFXg@KPJcMn` z(h1xdL?Mr-TZag4(Ekj}@pr?;+OZ1ApXYDLKzo(Qz(%OnEBt_rfNqR#e`gqa@Hc6_ ztJet6Q&A28qB(~k>fyeY>c#9e&{zY-)l%5i1A;SPykcrmj#Pj6Xnce_o{aP&9Nll6 z02zpp6%IN>A9P1MfEx{eD0hPCAc&HpMWH!1pn^MQLm{AvFh8IppdrI|X2BRnw?nSKH{F`}D49g1}Ny15Au+l_u8$~c03;K9u__+&Qp z8wjD-cGj99YQvt6?VSi2#Y@2G`U7}e<1Ie~enc%+l*d92-gJNs5DsNv+X+`_9dO0+ z_BXcNrPc5TqXBQ8jGBoyTmr%-3lRn-2Mhx=xVn4Tc5((_5)p10NUmoB5aNVkUH1WJ zsg6e*t_uLnyaL>kZ;+-5)*c)}w5yfETj4%)v`CTdPtc6H1ejcofJGFNF9HL>MCt4> zAQ}|$4U5%BIj{)lg(2`DU;8pY_-%(FFkz_Wo073@0Y64EINhI64r-$Ybf%`GeS5W% z^0BZQnmY)yUpOL#&h#0*;R_#QQ(yE?U@U*g2*;g*+WUvV#@ zzv9llqwprbU{HD`V2gYO?JWCW?i<%Z>fHdQ%S7+dSKPB0!w+coZ3=ratm4+9B=gaU z3f%eObbwf_ga*%b7#$D}+sxqIGE25rlJP z^OXhHTOq{zh>AT@eVhavQD8fkQ9T!6*<@|SCoA;f!&NFkDbfTVsZHTLmI8nNs%rt=N2 z%tCv7+iCr?K-z6FG>0II-RaT0`#UUArEh=*zqLn(`rP>~W*;t1c%+duiXFp_66TnsN zU+3WQ6*mrUgLOEsaJvjm`%O@9MO$01g>qtGV?)4E!r4Ielw4WrO904FaU#Ax60#Cpo zPF1XK0V+876=x5?wD2x2|0j`&Vi&VPObKORWVlLk3K1}$B80A@DLb)U+7n?%pf}&7 zbvVDmHU;o8Bo5&(>Otr380F{Mzx*&X5slLO3G4#~+yfQ_79WPmUqnQ+3&uE@VOlO= z)=dIJ1%ZC)%ytp{E3<_$ls|V_IXdHk3!3=jl#RF=8aIq!1kml&i!=p5th$%b4v9!q zIe6kQW+ARw;BEs?7+*&cs>+@k9X+_OIRWj+Vf*ds7R3N(q0s_1-k>;bOehfy(@pY2hK$s>Q@KB2g$`o1u_PIbeI734W9(8P7N1$w4V z$h#`#iS{qDQ&GFg$R!hjXS25AjiNtmy?{>|U7B!&r8gQyo6w>BqvNX_eO}bP?&1KL z5^qia%u~j{K#QpY0Ib#S5OE-YuLKgqx**Sn`$r|Uy}B1md*f&_J5g|kyN1BezbB;j zFjSTeput75hM#|}dR9_OPmrhP`-(j;eRG#Sn5c2<_<^H041IUxEKI_4h>E~oKVAcH zo}&up{Dz25;EWmBebN|!=!AM`;k-~3YaDw9WvP?i7g}J`dbZpMX{k7QF*-Z>H1^@i zq?of8myUD(W~zrWlUeIUC9lK9opb~XDFaI!oJFF^d=<$uI_63fNT9ubL%r?xAZ_~4 z_YQ}$525eCEk3}2&UTWp+94|$ZLWkT3rleEYPY{Sph5+1*rf~koel%0SZ?uYm*g|& zOl=zjJ~{QCGbV2AG_~7*V26QCfMA^ocMfC&XuAJHN!N-ig!o9U0l=l?DPiW9hcobUK}-gLdCI7@znUH zf_jfh9%t)YKL`&o3TVSc+9KrGl)_R7RTc~!nd5DVu@80+CuaxalQrPs{f$vLs*h&TuFo>MSN=OV0VE zLi+L)%j$V=R<_HL%Mcb2e_JHkijD}5I3S+bLQe9#!Yc(AY+r}BXyr`>E`7U2 z+FXubUB2vMBj>|>=H4Ua z#+^JA!(sL7u`*oQ*$nHfb2MdYap=X1Oj#E*3lsJRSF@8dvhrWQbKu?0!=3$#lSLa& z+wxkCq2x#j(|(S1x(vQZwz5YpO7N7`=aRcHuS-VbJnHkOIGmb1-SfGtKQ2q-I>ZDv z9n@ApGrPGt~kFWuG6NHSQ3R5x>}x_l%~IwnXWCx znwHdjBMzNnS(+3SXU~3+yW4S!V6_8kcLkYx- zQF8)!94J_n!{q5>{^md-vJM`NPqk-(YA z&>vjrE!Dg-nVdLuLQ~ctwv{j*twdUPu`01030INMM<=ZY6BHN4#KtjB@(-;((ko}a z=a1G8HIyuGCtm&g3hU5o3Cb0br&$d;u`uYyGrINK@&tLLf$U(?LMbBOBJ^$)tn1%N zFYS)f&TGZnH&cS#NNKoXvWdx8sl-&uQ@kc+U1?46VYixT&hnwy(Ep zWpYnst?tY!rk`Eetv!>D?O`4O81X>Fsc4Hh6%Q2^d3L$HMh^o2fQWht4rtSh}!H`HtOO9Ubf z)vIiWh@ft=;Gx`gfCC01V7RbYC))4bYW)qrXm$}r6)-{i| zwW%gYZSPy))wM=p`E3u};5he{qp?P+JL^783`@2sK4(m#hkrV|jemttrSEo<(G2%3 z8b9B^YDH1ot`~1|np)2L!DQj5eU^AITGq??z=i&1=fVkE3YBWNqiE}9Bxfs{Nin;G zOv_5o4E^1B8Scp%#+$+k&89P86thlgPk+gm(+p37nh$z@`?S zOmHyHU7oUL*|I*;-9*Omf%QCr5wjqX>8AHoa)R=-_sa5pS1?J1aLVJw}WV>o@CR z2Mp?-DShMLocQKcpZtAg-KBc%0jRV ztnNTuPvuWihN(wTPc2#Bs#;gIpDq&gR_bi;dR!5!V$TrAYI@9?=~4MemL)V)M5 zX{?5us<>$C*tPU%3U{6F`QFCUHrRrC1*y}sa2&>nlpku6I_Yv6uCyuP-oXNm!T9*w zrAqAFtCiEN;^aJMzU^`xc!BifQ=qr%QCRbQ3bsz0AgiyRk2;SHeYi(G^S#%!{Z!(q zwK&%u)F;n+zPL>`EqFXPEpyhku2co~FVDX+? zK_g!Y*b4VUoF`1U%NSg;w|MM>`#<8a@?ovG&d9bPvIZ`JD~~O0zqoe4yH@wkzBc}? z7-10s$%8naYzuCf4EY(^R((@&Ty4`E-|!bew&`V!i15M3A0K5plz$k?S&EJ0_s4?M z=}y}w6)0+^IUMjJV&f)eyXqr-jgX%i2bSUOw|<(#?_{__=pnK1T8@-oXy7=$j7qQz z-@MT8nf|=2vM&#B0&X55n!N7Ns{IN6Z%~;dsEFnx;P#UM*^iJFCGj{8~CwtP0#S^X{HXxw@ez=ZM-#w~ocd0j7Tho*z6idXnD}2X`*mXPbJZc|;P_BdSS?U^ z&JyCC2uAdNGd18G=s*4sz296!)sHx$Bg=SE*PNoc_czlQgeX04d3*I2q7;ncdt*Oy z$p%++yWi&0uHR4Q96Fsv=iKQ>?zIQzQIB%Gf>8_e`h5wzzZ|`ML?Ty*+1t)#`28hd zE@AYDEbxe=wZkSq*UD1?sEyGTxe%RP~6gYh@P|KToDR09NZc@VOd0v9pASYQ z`>5q`5_wn8Q}k@tR+QVi!|s>(gee=(Tw7;yBFg*oShShYBI{_E;C#f{%U;oApTM)0U3?LRb*$;a|Xj&xs6n3v6O`>7{N)B5eDw)LZ zi$x_9C(2kI{>Kv1pEs>Q8A^i8^Kp3&-#-ksQxsd$L^If~=d|1M8fe+XnZtJu?Wu*| zmcIn?z-R0JN2{INvnnTze?z7QD>l@)srRv?U+6|!GUNtz2K`oFY`&^+*I6R-0&_~r zeqm@hda_*0+S*|7W!R1RzzekwL1V9fte7CIAz z0is42!(Xjy)={h#PicA!drCiUcuzt2=CD}XI`UAJP?3;#_5(0mlTv34bqbj~Fw~?A zqG0?2ctIg;l>T;Llc1e(u%dU5Cx#3lyhcKwdfVYf+DJidW#wcmM}4N}0z|mLWl;3q zBfx3Qn6FdvCr8!(ZAb(#ErBk&=D3$KC(8uvvJJ~L*wMYL$;l}aX zPI@2mU%1cCCrdjL9ih5L^)(kUay)1aPX+pI0!s>$Ob41f;-&3O2WL8EA}!(8`iJWc zi#y8W)gP`%TW@2Ur4#A0;cs0?mgI#}g-#GWlLNN%Dj+8?>2u&g|EY)fCJ+p%4(1B- z^Ivgy6@DbO3sUA11LmNkNEP@)+|2_T83->mwlT{H6$aQRjr@471AT2)#mF`WasHVb zcPy}u1qfnIZnzA~0*Wm();!M$;oSU$HJ&sNKKoG#7hQ9Eg-)btp?0Y3qQtZ?(+Eqq ziKXgCySs?SN6oB1D*bkKw#hBOR-jM+8a#~0C4n8Jl*A`bZ2SD_0k(+5`vdJAOJHq+ zJv9T&jOu;p+oqv&VNj~%`J?Z*)S=-Fwg24$@L9oN#vz{x${j(Zs`_r5?&46#J>uI3 z^`Khjq4(>~^+)Gk51nr2b&e~vYHbLqBDYV7Tk z_Y=#;F^s~(o7dIY-9C+GD50M_;Tcn~s{LaQUpjbs-7goZ<~?#=*Jxov-vK(?J4pbP zO+Z9BY-i2Y?i%N?neteetpKvW*lkFy53TLuSKJ3(5MQG{1+9_?=xdwY1jka*x z--k;k_rPm;XIgJ0;+k2F?A8gcA;KY5$a+o9t%_?%e~Ln4h+D=S-B>h&=`Z~#rJnjc zlmkX_K?D@@_cD1)!Z|?4Er@te8R(_)mY7RQVpB-G^8i*TLd%3Ohx`sXX|*zqkI%k1 zmUSWJ>?q@#xGq|)Df(F%>VZ)HEM(aKYt4gY+Og?G;U=`#4D2cB+Wz(bsOkSN{UZL5 zM?KNcb~dR@K7Nj&FhTT!`ocue6c!~waAlNlwI)bH8@Ocjw z9=x$Q#$C{&f|WXvhJ`{z;t|LQo=5-t&(NW7PiMX|Tt(~g=HB0v$amX7=9zWslX{9M z11e~v4vd6&wNP6TUb-~wq6fmTLn!|+;pc}fIOiZGYrdjNQLHUz81*L6|IdKpT;fj{|1u{o4quM&svsw8I;=uSl!*=mNhMBR)1CBY`jTl zr@rDGhP8|4(;U}}=g0wT)6;$UV3IT@^cd+LcvCX4m`bI%6tC6|?w2e(A1qDdJVx7& z!+e~EW=@2T40uc%o;64DxK$7oM`kV9;IX|A#VGt>-y z7ewl$#JS$vsN|f`(%lMGOI|!$r281fOhJovp|>B@iKeh;m^obA}}^ z=FCPxW{VjyeiDPk4iV*y|%d4Iy-~w`fI%`?5@128BRqUkBteRrz4A> z{<0pua4DT0XGsifNlMMS+7l$bsd(J>sI>O;$4;h8wW%f+#8o1{Sy(uq_@bz|!p*!g zag{{qFjrM72NAa%5tXPVjjQ5hE4*|9Kvo~)#y%-=F|_@|bpgJeR|D zR^+LcS#`TZQydK(&JD^j+5o8xM96&e6C{Is)C9MAX2J^yZ+yrw!AbHM*l)U@p$zab zg9F7&r|Ee#4+5$1k%O-HL*cajq_s{Pzuhdc%aO#tito|s)$u-9@Ag=fcYjJ;+88X& zU#eW*59tpk`BPuCx(U~rqJnY8w`9gZ$B{d{FyVTD5vZ^Q9_apI_*$mDHecZA;EAQs zLWuOkY_$vur1FK5I` zBgB}`v8xh3FEL5FVH$L-E$XrRv3Wy_G4<~XTaad^xL_nSj#B+wNxoH?#)cmLz8-4& zTEX&{mg`Rv_kW4?wrhT!e$DTU{QZsjwLn9mBtHS=adWIlRr*(waCe%BdxWf4NX-MJ z(`E`$Y$X$Zql;#pe7mHny8yR6@TcRNZf;PVtqT97h?Wrf6`ap&^~_-Q3<$`@Bbz1w;dzzUEAe{B|yt$;Ut%^DYYFJOwPu8e2*w(2W--j3t0KCR|B?>9)sPjhg% z=)0Q6JxSWMnh}DZZXOXw+8^(@`kI|hlj*_MH#&XvqL}9griJ7m*s-*Gdgi^=8+AA{|nz>UAX$0hq3dEZneU{mtsFcj^b3e+N@ zC%~N6$A>9P5iG(A(!n!o0oG@iAj8*){Kv>mi&#L4mAZBTmFMts5A|-T^W~sh3c+Wd->|#FFzLVYR=+s%TkcNq2W`opQZaYXg-SoLV?7> z^WCv!Rl6f6rMXxnp59;h$K-6yt%Qmh2vl6{o??HhNGM$ zr1p?QG3L(Xs>2JG(_vr3V<@1&F{Gy1z8!Y`|Kq`d9m!mHu%T4|UTtzL&cJ!Gh(2Zsk@u(qAg zzv5Q#KT$h6Yz5qVm}m#ie-ewliX(6b!9QmR%&~W$(PnT~2qacW}4d#+7 z5+!ahD>UGdE(tcjO!0cwik;ggVb`3MDAo2Z78QHKNAeY}dNe*d64rLxqA+5wVT_k< z-fUOdGW1$N0cOUazWimPDfUxV%04CL}>)chF8D)ys23$T;$8tH{bo+G9e z%-;R=x;#m=;WHvI5`XLev%b#Q0WHOLQm!H$>Sj=0P!uJ0=kWqr5WdmZA*UeEEoYJZ zEXS|ETZ^X~r`I0ViV=b;{ocV|~M2=td5*@bAp&853C-9QHu}x?XbUuJR8-W*n zNv%(2$HmqEmG7#oYra`p)%?ci#7*_zyP73{&p_W9Jiff9?N|3ZYu2v_&ws8Y$%;a< z#x=i>as17}PdlNwpmE+Bk%DaZYAKS9$3Rrz;Z{Av&rhGG0KS{Msf(5$q11llrxfL^Pi_Dt9%|0{6PpfdP=)V2IZ9lmtmiD+J`w4@au~T6H6~Sc z_baT3qY=-vii@o}qx;~(!uK=$-caR5te=`XkGtzzIiUJK+$eyTm#EK>y}hsfkH&1+ zcR!EFnRiXCL1onMu20$DmSo!grP9%I$x@BHD_t{f*3M>lq8Tb6>2_Hpi1GFYP$+q` zZz{IwabX8PJ(PidsqPRF+>eppBJ_L+f~gTq^#med9*V8hR&*ZTIrVU*f{N?7zC%T7 zv(_XQ4~&mAHNUaw;jY_G4L;epf^8jWgO>E_urvkc|M^w z7cU}G!MX4#8zS)+JxTkm_p*pKPX^}?>H7s_??;$g*FCW%WatGGhIV*uT zqsm+hFfSiwCxMA5uELkgnR^ zA5oGCOx%HuQgd#XFdQs504E<*;uj)r{{EJ+!= zqDqy*`gBVa%29q5!czO#3yxx1zi_R{J?p||p!6mH0*1xhz<&Kc>NNt38;&woVW(23 zr>Ozk?0he`7t#0xeR{)sMUtzsjX}%4VZXnH^l*pdh;~Oa(@xQ!+=c=|;zh=94j1c* zTMtg!t7J$?d0*cjrDD;LcsFO|bdxWPqDDvA-G^V44j$*wbk&f=tP6+E8~7PBbDbBG z8nxXHDU3Y~w1ABFtngUwM8s-03nmME>JM?4;i6Fe%&Ydp+!nD) z;{Sa#MgG4IiT~*`zEc#r7my6W(!q{V20B0q23Fg!ObX}w+xoa}VKQnbMUhYEW|2QE z0T?I`c$Nf;@cB@1?k}HGLK~KhwkTdF=Ip@oA66Hg7PG;ewhk zNb{51=wyTaduFpduVy^&l6H|@&_X#U(ALUZqN@QJ|j57Mqls+d_->aeJHxJH^dC~X;hVYldo;xr}!}ikFTbuse zl83z$qG(wS10dldX`Z22=KsGq3j8+l)|tJ`k@|gpepPWPk;7~HC0NF^5h_8mfb6`B zmjf?3-5EgY9$Egv9WZO!v^*aW>&;3DGDt%J=%KugM|pEqpHlO4Z3zxppGs8?km*); z4BlN{`^4+O{R8Tf5{JgU#ZAJgKqo>BO#kCt=)JUx0>)4j0_0<(C<&!0!+IGZ3&7j( z)(MMo^2DV3&0@}S9C!CCPws= zT1j_a!XPSAW3t1GDfj4zYb#7Kvd291$=Fzg0}i!VNxnHg0);JLF+^dfCUESYwyj1g zASm@JDZX#XTix~3_t2YaPM(A4l9GP6P~rkX@C(`##jMSW3}>QN!g#E$IX7XfM(cex z<$;f1rX4}7OwxlYeLY}0Yd`Y?ND!iP{+DjmgUa8Fe(=DC>qG!6BGNI?HqrsNjv#=* zfLKkZ`kR_6dtwsn+9T;t5zec0=St7MC-Z^1xNUqe65A{Y&-kYSePIP8M^tb!_Ftae#U z_dQD+%SOmgLA-Ht|D$buzEX#t98t^Y4Zh)Beykv6wv6iakvCi^&?$+NCv=dbNmT}p z0Sz?2l?&^8d(SvVeOxND_Ebmrxmj_Kb{WkUd!3LO9chobCy}u+2xtGu`7!bmUyeca z$BNcJ7D{0!53x4e;+VWKZ3yRAAi>2XD?+eg6R$wZR_GTFzYxTiD_aU&(=D#zE?fHj zhn-ngqWcg9ErOSVcj8XQWGg{l(h}(jgH{iGYMg6`BKokB)np9TDpR` zTgG!&G$X2ON_F$Llp7D$`q!K^ev~tVpdWp~?3~H+n_Y5Og)>YvU*qk%RPK3Ps4{NP znst87`pAw#jVr6p8g5W^2@78s!uk{$PM&VL`U%!z@G|7I*16M!MJ;bKT|TY6)-#aW zhW8P+V^RZ;Q>@hA*xYQ69B3`}x;lltKOLB=&mMj7rIax-)$Y47-)J>SRK|Y>u*;#iDfk%sqhnwbvj~5mGdgStB{;3$Hcbm zw7cEZZ=`6|c5=J^ z(Qzw zcEpmJfMgx7XIr~&Vx;o>1kb^WZ6OJ-C+gtb{7L_%r8#i^`N0s`36{Cp>G6aAsX?&X zs6b=yg>^W=xr1~piBGIp+2FW@rg-XH!ptb8;o~9N3M0QQXxqvz(Smlw&cTAdW%Frh zL5o$_X!lf)rqV-nCVJOpwrMIl6}M;F3??2zZ8B*h-B^ zxs`DXV`h5KsBLG)?q=J4|9C&2_Ya@($8Ub~JkL4L^PKZN-*e8@oKP0&?UPAj#;P`% z;d8E!!}e~nmJzu7uMOvK9iBP@a3jsZlHo?uy`LB?zIo9~m83TpD9d}za{jDxpXs0N zRf4={Sp_LZSI&#P&XuG(=O(>;9POac!omcBLuE5{*n}sJz53jb>}*R&36l^ zQ%nrv7sPL;H+0X}ezwEO$=On6Oxm&hM4d)bpj^0xmOdvrHj0&?0=}M~vT;3e@XB$( znm8C4Bl>wOGvzgf6;=bLer(8f{%~TZRRg|{j!zijG%%h0cHec=H#lY{{4eyV`L%K> zgaIkzU}$Jj7;lv)bvZ$IZq!6<^U~ep_(?4Y*>hZN@xLy?iSWjHGx*A6Pa!Tm{yA@q zKb#ZVpYnD8+yQ*gPz9fmn_YnAs`-ulGi(QW1&%k8GN&%udsOPOg&yTCksVd^&ni(*eT`AUu!YR+>*KcR@L zw6Gt%@J`0o4vo1;#+549UdoDh>A@aKU6tU3ok%;aZ>b(|)M{^yPJkDxj4fS0cfX~8 zBYpG2_>o^!nT!2t!&DlZ;Y}yyGb(Uh=neLj0;GKPhP+(2vU?={*j} zbJFpY>6zlSqquqFt!Umf97KsXegqYIcy!_Y8@L%vyxegk4gq`$Z(IPkK$GMoWLiE> zyvd!{^&T?kKHE6p=I>`U!of7BeuuAIkPcT$i7i*dGu|{5 z=$J7&P5-ZQPrl{c_+=ic;L-0|B?~E!+$nhD(Ge+_33C zkCIGC>a4Tq7g4>+qr=`{u!tZ5IHt~;Ymn4;d>%A*x{di@Uk2Cz#ENUbNA_F5sK$@xX%K8y>;7Es1BgqXtqDnfS zC!`Hd-5kJw{Mmx49Q=p(3szCcJq|nr%&E*(X#me0W200QWaB;c>b-L9jnx$k;-6$+ zX3au{MIjRhJ6tOZv3KxNerb!J+;Nt_9!1?5t=n}_pER_*ZnaNBL#{9MyX2kNEgnv} z=fO-#2cUxDHD)9 z!aGLio=G!&4GO@Emo!Kr6C1DmgNZ=`MBbD_X| zlYQ7Bu?fm2B_E2$iHE>0;A*#AF$KJQ!(GEzgo-_U>AZcLhhAvz zozs}`6Cyj4k>lE_V&FnGQo84y$G0!WM_L2AQ9ZYcT>+nsCXSin4csfh!A3vMzEDq% zcYk1)4thE&3610Y^12yz$=@XCm|wkS_at*&s-XFD=fL_PkwEP0MNnc33uI~189Yl) zP3Sh*@4TD$B>s-ECw@dN05n;I0E{J#8n?fp_lD&0*@H@BRb_Ex?>x7%W$8&5D{uB( zjr}sgXQ%P*GfSdIDcm!{>4nwchZF}jS;6aJvlxUL7|CbMTm$kKHWo+QQ=z3mt-MXw zuDGYDQ{6_y5gXBa%F>3q9?+;Rp`b_hnF77O&F+e@J!yq}7e3y~{$Nac=u}^I-mPpW z2{h@6mCoSZksz4X-?*Ch%EV5J34aAA&!vio)D84ni+Wx; z8+WyBpZ91_b;k{JQWky}Wnyl?DRxCBXmxtHVKm>V{RfAaHqW-;S)k(XOtE=x4kiVC zx>pl-i|6@XHr*RlH?k004|(XHl6ntg+@%xGOd?6)TOk9mo17^zz<+Ql^Do$tgPIVP zGNX;sG^^l8-B`W*owg5NuTs2SAbeayxLPXOD--Ns*nFA2s_O#Il$)M z?bD5yCb+$LD~dJS8YxSAP#f*XTYs^;`csuixn5VBkBO9sS&NwwMkpMSLu|M7L4t*i zJ`T$8i43q~b+G-(s?6v-hLRU9HZ}4Zq1?sQ6IGZIty{hF`FcfLll<@U6(Yhf2<=3^ zKv2_5nVN7KwoC_)FxV}Fr;d%DS$2i0Z%U*+z9f=ab|YzSsI)5 z*B5k4nKNWb_;dO__RejDN`;~981KOSQFGk8X@D7RBnx^Io8Ye4`3OCkR-XT4wOiafJ#g^5~(v_Dg^Oa+CyG-OTPF6dSk;xVc{Sp(51>;jk(R6pX zB~jpXO#>B`XYvrHrF?fs=$@K_QNowKV{Z%AyR9PSCe9XJn79W@{AY?gd_duJbO`f2 z^@LB{h$PZ_2&)A*T`mo8Dgyc{uN{yjt~BPdEu0MA=Wn843kfIY%D>_2*hm>*Dg zmr0D5@f{MbMklb1(d)tvp4 zo=|a2lL_MSc~Y5?=)T9{o_Y9Re&&JQ`poIOpH;@BxIatJ!`Cn2%quEPBY6;hVR7{lcr5?@f{EY-eG)Lp)C8lz-pvbmP&*bwOD s=NG7x&}Qj7WU->i{>2XhEA9?$R#i!XgtlEZ=oe0=jl&+yAKyLt4-P)b0ssI2 literal 0 HcmV?d00001 diff --git a/fbCollector/requirements.txt b/fbCollector/requirements.txt new file mode 100644 index 00000000..26e2479d --- /dev/null +++ b/fbCollector/requirements.txt @@ -0,0 +1,8 @@ +# requirements for the fbCollector tool +# Copyright (C) 2018-2023 Mark McIntyre +boto3 +Pillow +paramiko +meteortools +scp +pyqtgraph diff --git a/fbCollector/ukmda.ico b/fbCollector/ukmda.ico new file mode 100644 index 0000000000000000000000000000000000000000..e788188d8dc5d1b3e5284da0e908c9830808bbac GIT binary patch literal 16958 zcmds930M?Ywmyl8iBVscH*u1gyqK70MveOpvM&t{$f_XwCJ3@9vM4G_UXpP%iMv2D zipFO&gfS{AxS^=XJRQ`GTNFVAWQV4q_pYiwb8chLM|?38Cp4{=-_Pl)uDbXBXS?U# z?iSz~`e$PU^!FVYVFxe-07ek#5@ZtCb^T!j@Sg*_(S5-ZtW|>b z|1^Ws4*oTVElPX4HL^_WmD0nO$&!j0^V_86kuAJgcng}(ZQtKWFV!?jB2JEuSLA1FGoWD?-*`!KtjsL@4;#`RzyhIq-i(u_hyW`)#i_}jA z^*7aNvkaWFq+lIg*XM5V?&ekyXDPrrTLI#2>XCNKIxbHn`!LR<2N764>upfGto?~& zYS4bAw9F+}^Iy(cN)T^R_c?!;94&~mRp9!S_7%rJb{0&YR@cq zQz;*a*2=Qn^YD4bU-Y@6&vf9i1Hm<$2kXUmz`RP&{SI2qZI!uf zRl16^<=~d9>@)r|zu>^-3nk3lsea!sO>$@a+toPcRpS3zAAb^}4{(X=K-6_|2=BcOK#JQ{XMoRwpjW! zcJrnfWN~N`Q2v=ZxfWg5b`{P!P2<{X+&aMfYc+W9R*(1CF0Zx;ZWbmEqxHHe@hc0IPt$t1nPV0) zO8cVc4&(9YS-NLy6y|yd1O!vGBvvwPIKfwPI<-c!L z;CDdd@OfA~e z=LdfI>gAqWB@k3#TO}9I9T0pF!7j7JsJso0ha4gPq=TSB0Rj(cA-I@(b5^eWuhyQ8-TbG}=QqX#)jJMz z$?x-&yFyBYL2L|x2aG%K8Myb9L%(PS{{u48ff|C3X|24s$ZD+I8gM#spHBOsy@sjO z76WUOqk)RFD0T%Z4$ZS8u<`;tQwRUenpY@%R zkV5sQ@Kf3~{(BVKzGSq}LGtGaLlucS7uS z92{LPjF?--2L)HG_!-lxiXH$;^(wE1_Qcc8ZZOmxbix5%5Igk#PN1o+}Mw}F0 zh&aVhiX)u|9aF+5B_`s2S_iS`k?gI8=nMRN;l-Maz&vH??5wUPkB>XlqUaW#rKnz? zr&+H4#USpsRQ?Y*+NWqcX`M+JLMM72yBtJZvZ?xjvWT zKJEtpT2P+idpo}-w4T{W^VK7Qp0RT4#)HqZ>)wBo@-Y{9zo>Is$gH|&5}pt1H2NF| ziIq4gbib;U+27iOxHHgv_lrRHvYg?yN-B7LA{Fcd@3&!io^$;x0h`Wezm_A#II;GhvrW)LKRwLT^HzD)ee+jQfWPF2mD{5me@lmlC`4qn#S(oi> zdY;~6&odpmQmr@doOVsXUO8mkHqAWOlXK742^SDz{!a_B<$UPu4V^l_tS))zcgl9> z`K>g5;xX%I;M0U6TC+tSRTnQ;WB(P1&(A%sKE!hVKc6KXc&wCQ%y}*?>Z}G9UlDqC zZ}QjkE(7Lo10D*pvz>o427V=PPq-$SMV?ec zOqq7A8`Td?Z_E$av;KdUDdMjR_Vdd*sKz{#bN&M)Veg%v6x=C9mF#$T^9dT-_uiv1EzK~g0SjK3tn=QVxvJ)R$N z{J9h2pFloIcq8HOymfLNzC!jdb(+_Tc5}Mz%WVA9XhMe&TEpkabn%z@sMUziMHOob zS?sX;-Iq!*vPgY3@wzY}{(I8##mCF{^=cvh^RMs_dyNA&#$3vUF{Ro(*F|k;7tpNF zZ!E|C6jSeq}1Y@QODGmS^V1>c+Zo-%1&TIb?zE6n@uJM0jJm`Gy%=C72B(7|K+5lGfuxt9- z?v&;_>2BDf3ew@7CzF5DZ2^+2CLxQ5{Rus{k_#hmk zTG4l77cRUmtQzvpO-QSHGWjQ7L5f`tm<>OrIdDlG4hFaci*5=ZviN7bSzh>mpxo%1 zOnKn6<_hJ{*Hfw32hivq-t2! zhQrhDKDqpR>MFCH=|Kk>QXG_ckvp!R2jkk1as3hE+fclq4Wo~$>z8-n@Wr)H2Y;rE zRN_zh0Ekm?=pF^fKBLlu_&OUv*Ia3RA6KfmwxSD%EP0yvR|>GCPVWP+CtTpl-DzGF zFC=>#kkqXQpEXikbc5gh8P$Qy>YvQ~*VFe6ueZ|L;G-@Css$jeTKCDEtS%H2XO~6o zdWe~%y^Y8Ml{&ovz0+!RklOh4_&?JF)dbA{A-z!-5V~8T6H~5doDB%fm0`+lVdu&& z9FX3igUly%{x{gXr_89)Li&CFOB*mQc&8l2$z*@3d);a66PBmIlnQ=-MhCvNv`Gg` z?mb<$&uB$h){L-HioZ>{C7hu(0s3y}#ISD_eBuRdMS49h&6MG5>>kGbj}1>x>@z>e zXw$)pHr>EwO}J@EjZT()ov&J4tt(jGiVHqe;QPMxn-qEtOB;!Q%j3lzk$yMXkA7K> zQu>bqn7m7jm@FZ=l4K@HYmy{jD5BnV4f16~{nt(m@`d~74@&Ale;-Nx&op~mKG>h( zoW`XeH Date: Mon, 11 Aug 2025 17:17:36 +0100 Subject: [PATCH 07/83] bugfix to prevent mergeNewOrbit running twice at the same time --- archive/cronjobs/mergeNewOrbit.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/archive/cronjobs/mergeNewOrbit.sh b/archive/cronjobs/mergeNewOrbit.sh index cfdf264b..09335d20 100644 --- a/archive/cronjobs/mergeNewOrbit.sh +++ b/archive/cronjobs/mergeNewOrbit.sh @@ -5,7 +5,11 @@ source $here/../config.ini >/dev/null 2>&1 conda activate $HOME/miniconda3/envs/${WMPL_ENV} mkdir -p $DATADIR/manualuploads -cd $DATADIR/manualuploads +cd $DATADIR/manualuploadsif [ -f ./running ] ; then + echo "already running" + exit +fi +touch .running aws s3 sync $UKMONSHAREDBUCKET/fireballs/uploads . --exclude "*" --include "*.zip" --exclude "*.done" newfiles=$(ls -1 *.zip 2> /dev/null) @@ -57,4 +61,5 @@ EOD #else # echo nothing to process fi +rm $DATADIR/manualuploads/.running find $SRC/logs -name "mergeNewOrbit*" -mtime +10 -exec rm -f {} \; \ No newline at end of file From 1bc72319953480dfa449a66b400629c6627830f9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 11 Aug 2025 17:17:48 +0100 Subject: [PATCH 08/83] gbfollector config file --- fbCollector/config.ini | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 fbCollector/config.ini diff --git a/fbCollector/config.ini b/fbCollector/config.ini new file mode 100644 index 00000000..9dfab68b --- /dev/null +++ b/fbCollector/config.ini @@ -0,0 +1,30 @@ +# tailor this config file to your needs +# $HOME will be translated into your homedir (~ on linux, $env:userprofile on windows) +[Fireballs] +basedir=$HOME/OneDrive/pictures/meteors/fireballs + +livebucket=ukmda-live +uploadbucket=ukmda-shared +uploadfolder=fireballs/uploads +ukmonprofile=ukmonshared + +[gmnconnection] +# the GMN key needs to be configured with Denis Vida +# if you have permission you can then collect data from the GMN server +gmnkey=~/.ssh/gmnanalysis +gmnserver=gmn.uwo.ca +gmnuser=analysis + +[reduction] +# To use the reduction options RMS must be installed and an RMS python conda environment created +rms_loc=$HOME/OneDrive/dev/RMS +rms_env=RMS + +[solver] +# To use the solver options WMPL must be installed and a WMPL python conda environment created +wmpl_loc=$HOME/OneDrive/dev/WesternMeteorPyLib +wmpl_env=ukmon-shared + +[sharing] +# create this folder if you want to share raw data with other UKMON members +shrfldr=$HOME/Dropbox/Raw_Data \ No newline at end of file From 10d02017c52693149eefe724858bc806e9b25e23 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 11 Aug 2025 17:18:15 +0100 Subject: [PATCH 09/83] helperfunctions used by various powershell scripts --- utils/helperfunctions.ps1 | 125 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 utils/helperfunctions.ps1 diff --git a/utils/helperfunctions.ps1 b/utils/helperfunctions.ps1 new file mode 100644 index 00000000..4f1c77a8 --- /dev/null +++ b/utils/helperfunctions.ps1 @@ -0,0 +1,125 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +# helper functions + +Function Get-IniContent { + <# + .Synopsis + Gets the content of an INI file + + .Description + Gets the content of an INI file and returns it as a hashtable + + .Notes + Author : Oliver Lipkau + Blog : http://oliver.lipkau.net/blog/ + Source : https://github.com/lipkau/PsIni + http://gallery.technet.microsoft.com/scriptcenter/ea40c1ef-c856-434b-b8fb-ebd7a76e8d91 + Version : 1.0 - 2010/03/12 - Initial release + 1.1 - 2014/12/11 - Typo (Thx SLDR) + Typo (Thx Dave Stiff) + + #Requires -Version 2.0 + + .Inputs + System.String + + .Outputs + System.Collections.Hashtable + + .Parameter FilePath + Specifies the path to the input file. + + .Example + $FileContent = Get-IniContent "C:\myinifile.ini" + ----------- + Description + Saves the content of the c:\myinifile.ini in a hashtable called $FileContent + + .Example + $inifilepath | $FileContent = Get-IniContent + ----------- + Description + Gets the content of the ini file passed through the pipe into a hashtable called $FileContent + + .Example + C:\PS>$FileContent = Get-IniContent "c:\settings.ini" + C:\PS>$FileContent["Section"]["Key"] + ----------- + Description + Returns the key "Key" of the section "Section" from the C:\settings.ini file + + .Link + Out-IniFile + #> + + [CmdletBinding()] + Param( + [ValidateNotNullOrEmpty()] + [ValidateScript({(Test-Path $_) -and ((Get-Item $_).Extension -eq ".ini")})] + [Parameter(ValueFromPipeline=$True,Mandatory=$True)] + [string]$FilePath + ) + + Begin + {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function started"} + + Process + { + Write-Verbose "$($MyInvocation.MyCommand.Name):: Processing file: $Filepath" + + $ini = @{} + switch -regex -file $FilePath + { + "^\[(.+)\]$" # Section + { + $section = $matches[1] + $ini[$section] = @{} + $CommentCount = 0 + } + "^(;.*)$" # Comment + { + if (!($section)) + { + $section = "No-Section" + $ini[$section] = @{} + } + $value = $matches[1] + $CommentCount = $CommentCount + 1 + $name = "Comment" + $CommentCount + $ini[$section][$name] = $value + } + "(.+?)\s*=\s*(.*)" # Key + { + if (!($section)) + { + $section = "No-Section" + $ini[$section] = @{} + } + $name,$value = $matches[1..2] + $ini[$section][$name] = $value + } + } + Write-Verbose "$($MyInvocation.MyCommand.Name):: Finished Processing file: $FilePath" + Return $ini + } + + End + {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function ended"} +} + +Function Test-CommandExists +{ + Param ($command) + $oldPreference = $ErrorActionPreference + $ErrorActionPreference = ‘stop’ + try {if(Get-Command $command){RETURN $true}} + Catch {Write-Host “$command does not exist”; RETURN $false} + Finally {$ErrorActionPreference=$oldPreference} +} #end function test-CommandExists + +Function New-TemporaryFolder { + # Make a new folder based upon a TempFileName + $T="$($Env:temp)\tmp$([convert]::tostring((get-random 65535),16).padleft(4,'0')).tmp" + New-Item -ItemType Directory -Path $T +} \ No newline at end of file From 1a7b46e3d9d81d3f11d5a4e944d8e177bae49ea4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 14 Aug 2025 10:05:25 +0100 Subject: [PATCH 10/83] fix powershell scripts to be machine-independent --- utils/addVidorImg.ps1 | 3 ++- utils/analysis.ini | 29 +++++++++++++++++++++++++++++ utils/getBrightest.ps1 | 20 ++++++++++++++------ utils/uploadOrbit.ps1 | 2 +- 4 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 utils/analysis.ini diff --git a/utils/addVidorImg.ps1 b/utils/addVidorImg.ps1 index a3bc5757..933c75e4 100644 --- a/utils/addVidorImg.ps1 +++ b/utils/addVidorImg.ps1 @@ -15,7 +15,8 @@ set-location $PSScriptRoot # load the helper functions . .\helperfunctions.ps1 $ini=get-inicontent analysis.ini -$fbfolder = $ini['localdata']['fbfolder'] +$fbfolder = $ini['localdata']['fbfolder'].replace('$HOME',$home) +$fbfolder = $fbfolder + "/fireballs" set-location $fbfolder $traj = (split-path -leaf $pth) diff --git a/utils/analysis.ini b/utils/analysis.ini new file mode 100644 index 00000000..5be0f402 --- /dev/null +++ b/utils/analysis.ini @@ -0,0 +1,29 @@ +# Parameters that define your local configuration +# +# +[pylib] +pylib=$HOME/OneDrive/dev/ukmda-dataprocessing/archive/ukmon_pylib + +[localdata] +FBFOLDER=$HOME/OneDrive/pictures/meteors +MTHLYFOLDER=$HOME/OneDrive/pictures/meteors/mthlystacks + +[rms] +RMS_INSTALLED=1 +RMS_LOC=$HOME/OneDrive/dev/RMS +RMS_ENV=RMS + +[wmpl] +WMPL_INSTALLED=1 +WMPL_LOC=$HOME/OneDrive/dev/WesternMeteorPyLib +WMPL_ENV=wmpl + +[gmn] +IDFILE=~/.ssh/gmnanalysis + +# camid to hostname mappings, yours may be different +[stations] +UK0006=uk0006 +UK000F=uk000f +UK002F=uk002f +UK001L=uk001l diff --git a/utils/getBrightest.ps1 b/utils/getBrightest.ps1 index a40442b2..1cbe529e 100644 --- a/utils/getBrightest.ps1 +++ b/utils/getBrightest.ps1 @@ -5,17 +5,24 @@ # requirements - you must clone WesternMeteorPyLib and ukmda-dataprocessing from github # to your local development space. I use onedrive\dev for my code - set this location in $codeloc -# you also need to specify the output folder, $outdir # copyright (c) Mark McIntyre, 2025- -$codeloc ="$env:userprofile\onedrive\dev" -$outdirw = "$env:userprofile\pictures\_ToBeProcessed\brightest" +set-location $PSScriptRoot -$repdir = "$codeloc\ukmda-dataprocessing\archive\ukmon_pylib" -$outdir = $outdirw.replace('\','/') +# load the helper functions +. .\helperfunctions.ps1 +$ini=get-inicontent analysis.ini +$bdir = $ini['localdata']['fbfolder'].replace('$HOME',$home) +$bdir = $bdir + "/brightest" +set-location $bdir -$env:pythonpath="$codeloc\WesternMeteorPyLib" +$outdir = $bdir.replace('\','/') + +$wmplloc = $ini['wmpl']['wmpl_loc'].replace('$HOME',$home) +$repdir = $ini['pylib']['pylib'].replace('$HOME',$home) + +$env:pythonpath="$wmplloc" Push-Location $repdir conda activate ukmon-shared @@ -28,6 +35,7 @@ if ($args.count -eq 0) { python -c "from reports.findBestMp4s import getBestNSingles;getBestNSingles(numtoget=30,outdir='$outdir', reqdate='$reqdate')" } +$outdirw = $outdir.replace('/','\') explorer "$outdirw" Pop-Location \ No newline at end of file diff --git a/utils/uploadOrbit.ps1 b/utils/uploadOrbit.ps1 index e470d096..db92db60 100644 --- a/utils/uploadOrbit.ps1 +++ b/utils/uploadOrbit.ps1 @@ -17,7 +17,7 @@ Set-Location $Loc Add-Type -AssemblyName System.Windows.Forms conda activate $ini['wmpl']['wmpl_env'] -$wmplloc = $ini['wmpl']['wmpl_loc'] +$wmplloc = $ini['wmpl']['wmpl_loc'].replace('$HOME',$home) $env:pythonpath="$wmplloc;$env:pythonpath" write-output "Find the pickle" From e23c6f4dd67b4491b77e67c12a69e26b5fe5ef65 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 25 Aug 2025 11:37:39 +0100 Subject: [PATCH 11/83] automate DNS management --- .../updateDnsRecords/lambda_function.py | 130 ++++++++++++++++++ .../updateDnsRecords/samconfig.toml | 14 ++ .../updateDnsRecords/template.yaml | 33 +++++ 3 files changed, 177 insertions(+) create mode 100644 archive/samfunctions/updateDnsRecords/lambda_function.py create mode 100644 archive/samfunctions/updateDnsRecords/samconfig.toml create mode 100644 archive/samfunctions/updateDnsRecords/template.yaml diff --git a/archive/samfunctions/updateDnsRecords/lambda_function.py b/archive/samfunctions/updateDnsRecords/lambda_function.py new file mode 100644 index 00000000..0a855287 --- /dev/null +++ b/archive/samfunctions/updateDnsRecords/lambda_function.py @@ -0,0 +1,130 @@ +import os +import boto3 +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +""" Default values """ +DNS_TAG_KEY = 'Route53FQDN' +DNS_RECORD_TYPE = 'CNAME' +DNS_RECORD_TTL = 60 + + +def update_route53_record(region, instance_id, dns_tag_key, dns_record_type, dns_record_ttl): + """ + update function + """ + dns_record_type = dns_record_type.upper() + + """ Get EC2 Description """ + ec2 = boto3.client('ec2', region_name=region) + ec2_desc_res = ec2.describe_instances(InstanceIds=[instance_id]) + if ('Reservations' not in ec2_desc_res) or (len(ec2_desc_res['Reservations']) < 1) or (len(ec2_desc_res['Reservations'][0]['Instances']) < 1): + logger.warning('Error hetting EC2 instance information failure. Instance Id: ' + instance_id) + return + if (ec2_desc_res['Reservations'][0]['Instances'][0]['State']['Name'] != 'running'): + logger.warning('EC2 state is not running. Current state: ' + ec2_desc_res['Reservations'][0]['Instances'][0]['State']['Name']) + return + ec2_public_dns = ec2_desc_res['Reservations'][0]['Instances'][0]['PublicDnsName'] + ec2_public_ip = ec2_desc_res['Reservations'][0]['Instances'][0]['PublicIpAddress'] + + """ Get EC2 Tags """ + ec2_tags_res = ec2.describe_tags(Filters=[ + {'Name': 'resource-id', 'Values': [instance_id]}, + {'Name': 'key', 'Values': [dns_tag_key]}]) + + if ('Tags' not in ec2_tags_res) or (len(ec2_tags_res['Tags']) < 1): + logger.warning('EC2 instance has no DNS tag. Instance Id: ' + instance_id) + return + + dns_record_name = ec2_tags_res['Tags'][0]['Value'] + if (dns_record_name[-1] == '.'): + dns_record_name = dns_record_name[:-1] + + ec2_tags_res = ec2.describe_tags(Filters=[ + {'Name': 'resource-id', 'Values': [instance_id]}, + {'Name': 'key', 'Values': ['DNSRecordType']}]) + if ('Tags' not in ec2_tags_res) or (len(ec2_tags_res['Tags']) < 1): + logger.warning('EC2 instance has no Record Type. Instance Id: ' + instance_id) + else: + dns_record_type = ec2_tags_res['Tags'][0]['Value'] + + """ Check DNS name """ + dns_record_names = dns_record_name.split('.') + if (len(dns_record_names) < 3): + logger.warning('The FQDN must over 3 levels at least. Instance Id: ' + instance_id + ', ' + dns_tag_key + ': ' + dns_record_name) + return + """ Get all hosted zones from Route53 """ + route53 = boto3.client('route53', region_name=region) + zones_res = route53.list_hosted_zones() + all_zones = zones_res['HostedZones'] + if (zones_res['IsTruncated']): + while zones_res and (not zones_res['IsTruncated']): + zones_res = route53.list_hosted_zones(Marker=zones_res['NextMarker']) + all_zones = all_zones + zones_res['HostedZones'] + + """ Find Route53 zone """ + zone_id = None + for i in range(1, len(dns_record_names)-1): + cur_zone_name = '.'.join(dns_record_names[i:len(dns_record_names)+1]) + '.' + cur_zone_matches = [zone for zone in all_zones if (zone['Name']==cur_zone_name)] + if (len(cur_zone_matches) > 0): + zone_id = cur_zone_matches[0]['Id'] + break + if (not zone_id): + print('Route53 zone is not found. DNS name: ' + dns_record_name) + return + + """ Update Route53 record record """ + route53.change_resource_record_sets( + HostedZoneId=zone_id, + ChangeBatch={ + 'Changes': [ + { + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'Name': dns_record_name + '.', + 'Type': dns_record_type, + 'ResourceRecords': [{'Value': (ec2_public_dns if (dns_record_type=='CNAME') else ec2_public_ip)}], + 'TTL': dns_record_ttl + } + } + ] + } + ) + logger.info('The Route53 record updated successfully. ' + dns_record_name + ': [' + dns_record_type + '] ' + (ec2_public_dns if (dns_record_type=='CNAME') else ec2_public_ip)) + return + + +def lambda_handler(event, context): + """ + main function + """ + region = os.environ.get("region") + instance_id = os.environ.get("instance_id") + dns_tag_key = os.environ.get("dns_tag_key") if os.environ.get("dns_tag_key") else DNS_TAG_KEY + dns_record_type = os.environ.get("dns_record_type") if os.environ.get("dns_record_type") else DNS_RECORD_TYPE + dns_record_ttl = int(os.environ.get("dns_record_ttl")) if os.environ.get("dns_record_ttl") else DNS_RECORD_TTL + + if (('region' in event) and event['region']): + region = str(event['region']) + if (not region): + print('Parameter "region" is not defined.') + return + + if (('detail' in event) and ('instance-id' in event['detail']) and event['detail']['instance-id']): + instance_id = str(event['detail']['instance-id']) + if (not instance_id): + print('Parameter "instance-id" is not defined.') + return + + if (('state' in event['detail']) and event['detail']['state'] and (event['detail']['state'] != 'running')): + logger.warning('EC2 state is not running. Current state: ' + event['detail']['state']) + return + + try: + update_route53_record(region, instance_id, dns_tag_key, dns_record_type, dns_record_ttl) + except Exception as error: + logger.exception(error) + return diff --git a/archive/samfunctions/updateDnsRecords/samconfig.toml b/archive/samfunctions/updateDnsRecords/samconfig.toml new file mode 100644 index 00000000..2703b7f3 --- /dev/null +++ b/archive/samfunctions/updateDnsRecords/samconfig.toml @@ -0,0 +1,14 @@ +# Copyright (C) 2018-2023 Mark McIntyre +version = 0.1 +[default] +[default.deploy] +[default.deploy.parameters] +stack_name = "updateDnsRecords" +s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-1ulna2hoktme4" +s3_prefix = "updateDnsRecords" +region = "eu-west-2" +confirm_changeset = true +capabilities = "CAPABILITY_IAM" +disable_rollback = true +image_repositories = [] +profile = "ukmda_admin" \ No newline at end of file diff --git a/archive/samfunctions/updateDnsRecords/template.yaml b/archive/samfunctions/updateDnsRecords/template.yaml new file mode 100644 index 00000000..f23b62b1 --- /dev/null +++ b/archive/samfunctions/updateDnsRecords/template.yaml @@ -0,0 +1,33 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: AWS Lambda function for updating Route53 record when public ip of EC2 instance. + +Resources: + updateDnsRecords: + Type: AWS::Serverless::Function + Properties: + FunctionName: updateDnsRecords + MemorySize: 128 + Handler: lambda_function.lambda_handler + Timeout: 10 + Policies: + - Statement: + - Action: + - ec2:DescribeInstances + - ec2:DescribeTags + - route53:ListHostedZones + - route53:ChangeResourceRecordSets + Resource: '*' + Effect: Allow + Runtime: python3.11 + Events: + EC2StateChangeEvent: + Type: CloudWatchEvent + Properties: + Pattern: + source: + - aws.ec2 + detail: + state: + - running + Description: AWS Lambda function for updating Route53 record when public ip of EC2 instance. \ No newline at end of file From 7df6d528b12cc9bc9035f234f7ba6c80d7da448a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 25 Aug 2025 11:37:59 +0100 Subject: [PATCH 12/83] add R53 tags to ukmonhelper2 --- archive/terraform/mjmm/ec2.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/archive/terraform/mjmm/ec2.tf b/archive/terraform/mjmm/ec2.tf index 2e396ea2..181ef241 100644 --- a/archive/terraform/mjmm/ec2.tf +++ b/archive/terraform/mjmm/ec2.tf @@ -34,6 +34,8 @@ resource "aws_instance" "ukmonhelper_g" { "Name" = "UKMonHelper2" "billingtag" = "ukmon" "project" = "UKMonHelper2" + "Route53FQDN" = "ukmonhelper2.markmcintyreastro.co.uk" + "DNSRecordType" = "A" } } From 572d20fe99ac7d387d41fce65b868a0c347e90c8 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 25 Aug 2025 11:38:26 +0100 Subject: [PATCH 13/83] add r53 tags to calcengines and adminserver --- archive/terraform/ukmda/ec2.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/ec2.tf index 8bcb950e..f91de92c 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/ec2.tf @@ -33,6 +33,8 @@ resource "aws_network_interface" "calcserver_if" { tags = { "Name" = "calcengine" "billingtag" = "ukmda" + "Route53FQDN" = "calcengine.ukmeteors.co.uk" + "DNSRecordType" = "A" } } ################################################ @@ -47,6 +49,8 @@ resource "aws_instance" "ubuntu_calc_server" { tags = { "Name" = "Calcengine2" "billingtag" = "ukmda" + "Route53FQDN" = "calcengine2.ukmeteors.co.uk" + "DNSRecordType" = "A" } root_block_device { tags = { @@ -88,6 +92,8 @@ resource "aws_instance" "admin_server" { tags = { "Name" = "AdminServer" "billingtag" = "ukmda" + "Route53FQDN" = "adminserver.ukmeteors.co.uk" + "DNSRecordType" = "A" } root_block_device { tags = { From 453cb8c67fe3999a9da700ab52a084e3825c6c5d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 25 Aug 2025 11:46:08 +0100 Subject: [PATCH 14/83] bugfix to handle inability to delete folders on cloud storage eg onedrive --- fbCollector/fireballCollector.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 649b247c..000bff87 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -623,11 +623,16 @@ def archiveFolder(self): os.makedirs(archdir, exist_ok=True) zfname = os.path.join(archdir, fldr) shutil.make_archive(zfname,'zip',self.fb_dir, fldr) + except Exception as e: + log.warning(f'unable to create archive {self.dir_path}') + log.warning(e) + try: + os.chdir(self.fb_dir) shutil.rmtree(self.dir_path) - self.dir_path = self.fb_dir except Exception as e: - log.warning(f'unable to archive {self.dir_path}') + log.warning(f'unable to remove folder, please do it manually {self.dir_path}') log.warning(e) + self.dir_path = self.fb_dir def delFolder(self): noimgdata = img.open(noimg_file).resize((640,360)) From 377a395f6a33c554db4c0360475990c4ae9cf9b7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 26 Aug 2025 12:10:13 +0100 Subject: [PATCH 15/83] add function to compress older data in the "archive" area --- .../maintenance/compressOldArchiveData.py | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 archive/ukmon_pylib/maintenance/compressOldArchiveData.py diff --git a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py new file mode 100644 index 00000000..bac1ec39 --- /dev/null +++ b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py @@ -0,0 +1,149 @@ +# copyright 2023- Mark McIntyre +# all rights reserved + +""" + a function to compress the ukmda-shared/archive data from before the current year + +usage: + 1) python compressOldArchiveData.py - this will scan and compress ALL data from prior years + 2) python compressOldArchiveData.py "archive/somefolder" will scan and compress only the named location folder + +note that any MP4 or AVI files that have been uploaded to this area are removed by this process. The videos should be +on the website in ukmda-website/img/mp4s in mp4 format +""" + +import boto3 +import zipfile +import io +import sys + +s3 = boto3.client('s3') +s3res = boto3.resource('s3') + + +def listLocations(srcbucket='ukmda-shared', prefixstr='archive/'): + """ + list all top level folders in a particular prefix + parameters: + srcbucket: [string] bucket name, default ukmda-shared + prefixstr: [string] prefix, default archive/ - note must have terminal slash + + returns: + list of prefixes + """ + response = s3.list_objects_v2(Bucket=srcbucket, Prefix=prefixstr, Delimiter='/') + if 'CommonPrefixes' in response: + return response['CommonPrefixes'] + else: + return [] + + +def listCameras(srcbucket='ukmda-shared', location='archive/Tackley/'): + response = s3.list_objects_v2(Bucket=srcbucket, Prefix=location, Delimiter='/') + if 'CommonPrefixes' in response: + return response['CommonPrefixes'] + else: + return [] + + +def listYears(srcbucket='ukmda-shared', camid='archive/Tackley/c1/'): + response = s3.list_objects_v2(Bucket=srcbucket, Prefix=camid, Delimiter='/') + if 'CommonPrefixes' in response: + return response['CommonPrefixes'] + else: + return [] + + +def getAllS3Objects(s3, **base_kwargs): + continuation_token = None + while True: + list_kwargs = dict(MaxKeys=1000, **base_kwargs) + if continuation_token: + list_kwargs['ContinuationToken'] = continuation_token + response = s3.list_objects_v2(**list_kwargs) + yield from response.get('Contents', []) + if not response.get('IsTruncated'): # At the end of the list? + break + continuation_token = response.get('NextContinuationToken') + + +def deleteFiles(flist, srcbucket='ukmda-shared'): + chunk_size = 900 + chunked_list = [flist[i:i + chunk_size] for i in range(0, len(flist), chunk_size)] + for ch in chunked_list: + delete_keys = {'Objects': []} + delete_keys['Objects'] = [{'Key': k} for k in ch] + s3.delete_objects(Bucket=srcbucket, Delete=delete_keys) + return + + +def compressObjects(srcbucket='ukmda-shared', camprefix=None): + lcmprefixes = [] + if camprefix is None: + locations = listLocations() + for location in locations: + camids = listCameras(location=location['Prefix']) + for camid in camids: + years = listYears(camid=camid['Prefix']) + for year in years: + lcmprefix = year['Prefix'][:-1] + lcmprefixes.append(lcmprefix) + else: + spls = camprefix[:-1].split('/') + print(spls) + if len(spls) < 3: + camids = listCameras(location=camprefix) + for camid in camids: + years = listYears(camid=camid['Prefix']) + for year in years: + lcmprefix = year['Prefix'][:-1] + lcmprefixes.append(lcmprefix) + else: + years = listYears(camid=camprefix) + for year in years: + lcmprefix = year['Prefix'][:-1] + lcmprefixes.append(lcmprefix) + + for lcmprefix in lcmprefixes: + spls = lcmprefix.split('/') + zfname = f'{spls[0]}/{spls[1]}/{spls[1]}_{spls[2]}_{spls[3]}.zip' + try: + if int(spls[3]) > 2024 or 'miscellan' in spls[3] or 'profiles' in spls[3]: + continue + except Exception: + pass + if 'testpi' in lcmprefix: + continue + bucket = s3res.Bucket(srcbucket) + files = [os.key for os in bucket.objects.filter(Prefix=lcmprefix)] + mp4files = [file for file in files if '.mp4' in file] + deleteFiles(mp4files) + avifiles = [file for file in files if '.avi' in file] + deleteFiles(avifiles) + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zipf: + for key in files: + if '.mp4' not in key and '.avi' not in key: + print(key) + data = s3.get_object(Bucket=srcbucket, Key=key) + content = data['Body'].read() + zipf.writestr(key, content) + + # Upload the zip file to the target bucket + zip_buffer.seek(0) + print(f'uploading {zfname}') + s3.upload_fileobj(zip_buffer, srcbucket, zfname) + # now delete the files + print(f'deleting files from {lcmprefix}') + deleteFiles(files) + + +if __name__ == '__main__': + source_bucket = 'ukmda-shared' + prefix_str = None + if len(sys.argv) > 1: + prefix_str = sys.argv[1] + if prefix_str[-1] != '/': + prefix_str = prefix_str + '/' + print(f'compressing {prefix_str}') + compressObjects(source_bucket, prefix_str) From 8e466866e44393e26ef97b4d43f4cc4f0915ce10 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 26 Aug 2025 12:22:25 +0100 Subject: [PATCH 16/83] parameterise cutoff year --- archive/ukmon_pylib/maintenance/compressOldArchiveData.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py index bac1ec39..efa3fce9 100644 --- a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py +++ b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py @@ -16,6 +16,9 @@ import zipfile import io import sys +import datetime + +maxyear = datetime.datetime.now().year - 1 s3 = boto3.client('s3') s3res = boto3.resource('s3') @@ -108,7 +111,7 @@ def compressObjects(srcbucket='ukmda-shared', camprefix=None): spls = lcmprefix.split('/') zfname = f'{spls[0]}/{spls[1]}/{spls[1]}_{spls[2]}_{spls[3]}.zip' try: - if int(spls[3]) > 2024 or 'miscellan' in spls[3] or 'profiles' in spls[3]: + if int(spls[3]) > maxyear or 'miscellan' in spls[3] or 'profiles' in spls[3]: continue except Exception: pass From 64a173477fb8179cb2e58519accbdd45dab36a11 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 26 Aug 2025 15:04:36 +0100 Subject: [PATCH 17/83] avoid crash if solver fails --- fbCollector/fireballCollector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 000bff87..ccbaf598 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -425,7 +425,8 @@ def solveOrbit(self): show_plots=False, v_init_part=velpart, v_init_ht=vinitht, show_jacchia=jacchia, enable_OSM_plot=True, mc_cores=8) tkMessageBox.showinfo('Info', 'Solver Finished') - self.soln_outputdir = traj.output_dir + if traj is not None: + self.soln_outputdir = traj.output_dir return def viewSolution(self): From 19e4564e31a0d58b03b84d5d88fa4d21a40c09ca Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 26 Aug 2025 15:04:52 +0100 Subject: [PATCH 18/83] improve resilience of stale key checker --- archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py index 62af22c1..34115d7a 100644 --- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py +++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py @@ -111,12 +111,17 @@ def getKeyStatuses(excludeadm=False): except ClientError as error: err = error.response['Error']['Code'] if err == 'CredentialReportNotPresentException' or err == 'CredentialReportExpiredException': + print('creating credentials report') res = iamc.generate_credential_report() + time.sleep(60) elif err == 'CredentialReportNotReadyException': + print('report not ready, sleeping 30s') time.sleep(30) gotit = False while not gotit: try: + res = iamc.generate_credential_report() + time.sleep(60) res = iamc.get_credential_report() gotit = True print('got the report') From fe5288181a40c88dc0d4aed329f0104376a62d1a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 27 Aug 2025 22:46:28 +0100 Subject: [PATCH 19/83] remove zip file from orbit pages and upgrade to python 3.13 as 3.9 is obsolete --- .../getExtraFilesV2/pythoncode/getExtraFiles.py | 16 ++-------------- .../getExtraFilesV2/tests/localTest.ps1 | 2 +- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py index 6d4827a7..24174ccc 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py +++ b/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py @@ -5,7 +5,6 @@ import sys import shutil import json -from zipfile import ZipFile, ZIP_DEFLATED from datetime import datetime, timedelta from boto3.dynamodb.conditions import Key @@ -157,7 +156,6 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): print('no extrampgs.html') pass - # pushFilesBack creates the zipfile so we need to do this first print('pushing files back') pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3) print('updating the index page') @@ -239,9 +237,8 @@ def findOtherFiles(evtdt, archbucket, websitebucket, outdir, fldr, s3): def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): - # get filelist before creating the zipfile! - flist = os.listdir(outdir) + flist = os.listdir(outdir) _, pth =os.path.split(outdir) yr = pth[:4] ym = pth[:6] @@ -251,13 +248,9 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): else: webpth = f'reports/{yr}/orbits/{ym}/{pth}/' - zipfname = os.path.join(outdir, pth +'.zip') - zipf = ZipFile(zipfname, 'w', compression=ZIP_DEFLATED, compresslevel=9) - for f in flist: - #print(f) + print(f) locfname = os.path.join(outdir, f) - zipf.write(locfname) # some files need to be pushed to the website, some to the archive bucket if '3dtrack' in f or '2dtrack' in f: key = os.path.join(webpth, f) @@ -277,11 +270,6 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3): extraargs = getExtraArgs(locfname) s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs) - zipf.close() - # now we push the zipfile - key = os.path.join(webpth, pth + '.zip') - extraargs = getExtraArgs(zipfname) - s3.meta.client.upload_file(zipfname, websitebucket, key, ExtraArgs=extraargs) return diff --git a/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 b/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 index 154cf55c..6c33151b 100644 --- a/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 +++ b/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 @@ -1,4 +1,4 @@ # Copyright (C) 2018-2023 Mark McIntyre -sam build +#sam build sam local invoke --profile ukmda_admin -e tests\testEvent.json #sam local invoke --profile ukmda_admin -e tests\testEvent2.json \ No newline at end of file From ab550f6d1fc31877af10dcddda9b4caa4b5895d4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 27 Aug 2025 22:47:04 +0100 Subject: [PATCH 20/83] update dockerfile to python 3.13 --- .../getExtraFilesV2/pythoncode/Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile b/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile index 801df951..26f9034f 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile +++ b/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile @@ -1,16 +1,17 @@ # Copyright (C) 2018-2023 Mark McIntyre -FROM public.ecr.aws/lambda/python:3.9 + +FROM public.ecr.aws/lambda/python:3.13 COPY requirements.txt ./ RUN python -m pip install -r requirements.txt -t . RUN python -m pip install numba watchdog basemap-data-hires -t . RUN python -m pip install cartopy -t . -RUN yum update -y -ENV LD_LIBRARY_PATH ./ +#RUN yum update -y +ENV LD_LIBRARY_PATH=./ COPY WesternMeteorPyLib/ ./WesternMeteorPyLib #COPY *.npy ./WesternMeteorPyLib/wmpl/share/ COPY *.py ./ -ENV PYTHONPATH ./WesternMeteorPyLib -ENV PROJ_LIB ./ +ENV PYTHONPATH=./WesternMeteorPyLib +ENV PROJ_LIB=./ CMD ["getExtraFiles.lambda_handler"] From c767e960597482bf2e6a11ba11a8c830fa7d863a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 27 Aug 2025 22:56:36 +0100 Subject: [PATCH 21/83] minor updates to aws provider 6.10 --- archive/terraform/mjmm/crossacct-role.tf | 6 ++- archive/terraform/mjmm/ec2.tf | 21 ++++---- archive/terraform/mjmm/s3.tf | 23 ++++----- archive/terraform/mjmm/secgrp.tf | 9 ++-- archive/terraform/ukmda/ukmdashared_bucket.tf | 48 ++++++++++++------- 5 files changed, 63 insertions(+), 44 deletions(-) diff --git a/archive/terraform/mjmm/crossacct-role.tf b/archive/terraform/mjmm/crossacct-role.tf index 90b8f135..fabcd4ec 100644 --- a/archive/terraform/mjmm/crossacct-role.tf +++ b/archive/terraform/mjmm/crossacct-role.tf @@ -24,8 +24,12 @@ data "aws_iam_policy_document" "terraformpoldoc" { resource "aws_iam_role" "terraformrole" { name = "TerraformRole" assume_role_policy = data.aws_iam_policy_document.terraformpoldoc.json - managed_policy_arns = [data.aws_iam_policy.terraformpol.arn] tags = { billingtag = "ukmon" } } + +resource "aws_iam_role_policy_attachment" "tfrole_pol_attachment" { + role = aws_iam_role.terraformrole.name + policy_arn = data.aws_iam_policy.terraformpol.arn +} diff --git a/archive/terraform/mjmm/ec2.tf b/archive/terraform/mjmm/ec2.tf index 181ef241..e5979d6e 100644 --- a/archive/terraform/mjmm/ec2.tf +++ b/archive/terraform/mjmm/ec2.tf @@ -2,10 +2,10 @@ # built in aws_infra project as shared resources data "aws_security_group" "ec2publicsg" { - name = "ec2PublicSG" + name = "ec2PublicSG" } data "aws_key_pair" "marks_key" { - key_name = "markskey" + key_name = "markskey" } @@ -15,6 +15,7 @@ resource "aws_instance" "ukmonhelper_g" { iam_instance_profile = aws_iam_instance_profile.S3FullAccess.name key_name = data.aws_key_pair.marks_key.key_name security_groups = [data.aws_security_group.ec2publicsg.name] + force_destroy = false root_block_device { tags = { @@ -24,17 +25,17 @@ resource "aws_instance" "ukmonhelper_g" { } volume_size = 50 volume_type = "gp3" - throughput = 125 - iops = 3000 - encrypted = true - kms_key_id = aws_kms_key.container_key.arn + throughput = 125 + iops = 3000 + encrypted = true + kms_key_id = aws_kms_key.container_key.arn } tags = { - "Name" = "UKMonHelper2" - "billingtag" = "ukmon" - "project" = "UKMonHelper2" - "Route53FQDN" = "ukmonhelper2.markmcintyreastro.co.uk" + "Name" = "UKMonHelper2" + "billingtag" = "ukmon" + "project" = "UKMonHelper2" + "Route53FQDN" = "ukmonhelper.markmcintyreastro.co.uk" "DNSRecordType" = "A" } } diff --git a/archive/terraform/mjmm/s3.tf b/archive/terraform/mjmm/s3.tf index a2369772..740ba4cc 100644 --- a/archive/terraform/mjmm/s3.tf +++ b/archive/terraform/mjmm/s3.tf @@ -56,6 +56,7 @@ data "aws_iam_policy_document" "websiteacesspolicy" { ######################################################################## resource "aws_s3_bucket" "mjmm-ukmon-shared" { bucket = "mjmm-ukmon-shared" + timeouts {} tags = { "billingtag" = "ukmon" } @@ -79,7 +80,8 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "mjmm_ukmon_shared } resource "aws_s3_bucket_lifecycle_configuration" "shared_lifecycle_rule" { - bucket = aws_s3_bucket.mjmm-ukmon-shared.id + bucket = aws_s3_bucket.mjmm-ukmon-shared.id + transition_default_minimum_object_size = "varies_by_storage_class" rule { id = "MoveToArchive" status = "Enabled" @@ -87,18 +89,16 @@ resource "aws_s3_bucket_lifecycle_configuration" "shared_lifecycle_rule" { days = 30 storage_class = "STANDARD_IA" } + filter {} } rule { id = "PurgeOldVersions" status = "Enabled" expiration { - days = 0 - expired_object_delete_marker = true - } - - filter { + days = 0 } + filter {} noncurrent_version_expiration { newer_noncurrent_versions = "1" @@ -140,7 +140,9 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "mjmm_ukmon_live_e } resource "aws_s3_bucket_lifecycle_configuration" "live_lifecycle_rule" { - bucket = aws_s3_bucket.mjmm-ukmon-live.id + bucket = aws_s3_bucket.mjmm-ukmon-live.id + transition_default_minimum_object_size = "varies_by_storage_class" + rule { id = "MoveToArchive" status = "Enabled" @@ -148,18 +150,17 @@ resource "aws_s3_bucket_lifecycle_configuration" "live_lifecycle_rule" { days = 30 storage_class = "STANDARD_IA" } + filter {} } rule { id = "PurgeOldVersions" status = "Enabled" expiration { - days = 0 - expired_object_delete_marker = true + days = 0 } - filter { - } + filter {} noncurrent_version_expiration { newer_noncurrent_versions = "1" diff --git a/archive/terraform/mjmm/secgrp.tf b/archive/terraform/mjmm/secgrp.tf index 29013618..859d8cff 100644 --- a/archive/terraform/mjmm/secgrp.tf +++ b/archive/terraform/mjmm/secgrp.tf @@ -14,6 +14,7 @@ resource "aws_security_group" "default" { description = "default VPC security group" vpc_id = data.aws_vpc.main_vpc.id revoke_rules_on_delete = false + timeouts {} ingress = [ { description = "http in" @@ -60,9 +61,9 @@ resource "aws_security_group" "default" { self = false } ] -egress = [ + egress = [ { - cidr_blocks = ["0.0.0.0/0",] + cidr_blocks = ["0.0.0.0/0", ] description = "" from_port = 0 ipv6_cidr_blocks = [] @@ -120,13 +121,13 @@ resource "aws_security_group" "launch-wizard-4" { cidr_blocks = [] description = "IPv6 SSH for Admin" from_port = 22 - ipv6_cidr_blocks = ["::/0",] + ipv6_cidr_blocks = ["::/0", ] prefix_list_ids = [] protocol = "tcp" security_groups = [] self = false to_port = 22 - }, + }, ] egress = [ { diff --git a/archive/terraform/ukmda/ukmdashared_bucket.tf b/archive/terraform/ukmda/ukmdashared_bucket.tf index 7e992f09..692de9e9 100644 --- a/archive/terraform/ukmda/ukmdashared_bucket.tf +++ b/archive/terraform/ukmda/ukmdashared_bucket.tf @@ -24,9 +24,9 @@ resource "aws_s3_bucket_policy" "ukmdasharedbp" { Effect = "Allow" Principal = { AWS = [ - "arn:aws:iam::${var.remote_account_id}:role/S3FullAccess", # role used by lambdas + "arn:aws:iam::${var.remote_account_id}:role/S3FullAccess", # role used by lambdas "arn:aws:iam::${var.remote_account_id}:role/lambda-s3-full-access-role", # role used by SAM functions - "arn:aws:iam::${var.remote_account_id}:role/ecsTaskExecutionRole", # role used by ECS tasks + "arn:aws:iam::${var.remote_account_id}:role/ecsTaskExecutionRole", # role used by ECS tasks "arn:aws:iam::${var.remote_account_id}:user/Mary", "arn:aws:iam::${var.remote_account_id}:user/Mark" ] @@ -38,22 +38,22 @@ resource "aws_s3_bucket_policy" "ukmdasharedbp" { Sid = "DelegateS3Access" }, { - "Sid": "BlockAccessToAdmin", - "Effect": "Deny", - "Principal": "*", - "Action": "s3:*", - "Resource": "arn:aws:s3:::ukmda-shared/admin/*", - "Condition": { - "StringNotLike": { - "aws:userId": [ # update with new relevant IDs - "AIDASVSZXPTTB3UZT4E2B", # MarkMcIntyreUKM - "AROAUUCG4WH4GFCTQIKH3", # S3FullAccess in MJMM account - "AROAUUCG4WH4GFCTQIKH3:*", # S3FullAccess in MJMM account - "${data.aws_caller_identity.current.account_id}", # root account - "AROA36ZZGKDHYW6XYFNJD:*" - ] - } + "Sid" : "BlockAccessToAdmin", + "Effect" : "Deny", + "Principal" : "*", + "Action" : "s3:*", + "Resource" : "arn:aws:s3:::ukmda-shared/admin/*", + "Condition" : { + "StringNotLike" : { + "aws:userId" : [ # update with new relevant IDs + "AIDASVSZXPTTB3UZT4E2B", # MarkMcIntyreUKM + "AROAUUCG4WH4GFCTQIKH3", # S3FullAccess in MJMM account + "AROAUUCG4WH4GFCTQIKH3:*", # S3FullAccess in MJMM account + "${data.aws_caller_identity.current.account_id}", # root account + "AROA36ZZGKDHYW6XYFNJD:*" + ] } + } } ] Version = "2012-10-17" @@ -94,10 +94,22 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" { } transition { - days = 45 + days = 30 storage_class = "STANDARD_IA" } } + rule { + status = "Enabled" + id = "Transition to Glacier" + filter { + prefix = "archive/" + } + + transition { + days = 45 + storage_class = "GLACIER_IR" + } + } rule { id = "purge athena queries" status = "Enabled" From 0ff687bb7fea88fce0742f43330716ab6cfcafca Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 27 Aug 2025 22:57:05 +0100 Subject: [PATCH 22/83] automatically update DNS records --- .../updateDnsRecords/lambda_function.py | 260 +++++++++--------- 1 file changed, 130 insertions(+), 130 deletions(-) diff --git a/archive/samfunctions/updateDnsRecords/lambda_function.py b/archive/samfunctions/updateDnsRecords/lambda_function.py index 0a855287..255058f6 100644 --- a/archive/samfunctions/updateDnsRecords/lambda_function.py +++ b/archive/samfunctions/updateDnsRecords/lambda_function.py @@ -1,130 +1,130 @@ -import os -import boto3 -import logging - -logger = logging.getLogger() -logger.setLevel(logging.INFO) - -""" Default values """ -DNS_TAG_KEY = 'Route53FQDN' -DNS_RECORD_TYPE = 'CNAME' -DNS_RECORD_TTL = 60 - - -def update_route53_record(region, instance_id, dns_tag_key, dns_record_type, dns_record_ttl): - """ - update function - """ - dns_record_type = dns_record_type.upper() - - """ Get EC2 Description """ - ec2 = boto3.client('ec2', region_name=region) - ec2_desc_res = ec2.describe_instances(InstanceIds=[instance_id]) - if ('Reservations' not in ec2_desc_res) or (len(ec2_desc_res['Reservations']) < 1) or (len(ec2_desc_res['Reservations'][0]['Instances']) < 1): - logger.warning('Error hetting EC2 instance information failure. Instance Id: ' + instance_id) - return - if (ec2_desc_res['Reservations'][0]['Instances'][0]['State']['Name'] != 'running'): - logger.warning('EC2 state is not running. Current state: ' + ec2_desc_res['Reservations'][0]['Instances'][0]['State']['Name']) - return - ec2_public_dns = ec2_desc_res['Reservations'][0]['Instances'][0]['PublicDnsName'] - ec2_public_ip = ec2_desc_res['Reservations'][0]['Instances'][0]['PublicIpAddress'] - - """ Get EC2 Tags """ - ec2_tags_res = ec2.describe_tags(Filters=[ - {'Name': 'resource-id', 'Values': [instance_id]}, - {'Name': 'key', 'Values': [dns_tag_key]}]) - - if ('Tags' not in ec2_tags_res) or (len(ec2_tags_res['Tags']) < 1): - logger.warning('EC2 instance has no DNS tag. Instance Id: ' + instance_id) - return - - dns_record_name = ec2_tags_res['Tags'][0]['Value'] - if (dns_record_name[-1] == '.'): - dns_record_name = dns_record_name[:-1] - - ec2_tags_res = ec2.describe_tags(Filters=[ - {'Name': 'resource-id', 'Values': [instance_id]}, - {'Name': 'key', 'Values': ['DNSRecordType']}]) - if ('Tags' not in ec2_tags_res) or (len(ec2_tags_res['Tags']) < 1): - logger.warning('EC2 instance has no Record Type. Instance Id: ' + instance_id) - else: - dns_record_type = ec2_tags_res['Tags'][0]['Value'] - - """ Check DNS name """ - dns_record_names = dns_record_name.split('.') - if (len(dns_record_names) < 3): - logger.warning('The FQDN must over 3 levels at least. Instance Id: ' + instance_id + ', ' + dns_tag_key + ': ' + dns_record_name) - return - """ Get all hosted zones from Route53 """ - route53 = boto3.client('route53', region_name=region) - zones_res = route53.list_hosted_zones() - all_zones = zones_res['HostedZones'] - if (zones_res['IsTruncated']): - while zones_res and (not zones_res['IsTruncated']): - zones_res = route53.list_hosted_zones(Marker=zones_res['NextMarker']) - all_zones = all_zones + zones_res['HostedZones'] - - """ Find Route53 zone """ - zone_id = None - for i in range(1, len(dns_record_names)-1): - cur_zone_name = '.'.join(dns_record_names[i:len(dns_record_names)+1]) + '.' - cur_zone_matches = [zone for zone in all_zones if (zone['Name']==cur_zone_name)] - if (len(cur_zone_matches) > 0): - zone_id = cur_zone_matches[0]['Id'] - break - if (not zone_id): - print('Route53 zone is not found. DNS name: ' + dns_record_name) - return - - """ Update Route53 record record """ - route53.change_resource_record_sets( - HostedZoneId=zone_id, - ChangeBatch={ - 'Changes': [ - { - 'Action': 'UPSERT', - 'ResourceRecordSet': { - 'Name': dns_record_name + '.', - 'Type': dns_record_type, - 'ResourceRecords': [{'Value': (ec2_public_dns if (dns_record_type=='CNAME') else ec2_public_ip)}], - 'TTL': dns_record_ttl - } - } - ] - } - ) - logger.info('The Route53 record updated successfully. ' + dns_record_name + ': [' + dns_record_type + '] ' + (ec2_public_dns if (dns_record_type=='CNAME') else ec2_public_ip)) - return - - -def lambda_handler(event, context): - """ - main function - """ - region = os.environ.get("region") - instance_id = os.environ.get("instance_id") - dns_tag_key = os.environ.get("dns_tag_key") if os.environ.get("dns_tag_key") else DNS_TAG_KEY - dns_record_type = os.environ.get("dns_record_type") if os.environ.get("dns_record_type") else DNS_RECORD_TYPE - dns_record_ttl = int(os.environ.get("dns_record_ttl")) if os.environ.get("dns_record_ttl") else DNS_RECORD_TTL - - if (('region' in event) and event['region']): - region = str(event['region']) - if (not region): - print('Parameter "region" is not defined.') - return - - if (('detail' in event) and ('instance-id' in event['detail']) and event['detail']['instance-id']): - instance_id = str(event['detail']['instance-id']) - if (not instance_id): - print('Parameter "instance-id" is not defined.') - return - - if (('state' in event['detail']) and event['detail']['state'] and (event['detail']['state'] != 'running')): - logger.warning('EC2 state is not running. Current state: ' + event['detail']['state']) - return - - try: - update_route53_record(region, instance_id, dns_tag_key, dns_record_type, dns_record_ttl) - except Exception as error: - logger.exception(error) - return +import os +import boto3 +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +""" Default values """ +DNS_TAG_KEY = 'Route53FQDN' +DNS_RECORD_TYPE = 'CNAME' +DNS_RECORD_TTL = 60 + + +def update_route53_record(region, instance_id, dns_tag_key, dns_record_type, dns_record_ttl): + """ + update function + """ + dns_record_type = dns_record_type.upper() + + """ Get EC2 Description """ + ec2 = boto3.client('ec2', region_name=region) + ec2_desc_res = ec2.describe_instances(InstanceIds=[instance_id]) + if ('Reservations' not in ec2_desc_res) or (len(ec2_desc_res['Reservations']) < 1) or (len(ec2_desc_res['Reservations'][0]['Instances']) < 1): + logger.warning('Error hetting EC2 instance information failure. Instance Id: ' + instance_id) + return + if (ec2_desc_res['Reservations'][0]['Instances'][0]['State']['Name'] != 'running'): + logger.warning('EC2 state is not running. Current state: ' + ec2_desc_res['Reservations'][0]['Instances'][0]['State']['Name']) + return + ec2_public_dns = ec2_desc_res['Reservations'][0]['Instances'][0]['PublicDnsName'] + ec2_public_ip = ec2_desc_res['Reservations'][0]['Instances'][0]['PublicIpAddress'] + + """ Get EC2 Tags """ + ec2_tags_res = ec2.describe_tags(Filters=[ + {'Name': 'resource-id', 'Values': [instance_id]}, + {'Name': 'key', 'Values': [dns_tag_key]}]) + + if ('Tags' not in ec2_tags_res) or (len(ec2_tags_res['Tags']) < 1): + logger.warning('EC2 instance has no DNS tag. Instance Id: ' + instance_id) + return + + dns_record_name = ec2_tags_res['Tags'][0]['Value'] + if (dns_record_name[-1] == '.'): + dns_record_name = dns_record_name[:-1] + + ec2_tags_res = ec2.describe_tags(Filters=[ + {'Name': 'resource-id', 'Values': [instance_id]}, + {'Name': 'key', 'Values': ['DNSRecordType']}]) + if ('Tags' not in ec2_tags_res) or (len(ec2_tags_res['Tags']) < 1): + logger.warning('EC2 instance has no Record Type. Instance Id: ' + instance_id) + else: + dns_record_type = ec2_tags_res['Tags'][0]['Value'] + + """ Check DNS name """ + dns_record_names = dns_record_name.split('.') + if (len(dns_record_names) < 3): + logger.warning('The FQDN must over 3 levels at least. Instance Id: ' + instance_id + ', ' + dns_tag_key + ': ' + dns_record_name) + return + """ Get all hosted zones from Route53 """ + route53 = boto3.client('route53', region_name=region) + zones_res = route53.list_hosted_zones() + all_zones = zones_res['HostedZones'] + if (zones_res['IsTruncated']): + while zones_res and (not zones_res['IsTruncated']): + zones_res = route53.list_hosted_zones(Marker=zones_res['NextMarker']) + all_zones = all_zones + zones_res['HostedZones'] + + """ Find Route53 zone """ + zone_id = None + for i in range(1, len(dns_record_names)-1): + cur_zone_name = '.'.join(dns_record_names[i:len(dns_record_names)+1]) + '.' + cur_zone_matches = [zone for zone in all_zones if (zone['Name']==cur_zone_name)] + if (len(cur_zone_matches) > 0): + zone_id = cur_zone_matches[0]['Id'] + break + if (not zone_id): + print('Route53 zone is not found. DNS name: ' + dns_record_name) + return + + """ Update Route53 record record """ + route53.change_resource_record_sets( + HostedZoneId=zone_id, + ChangeBatch={ + 'Changes': [ + { + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'Name': dns_record_name + '.', + 'Type': dns_record_type, + 'ResourceRecords': [{'Value': (ec2_public_dns if (dns_record_type=='CNAME') else ec2_public_ip)}], + 'TTL': dns_record_ttl + } + } + ] + } + ) + logger.info('The Route53 record updated successfully. ' + dns_record_name + ': [' + dns_record_type + '] ' + (ec2_public_dns if (dns_record_type=='CNAME') else ec2_public_ip)) + return + + +def lambda_handler(event, context): + """ + main function + """ + region = os.environ.get("region") + instance_id = os.environ.get("instance_id") + dns_tag_key = os.environ.get("dns_tag_key") if os.environ.get("dns_tag_key") else DNS_TAG_KEY + dns_record_type = os.environ.get("dns_record_type") if os.environ.get("dns_record_type") else DNS_RECORD_TYPE + dns_record_ttl = int(os.environ.get("dns_record_ttl")) if os.environ.get("dns_record_ttl") else DNS_RECORD_TTL + + if (('region' in event) and event['region']): + region = str(event['region']) + if (not region): + print('Parameter "region" is not defined.') + return + + if (('detail' in event) and ('instance-id' in event['detail']) and event['detail']['instance-id']): + instance_id = str(event['detail']['instance-id']) + if (not instance_id): + print('Parameter "instance-id" is not defined.') + return + + if (('state' in event['detail']) and event['detail']['state'] and (event['detail']['state'] != 'running')): + logger.warning('EC2 state is not running. Current state: ' + event['detail']['state']) + return + + try: + update_route53_record(region, instance_id, dns_tag_key, dns_record_type, dns_record_ttl) + except Exception as error: + logger.exception(error) + return From f2ef3726467c5acc17eac0c95abe5fa0f440875f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 27 Aug 2025 22:58:00 +0100 Subject: [PATCH 23/83] minor tweaks to fireball collector and analyser --- fbCollector/config.ini | 58 +- fbCollector/fbCollector.ps1 | 48 +- fbCollector/fbcollector.yml | 44 +- fbCollector/fireballCollector.py | 1982 +++++++++++++++--------------- fbCollector/requirements.txt | 16 +- 5 files changed, 1074 insertions(+), 1074 deletions(-) diff --git a/fbCollector/config.ini b/fbCollector/config.ini index 9dfab68b..35305ab7 100644 --- a/fbCollector/config.ini +++ b/fbCollector/config.ini @@ -1,30 +1,30 @@ -# tailor this config file to your needs -# $HOME will be translated into your homedir (~ on linux, $env:userprofile on windows) -[Fireballs] -basedir=$HOME/OneDrive/pictures/meteors/fireballs - -livebucket=ukmda-live -uploadbucket=ukmda-shared -uploadfolder=fireballs/uploads -ukmonprofile=ukmonshared - -[gmnconnection] -# the GMN key needs to be configured with Denis Vida -# if you have permission you can then collect data from the GMN server -gmnkey=~/.ssh/gmnanalysis -gmnserver=gmn.uwo.ca -gmnuser=analysis - -[reduction] -# To use the reduction options RMS must be installed and an RMS python conda environment created -rms_loc=$HOME/OneDrive/dev/RMS -rms_env=RMS - -[solver] -# To use the solver options WMPL must be installed and a WMPL python conda environment created -wmpl_loc=$HOME/OneDrive/dev/WesternMeteorPyLib -wmpl_env=ukmon-shared - -[sharing] -# create this folder if you want to share raw data with other UKMON members +# tailor this config file to your needs +# $HOME will be translated into your homedir (~ on linux, $env:userprofile on windows) +[Fireballs] +basedir=$HOME/OneDrive/pictures/meteors/fireballs + +livebucket=ukmda-live +uploadbucket=ukmda-shared +uploadfolder=fireballs/uploads +ukmonprofile=ukmonshared + +[gmnconnection] +# the GMN key needs to be configured with Denis Vida +# if you have permission you can then collect data from the GMN server +gmnkey=~/.ssh/gmnanalysis +gmnserver=gmn.uwo.ca +gmnuser=analysis + +[reduction] +# To use the reduction options RMS must be installed and an RMS python conda environment created +rms_loc=$HOME/OneDrive/dev/RMS +rms_env=RMS + +[solver] +# To use the solver options WMPL must be installed and a WMPL python conda environment created +wmpl_loc=$HOME/OneDrive/dev/WesternMeteorPyLib +wmpl_env=ukmon-shared + +[sharing] +# create this folder if you want to share raw data with other UKMON members shrfldr=$HOME/Dropbox/Raw_Data \ No newline at end of file diff --git a/fbCollector/fbCollector.ps1 b/fbCollector/fbCollector.ps1 index 1220f741..9854e1e8 100644 --- a/fbCollector/fbCollector.ps1 +++ b/fbCollector/fbCollector.ps1 @@ -1,24 +1,24 @@ -# -# powershell script to launch the fireball data collector tool -# Copyright (C) 2018-2023 Mark McIntyre -# -push-location $PSScriptRoot - -. .\helperfunctions.ps1 -$ini=get-inicontent .\config.ini - -conda activate $ini['solver']['wmpl_env'] -$wmplloc = $ini['solver']['wmpl_loc'] -$wmplloc = $wmplloc.replace('$HOME',$env:userprofile) - -$env:pythonpath="$wmplloc" - -if ($args.count -lt 1) { - python fireballCollector.py -}else { - python fireballCollector.py -d $args[0] -} - -Pop-Location - - +# +# powershell script to launch the fireball data collector tool +# Copyright (C) 2018-2023 Mark McIntyre +# +push-location $PSScriptRoot + +. .\helperfunctions.ps1 +$ini=get-inicontent .\config.ini + +conda activate $ini['solver']['wmpl_env'] +$wmplloc = $ini['solver']['wmpl_loc'] +$wmplloc = $wmplloc.replace('$HOME',$env:userprofile) + +$env:pythonpath="$wmplloc" + +if ($args.count -lt 1) { + python fireballCollector.py +}else { + python fireballCollector.py -d $args[0] +} + +Pop-Location + + diff --git a/fbCollector/fbcollector.yml b/fbCollector/fbcollector.yml index 11707f86..9d18f218 100644 --- a/fbCollector/fbcollector.yml +++ b/fbCollector/fbcollector.yml @@ -1,22 +1,22 @@ ---- -- hosts: ukmonhelper2 - gather_facts: no - tasks: - - name: import variables - include_vars: ./vars.yml - tags: [dev,prod] - - name: Ensures {{destdir}} exists - file: path={{destdir}} state=directory - delegate_to: localhost - tags: [dev,prod] - - name: Copy files - copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} - tags: [dev,prod] - delegate_to: localhost - with_items: - - {src: '{{srcdir}}/fbCollector.ps1', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/fireballCollector.py', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/download_events.sh', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/config.ini', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/noimage.jpg', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/ukmda.ico', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } +--- +- hosts: ukmonhelper2 + gather_facts: no + tasks: + - name: import variables + include_vars: ./vars.yml + tags: [dev,prod] + - name: Ensures {{destdir}} exists + file: path={{destdir}} state=directory + delegate_to: localhost + tags: [dev,prod] + - name: Copy files + copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} + tags: [dev,prod] + delegate_to: localhost + with_items: + - {src: '{{srcdir}}/fbCollector.ps1', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/fireballCollector.py', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/download_events.sh', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/config.ini', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/noimage.jpg', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/ukmda.ico', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index ccbaf598..658dad4c 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -1,991 +1,991 @@ -# -# UI to manage fireball data collection -# Copyright (C) 2018-2023 Mark McIntyre -# -import os -import sys -import argparse -import logging -import logging.handlers -import datetime -import configparser -import shutil -import platform -import subprocess -import glob -import xmltodict -from PIL import Image -import requests - -import boto3 -import paramiko -from scp import SCPClient - -from meteortools.ukmondb import getECSVs as getecsv -from wmpl.Formats.ECSV import loadECSVs -from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric - -import tkinter as tk -import tkinter.filedialog as tkFileDialog -import tkinter.messagebox as tkMessageBox -from tkinter.simpledialog import askstring -from tkinter import StringVar, Frame, ACTIVE, END, Listbox, Menu, Entry, Button -from tkinter.ttk import Label, Style, LabelFrame, Scrollbar - -from PIL import Image as img -from PIL import ImageTk - -from meteortools.ukmondb import getLiveJpgs, createTxtFile - - -config_file = '' -noimg_file = '' -global_bg = "Black" -global_fg = "Gray" - - -def quitApp(): - # Cleanly exits the app - root.quit() - root.destroy() - - -def log_timestamp(): - """ Returns timestamp for logging. - """ - return datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') - - -class StyledButton(Button): - """ Button with style. - """ - def __init__(self, *args, **kwargs): - Button.__init__(self, *args, **kwargs) - - self.configure(foreground = global_fg, background = global_bg, borderwidth = 3) - - -class StyledEntry(Entry): - """ Entry box with style. - """ - def __init__(self, *args, **kwargs): - Entry.__init__(self, *args, **kwargs) - - self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg, disabledbackground = global_bg, disabledforeground = "DimGray") - - -class ConstrainedEntry(StyledEntry): - """ Entry box with constrained values which can be input (e.g. 0-255). - """ - def __init__(self, *args, **kwargs): - StyledEntry.__init__(self, *args, **kwargs) - self.maxvalue = 255 - vcmd = (self.register(self.on_validate), "%P") - self.configure(validate="key", validatecommand=vcmd) - # self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg) - - def disallow(self): - """ Pings a bell on values which are out of bound. - """ - self.bell() - - def update_value(self, maxvalue): - """ Updates values in the entry box. - """ - self.maxvalue = maxvalue - vcmd = (self.register(self.on_validate), "%P") - self.configure(validate="key", validatecommand=vcmd) - - def on_validate(self, new_value): - """ Checks if entered value is within bounds. - """ - try: - if new_value.strip() == "": - return True - value = int(new_value) - if value < 0 or value > self.maxvalue: - self.disallow() - return False - except ValueError: - self.disallow() - return False - - return True - - -class fbCollector(Frame): - - def __init__(self, parent, patt=None): - Frame.__init__(self, parent, bg = global_bg) - parent.configure(bg = global_bg) # Set backgound color - parent.grid_columnconfigure(0, weight=1) - parent.grid_rowconfigure(0, weight=1) - - self.grid(sticky="NSEW") # Expand frame to all directions - self.parent = parent - - self.fb_dir = '' - self.upload_bucket = '' - self.upload_folder = '' - self.live_bucket = '' - self.gmn_key = '' - self.gmn_user = '' - self.gmn_server = '' - self.wmpl_loc = '' - self.wmpl_env = '' - self.rms_loc = '' - self.rms_env = '' - self.selected = {} - self.evtMonTriggered = None - self.review_stack = False - self.soln_outputdir = None - self.log_files_to_keep = 30 - - self.readConfig() - - self.patt = patt - if patt is None: - self.dir_path = self.fb_dir.strip() - else: - self.dir_path = os.path.join(self.fb_dir, patt) - log.info(f"Fireball folder is {self.fb_dir}") - - self.initUI() - if self.dir_path != self.fb_dir: - bin_list = self.get_bin_list() - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - - # Update UI changes - parent.update_idletasks() - parent.update() - - return - - def readConfig(self): - localcfg = configparser.ConfigParser() - localcfg.read(config_file) - self.fb_dir = os.path.expanduser(localcfg['Fireballs']['basedir'].replace('$HOME','~')).replace('\\','/') - self.upload_bucket = localcfg['Fireballs']['uploadbucket'] - self.upload_folder = localcfg['Fireballs']['uploadfolder'] - self.live_bucket = localcfg['Fireballs']['livebucket'] - os.makedirs(self.fb_dir, exist_ok=True) - - try: - self.gmn_key = localcfg['gmnconnection']['gmnkey'] - self.gmn_user = localcfg['gmnconnection']['gmnuser'] - self.gmn_server = localcfg['gmnconnection']['gmnserver'] - except: - pass - - self.wmpl_loc = os.path.expanduser(localcfg['solver']['wmpl_loc'].replace('$HOME','~')).replace('\\','/') - self.wmpl_env= localcfg['solver']['wmpl_env'] - self.rms_loc = os.path.expanduser(localcfg['reduction']['rms_loc'].replace('$HOME','~')).replace('\\','/') - self.rms_env = localcfg['reduction']['rms_env'] - - self.shareloc = os.path.expanduser(localcfg['sharing']['shrfldr'].replace('$HOME','~')).replace('\\','/') - return - - def quitApplication(self): - print('quitting') - logdir = os.path.join(os.getenv('TMP'), 'fbcollector') - logfiles = os.listdir(logdir) - numtokeep = self.log_files_to_keep - if len(logfiles) > numtokeep: - logfiles.sort() - for i in range(len(logfiles)-numtokeep): - try: - os.remove(os.path.join(logdir, logfiles[i])) - except: - pass - quitApp() - - def initUI(self): - """ Initialize GUI elements. - """ - - self.parent.title("Fireball Downloader") - - # Configure the style of each element - s = Style() - s.configure("TButton", padding=(0, 5, 0, 5), font='serif 10') - s.configure('TLabelframe.Label', foreground=global_fg, background=global_bg) - s.configure('TLabelframe', foreground =global_fg, background=global_bg, padding=(3, 3, 3, 3)) - s.configure("TRadiobutton", foreground = global_fg, background = global_bg) - s.configure("TLabel", foreground = global_fg, background = global_bg) - s.configure("TCheckbutton", foreground = global_fg, background = global_bg) - s.configure("Vertical.TScrollbar", background=global_bg, troughcolor = global_bg) - - self.columnconfigure(0, pad=3) - self.columnconfigure(1, pad=3) - self.columnconfigure(2, pad=3) - self.columnconfigure(3, pad=3) - self.columnconfigure(4, pad=3) - self.columnconfigure(5, pad=3) - self.columnconfigure(6, pad=3) - self.columnconfigure(7, pad=3) - self.columnconfigure(8, pad=3) - - self.rowconfigure(0, pad=3) - self.rowconfigure(1, pad=3) - self.rowconfigure(2, pad=3) - self.rowconfigure(3, pad=3) - self.rowconfigure(4, pad=3) - self.rowconfigure(5, pad=3) - self.rowconfigure(6, pad=3) - self.rowconfigure(7, pad=3) - self.rowconfigure(8, pad=3) - self.rowconfigure(9, pad=3) - self.rowconfigure(10, pad=3) - - # Make menu - self.menuBar = Menu(self.parent) - self.parent.config(menu=self.menuBar) - - # File menu - fileMenu = Menu(self.menuBar, tearoff=0) - fileMenu.add_command(label="Load Folder", command=self.loadFolder) - fileMenu.add_command(label="Archive This Folder", command=self.archiveFolder) - fileMenu.add_command(label="Open in Explorer", command=self.openFolder) - fileMenu.add_separator() - fileMenu.add_command(label="Delete This Folder", command=self.delFolder) - fileMenu.add_separator() - fileMenu.add_command(label="Configuration", command=self.showConfig) - fileMenu.add_command(label="View Logs", command=self.viewLogs) - fileMenu.add_separator() - fileMenu.add_command(label="Exit", command=self.quitApplication) - self.menuBar.add_cascade(label="File", underline=0, menu=fileMenu) - - rawMenu = Menu(self.menuBar, tearoff=0) - rawMenu.add_command(label="Get Live Images", command=self.get_data) - rawMenu.add_command(label="Get Videos", command=self.get_vids) - rawMenu.add_command(label="Get Traj Pickle", command=self.get_trajpickle) - rawMenu.add_separator() - rawMenu.add_command(label="Get GMN Raw Data", command=self.getGMNData) - rawMenu.add_separator() - rawMenu.add_command(label="Get ECSVs", command=self.getECSVs) - self.menuBar.add_cascade(label="Raw", underline=0, menu=rawMenu) - - watchMenu = Menu(self.menuBar, tearoff=0) - watchMenu.add_command(label="Get Watchlist", command=self.getWatchlist) - watchMenu.add_command(label="View Watchlist", command=self.viewWatchlist) - watchMenu.add_command(label="Upload Watchlist", command=self.putWatchlist) - watchMenu.add_separator() - watchMenu.add_command(label="Fetch Event Data", command=self.getEventData) - self.menuBar.add_cascade(label="Watchlist", underline=0, menu=watchMenu) - - revMenu = Menu(self.menuBar, tearoff=0) - revMenu.add_command(label="Review Stacks", command=self.checkStacks) - revMenu.add_command(label="Clean Folder", command=self.clean_folder) - self.menuBar.add_cascade(label="Review", underline=0, menu=revMenu) - - solveMenu = Menu(self.menuBar, tearoff=0) - solveMenu.add_command(label="Reduce Data", command=self.reduceCamera) - solveMenu.add_command(label="Toggle Ignore", command=self.ignoreCamera) - solveMenu.add_separator() - solveMenu.add_command(label="View Raw Data", command=self.viewData) - solveMenu.add_command(label="Upload Raw Data", command=self.uploadRaw) - solveMenu.add_separator() - solveMenu.add_command(label="Solve", command=self.solveOrbit) - solveMenu.add_separator() - solveMenu.add_command(label="View Solution", command=self.viewSolution) - solveMenu.add_command(label="Delete Solution", command=self.removeSolution) - solveMenu.add_separator() - solveMenu.add_command(label="Upload Orbit", command=self.uploadOrbit) - self.menuBar.add_cascade(label="Solve", underline=0, menu=solveMenu) - # buttons - self.save_panel = LabelFrame(self, text=' Image Selection ') - self.save_panel.grid(row = 1, columnspan = 2, sticky='WE') - - self.newpatt = StringVar() - self.newpatt.set(self.patt) - - self.patt_entry = StyledEntry(self.save_panel, textvariable = self.newpatt, width = 20) - self.patt_entry.grid(row = 1, column = 1, columnspan = 2, sticky = "W") - save_bmp = StyledButton(self.save_panel, text="Get Images", width = 8, command = lambda: self.get_data()) - save_bmp.grid(row = 1, column = 3) - - save_bmp = StyledButton(self.save_panel, text="Remove", width = 8, command = lambda: self.remove_image()) - save_bmp.grid(row = 1, column = 4) - - - # Listbox - self.scrollbar = Scrollbar(self) - self.listbox = Listbox(self, width = 47, yscrollcommand=self.scrollbar.set, exportselection=0, activestyle = "none", bg = global_bg, fg = global_fg) - self.listbox.config(height = 25) # Listbox size - self.listbox.grid(row = 4, column = 0, rowspan = 7, columnspan = 2, sticky = "NS") # Listbox position - self.scrollbar.grid(row = 4, column = 2, rowspan = 7, sticky = "NS") # Scrollbar size - - self.listbox.bind('<>', self.update_image) - self.scrollbar.config(command = self.listbox.yview) - - # IMAGE - try: - # Show the TV test card image on program start - #noimage_data = img.open('noimage.bin').resize((640,360)) - noimgdata = img.open(noimg_file).resize((640,360)) - noimage = ImageTk.PhotoImage(noimgdata) - except: - noimage = None - - self.imagelabel = Label(self, image = noimage) - self.imagelabel.image = noimage - self.imagelabel.grid(row=3, column=3, rowspan = 4, columnspan = 3) - - # Timestamp label - self.timestamp_label = Label(self, text = "CCNNNN YYYY-MM-DD HH:MM.SS.mms", font=("Courier", 12)) - self.timestamp_label.grid(row = 7, column = 3, sticky = "E") - - def reduceCamera(self): - current_image = self.listbox.get(ACTIVE) - if current_image == '': - return - camid = current_image[3:9] - print('selected camera is', camid) - dirname = os.path.join(self.dir_path, camid) - tmpscr = os.path.join(os.getenv('TEMP'), 'reduce.ps1') - with open(tmpscr, 'w') as outf: - outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {dirname} -c {dirname}/.config\n') - _ = subprocess.run(['powershell.exe', tmpscr]) - frs = glob.glob(os.path.join(dirname, 'FR*.bin')) - if len(frs) > 0: - if not tkMessageBox.askyesno("Rerun", f'{len(frs)} FR files detected - rerun?'): - return - for fr in frs: - with open(tmpscr, 'w') as outf: - outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {fr} -c {dirname}/.config\n') - _ = subprocess.run(['powershell.exe', tmpscr]) - try: - os.remove(tmpscr) - except: - pass - return - - def ignoreCamera(self): - current_image = self.listbox.get(ACTIVE) - if current_image == '': - return - camid = current_image[3:9] - log.info(f'selected camera is {camid}') - dirname = os.path.join(self.dir_path, camid) - allecsvs = glob.glob(os.path.join(dirname, '*.ecsv')) - ecsvs = [e for e in allecsvs if 'REJECT' not in e.upper()] - rejs = [e for e in allecsvs if 'REJECT' in e.upper()] - if len(ecsvs) == 0 and len(rejs) == 0: - log.info('no files to reject or include') - return - elif len(ecsvs) == 1 and len(rejs) == 0: - os.rename(ecsvs[0], ecsvs[0].replace('.ecsv','_REJECT.ecsv')) - jpgname = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) - os.rename(jpgname[0], jpgname[0].replace('.jpg','_REJECT.jpg')) - elif len(ecsvs) == 0 and len(rejs) == 1: - os.rename(rejs[0], rejs[0].replace('_REJECT.ecsv','.ecsv')) - jpgname = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) - os.rename(jpgname[0], jpgname[0].replace('_REJECT.jpg','.jpg')) - else: - # more than one ECSV or REJ file to handle, urk - log.info('urk') - return - bin_list = [line for line in os.listdir(os.path.join(self.dir_path, 'jpgs')) if self.correct_datafile_name(line)] - #print(bin_list) - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - return - - def solveOrbit(self): - log.info('Using ECSV files:') - ecsv_names = [] - ecsv_paths = [] - for entry in sorted(os.walk(self.dir_path), key=lambda x: x[0]): - dir_name, _, file_names = entry - for fn in file_names: - if fn.lower().endswith(".ecsv") and 'REJECT' not in dir_name.upper() and 'REJECT' not in fn.upper(): - - # Add ECSV file, but skip duplicates - if fn not in ecsv_names: - ecsv_paths.append(os.path.join(dir_name, fn)) - ecsv_names.append(fn) - log.info(fn) - if len(ecsv_paths) < 2: - tkMessageBox.showinfo('Warning', 'Need at least two ECSV files') - return - jdt_ref, meteor_list = loadECSVs(ecsv_paths) - mcruns = 20 - max_toffset = 15.0 - velpart = None - vinitht = None - plotallspatial = True - uncertgeom = False - jacchia = False - traj = solveTrajectoryGeneric(jdt_ref, meteor_list, self.dir_path, - max_toffset=max_toffset, monte_carlo=True, mc_runs=mcruns, - geometric_uncert=uncertgeom, plot_all_spatial_residuals=plotallspatial, - show_plots=False, v_init_part=velpart, v_init_ht=vinitht, - show_jacchia=jacchia, enable_OSM_plot=True, mc_cores=8) - tkMessageBox.showinfo('Info', 'Solver Finished') - if traj is not None: - self.soln_outputdir = traj.output_dir - return - - def viewSolution(self): - self.review_stack = False - if not self.soln_outputdir: - solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') - if len(solndir) == 0: - return - solndir = os.path.join(self.dir_path, solndir[0]) - self.soln_outputdir = solndir - bin_list = [line for line in os.listdir(self.soln_outputdir) if self.correct_datafile_name(line)] - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - return - - def removeSolution(self): - if not self.soln_outputdir: - solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') - if len(solndir) == 0: - return - solndir = os.path.join(self.dir_path, solndir[0]) - self.soln_outputdir = solndir - if not tkMessageBox.askyesno("Delete file", f"delete {self.soln_outputdir}?"): - return - shutil.rmtree(os.path.join(self.dir_path, self.soln_outputdir)) - return - - def uploadOrbit(self): - pickles=[] - for path, _, files in os.walk(self.dir_path): - for name in files: - if '.pickle' in name and '_mc_' not in name: - pickles.append(os.path.join(path, name)) - - if len(pickles) == 0: - return - elif len(pickles) == 1: - pickfile = pickles[0] - else: - pickfile = tkFileDialog.askopenfilename(title='Select Orbit Pickle', defaultextension='*.pickle', - initialdir=self.dir_path, initialfile='*.pickle', - filetypes=[('pickles','*.pickle')]) - orbname = os.path.split(pickfile)[1] - tmpdir = os.path.join(self.dir_path, 'tmpzip') - os.makedirs(tmpdir, exist_ok=True) - shutil.copyfile(pickfile, os.path.join(tmpdir, orbname)) - if os.path.isdir(os.path.join(self.dir_path, 'jpgs')): - shutil.copytree(os.path.join(self.dir_path, 'jpgs'), os.path.join(tmpdir, 'jpgs')) - if os.path.isdir(os.path.join(self.dir_path, 'mp4s')): - shutil.copytree(os.path.join(self.dir_path, 'mp4s'), os.path.join(tmpdir, 'mp4s')) - for path, _, files in os.walk(self.dir_path): - for name in files: - if '_dyn_mass_fit' in name: - im = Image.open(os.path.join(path, name)).convert("RGB") - im.save(os.path.join(tmpdir,'jpgs', name[:-4] + '.jpg')) - break - zfname = os.path.join(self.dir_path, orbname[:15]) - shutil.make_archive(zfname,'zip',tmpdir) - try: - shutil.rmtree(tmpdir) - except Exception: - pass - apikey = open(os.path.expanduser('~/.ssh/fbuploadkey.txt')).readlines()[0].strip() - headers = {'Content-type': 'application/zip', 'Slug': orbname[:15], 'apikey': apikey} - url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbname[:15]}.zip' - r = requests.put(url, data=open(zfname+'.zip', 'rb'), headers=headers) #, auth=('username', 'pass')) - #print(r.text) - if r.status_code != 200: - tkMessageBox.showinfo('Warning', 'Problem with upload') - else: - tkMessageBox.showinfo('Info', 'Orbit Uploaded') - return - - def uploadRaw(self): - zfname = os.path.join(os.getenv('TMP'), os.path.basename(self.dir_path)) - log.info(f'zfname is {zfname}') - shutil.make_archive(zfname,'zip',self.dir_path) - try: - targname = os.path.join(self.shareloc, os.path.basename(zfname)+'.zip') - log.info(f'targname is {targname}') - shutil.copyfile(zfname+'.zip', targname) - tkMessageBox.showinfo('Info', 'Raw Data Uploaded to Dropbox') - subprocess.Popen(f'explorer "{self.shareloc}"') - except Exception: - tkMessageBox.showinfo('Warning', 'Problem with upload') - return - - def viewData(self): - dirpath = self.dir_path.replace('/', '\\') - log.info(f'self-dir-path {dirpath}') - subprocess.Popen(f'explorer "{dirpath}"') - return - - def getECSVs(self): - notgotlist=[] - img_list = self.get_bin_list() - for current_image in img_list: - if not self.getOneEcsv(current_image): - notgotlist.append(current_image) - if len(notgotlist) > 0: - tkMessageBox.showinfo('Info', f'No ECSVs for {notgotlist}') - return - - def getOneEcsv(self, current_image): - if current_image[:1] == 'M': - datestr = current_image[1:16] - statid = current_image[-11:-5] - elif current_image[:2] == 'FF': - statid = current_image[3:9] - datestr = current_image[10:29] - else: - return - #dtval = datetime.datetime.strptime(datestr, '%Y%m%d_%H%M%S') - #datestr = dtval.strftime('%Y-%m-%dT%H:%M:%S') - try: - lis = getecsv(statid, datestr, savefiles=True, outdir=os.path.join(self.dir_path, statid)) - for li in lis: - if 'issue getting data' in li: - return False - return True - except Exception: - return False - - def get_bin_list(self): - """ Get a list of image files in a given directory. - """ - if self.dir_path is None: - dirname = tkFileDialog.askdirectory(parent=root,initialdir=self.fb_dir, - title='Please select a directory') - _, thispatt = os.path.split(dirname) - self.dir_path = dirname - self.patt = thispatt - self.newpatt.set(self.patt) - - if self.review_stack is False: - targdir = os.path.join(self.dir_path, 'jpgs') - else: - targdir = os.path.join(self.dir_path, 'stacks') - if os.path.isdir(targdir): - bin_list = [line for line in os.listdir(targdir) if self.correct_datafile_name(line)] - else: - log.info('no jpgs available') - bin_list = [] - return bin_list - - def update_listbox(self, bin_list): - """ Updates the listbox with the current entries. - """ - self.listbox.delete(0, END) - for line in sorted(bin_list): - self.listbox.insert(END, line) - if line not in self.selected: - self.selected[line] = (0, '') - if self.selected[line][0] == 1: - self.listbox.itemconfig(END, fg = 'green') - - def checkStacks(self): - self.review_stack = True - bin_list = self.get_bin_list() - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - - - def loadFolder(self): - self.review_stack = False - self.dir_path = None - self.soln_outputdir = None - bin_list = self.get_bin_list() - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - - def openFolder(self): - dir_path = self.dir_path.replace("/","\\") - os.system(f'explorer.exe {dir_path}') - - def viewLogs(self): - logdir = os.path.join(os.getenv('TMP'), 'fbcollector') - os.system(f'explorer.exe {logdir}') - - def archiveFolder(self): - noimgdata = img.open(noimg_file).resize((640,360)) - noimage = ImageTk.PhotoImage(noimgdata) - self.imagelabel.configure(image = noimage) - self.imagelabel.image = noimage - if self.dir_path == self.fb_dir.strip(): - self.loadFolder() - if self.dir_path is not None and self.dir_path != self.fb_dir: - try: - _, fldr = os.path.split(os.path.normpath(self.dir_path)) - yr = fldr[:4] - archdir = os.path.join(self.fb_dir, yr) - os.makedirs(archdir, exist_ok=True) - zfname = os.path.join(archdir, fldr) - shutil.make_archive(zfname,'zip',self.fb_dir, fldr) - except Exception as e: - log.warning(f'unable to create archive {self.dir_path}') - log.warning(e) - try: - os.chdir(self.fb_dir) - shutil.rmtree(self.dir_path) - except Exception as e: - log.warning(f'unable to remove folder, please do it manually {self.dir_path}') - log.warning(e) - self.dir_path = self.fb_dir - - def delFolder(self): - noimgdata = img.open(noimg_file).resize((640,360)) - noimage = ImageTk.PhotoImage(noimgdata) - self.imagelabel.configure(image = noimage) - self.imagelabel.image = noimage - if self.dir_path is not None and self.dir_path != self.fb_dir: - try: - shutil.rmtree(self.dir_path) - self.dir_path = self.fb_dir - except Exception as e: - log.warning(f'unable to remove {self.dir_path}') - log.warning(e) - - def correct_datafile_name(self, line): - if ('.jpg' in line or '.png' in line) and 'noimage' not in line: - return True - return False - - def removeRelated(self, imgname): - camid = imgname[:6] - mp4s = os.listdir(os.path.join(self.dir_path, 'mp4s')) - badones = [x for x in mp4s if camid in x] - for ba in badones: - os.remove(os.path.join(self.dir_path, 'mp4s', ba)) - jpgs = os.listdir(os.path.join(self.dir_path, 'jpgs')) - badones = [x for x in jpgs if camid in x] - for ba in badones: - os.remove(os.path.join(self.dir_path, 'jpgs', ba)) - fldrs = os.listdir(self.dir_path) - badones = [x for x in fldrs if camid in x] - for ba in badones: - try: - shutil.rmtree(os.path.join(self.dir_path, ba)) - except Exception: - os.remove(os.path.join(self.dir_path, ba)) - os.remove(os.path.join(self.dir_path, 'stacks', imgname)) - - def remove_image(self): - """ Remove the selected image from disk - """ - current_image = self.listbox.get(ACTIVE) - if current_image == '': - return - if not tkMessageBox.askyesno("Delete file", f"delete {current_image}?"): - return - log.info(f'removing {current_image}') - if self.review_stack: - self.removeRelated(current_image) - else: - try: - os.remove(os.path.join(self.dir_path, 'jpgs', current_image)) - except Exception: - pass - xmlf = current_image.replace('P.jpg', '.xml') - try: - os.remove(os.path.join(self.dir_path, 'jpgs', xmlf)) - except Exception: - pass - self.selected[current_image] = (0,'') - self.update_listbox(self.get_bin_list()) - - def update_image(self, thing): - """ When selected, load a new image - """ - try: - # Check if the list is empty. If it is, do nothing. - if self.review_stack: - self.current_image = os.path.join(self.dir_path, 'stacks', self.listbox.get(self.listbox.curselection()[0])) - elif self.soln_outputdir: - self.current_image = os.path.join(self.soln_outputdir, self.listbox.get(self.listbox.curselection()[0])) - else: - self.current_image = os.path.join(self.dir_path, 'jpgs', self.listbox.get(self.listbox.curselection()[0])) - except: - return 0 - - with img.open(self.current_image).resize((640,360)) as imgdata: - thisimage = ImageTk.PhotoImage(imgdata) - self.imagelabel.configure(image = thisimage) - self.imagelabel.image = thisimage - - self.timestamp_label.configure(text = os.path.split(self.current_image)[1]) - return - - def save_image(self): - """ Marks the image as of interest - """ - current_image = self.listbox.get(ACTIVE) - if current_image == '': - return - log.info(f'marking {current_image}') - srcfile = createTxtFile(current_image, self.dir_path) - _, targfile = os.path.split(srcfile) - s3 = boto3.client('s3') - s3.upload_file(srcfile, self.upload_bucket, f'{self.upload_folder}/{targfile}') - cur_index = int(self.listbox.curselection()[0]) - self.listbox.itemconfig(cur_index, fg = 'green') - self.selected[current_image] = (1, srcfile) - - def clean_folder(self): - stacklist = os.listdir(os.path.join(self.dir_path, 'stacks')) - camlist = [x[:6] for x in stacklist if 'stack.jpg' in x] - datalist = os.listdir(self.dir_path) - datalist = [x for x in datalist if 'jpgs' not in x and 'mp4s' not in x and 'stacks' not in x] - for d in datalist: - keep = False - for c in camlist: - if c in d: - keep = True - if keep is False: - try: - os.remove(d) - except Exception: - pass - return - - def get_data(self): - thispatt = self.newpatt.get().strip() - self.patt = thispatt.ljust(15,'0') - self.dir_path = os.path.join(self.fb_dir, self.newpatt.get().strip()) - log.info(f'getting data matching {thispatt}') - os.makedirs(os.path.join(self.dir_path, 'jpgs'), exist_ok=True) - reqdate = datetime.datetime.strptime(self.patt, '%Y%m%d_%H%M%S') - reqdate = reqdate + datetime.timedelta(seconds=-30) - getLiveJpgs(reqdate.strftime('%Y%m%d_%H%M%S'), outdir=os.path.join(self.dir_path, 'jpgs')) - self.renameImages(self.dir_path) - self.update_listbox(self.get_bin_list()) - - def get_trajpickle(self): - basepatt = os.path.split(self.dir_path)[1] - ymd = basepatt[:8] - fullpatt = askstring('Trajectory Name', 'eg 20240101_010203.345_UK', initialvalue=basepatt) - trajpick = f'{basepatt}_trajectory.pickle' - url = f'https://archive.ukmeteors.co.uk/reports/{ymd[:4]}/orbits/{ymd[:6]}/{ymd}/{fullpatt}/{trajpick}' - log.info(f'{url}') - get_response = requests.get(url, stream=True) - if get_response.status_code == 200: - log.info(f'retrieved {trajpick}') - with open(os.path.join(self.dir_path, trajpick), 'wb') as f: - for chunk in get_response.iter_content(chunk_size=4096): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - else: - log.info(f'unable to retrieve {trajpick}, {get_response.status_code}') - return - - def get_vids(self): - jpglist = glob.glob1(os.path.join(self.dir_path,'jpgs'), 'FF*.jpg') - os.makedirs(os.path.join(self.dir_path, 'mp4s'), exist_ok=True) - count = 0 - for jpg in jpglist: - mp4 = jpg.replace('.jpg','.mp4') - ym = mp4[10:16] - url = f'https://archive.ukmeteors.co.uk/img/mp4/{ym[:4]}/{ym}/{mp4}' - get_response = requests.get(url, stream=True) - if get_response.status_code == 200: - log.info(f'retrieved {mp4}') - count += 1 - with open(os.path.join(self.dir_path, 'mp4s', mp4), 'wb') as f: - for chunk in get_response.iter_content(chunk_size=4096): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - log.info(f'retrieved {count} videos') - tkMessageBox.showinfo("Info", f'Retrieved {count} videos') - return - - def renameImages(self, dir_path): - xmllist = glob.glob(os.path.join(dir_path,'jpgs', 'M*.xml')) - for xmlf in xmllist: - jpgf = xmlf.replace('.xml','P.jpg') - if os.path.isfile(jpgf): - xmld = xmltodict.parse(open(xmlf).read()) - realfname = xmld['ufocapture_record']['@cap'].replace('.fits','.jpg') - if not os.path.isfile(os.path.join(dir_path, 'jpgs', realfname)): - os.rename(jpgf, os.path.join(dir_path, 'jpgs', realfname)) - else: - os.remove(jpgf) - os.remove(xmlf) - return - - def viewWatchlist(self): - evtfile = os.path.join(self.fb_dir,'event_watchlist.txt') - if platform.system() == 'Darwin': # macOS - procid = subprocess.Popen(('open', evtfile)) - elif platform.system() == 'Windows': # Windows - procid = subprocess.Popen(('cmd','/c',evtfile)) - else: # linux variants - procid = subprocess.Popen(('xdg-open', evtfile)) - procid.wait() - if not tkMessageBox.askyesno("Upload File", "Upload event watchlist?"): - return - else: - self.putWatchlist() - - def showConfig(self): - if platform.system() == 'Darwin': # macOS - procid = subprocess.Popen(('open', config_file)) - elif platform.system() == 'Windows': # Windows - procid = subprocess.Popen(('cmd','/c',config_file)) - else: # linux variants - procid = subprocess.Popen(('xdg-open', config_file)) - procid.wait() - - def getWatchlist(self): - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) - c = paramiko.SSHClient() - server=self.gmn_server - user=self.gmn_user - log.info(f'trying {user}@{server} with {self.gmn_key}') - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = SCPClient(c.get_transport()) - log.info('getting Watchlist') - scpcli.get('./event_watchlist.txt', self.fb_dir) - self.viewWatchlist() - - def putWatchlist(self): - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) - c = paramiko.SSHClient() - server=self.gmn_server - user=self.gmn_user - log.info(f'trying {user}@{server} with {self.gmn_key}') - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = SCPClient(c.get_transport()) - log.info('Uploading watchlist') - evtfile = os.path.join(self.fb_dir,'event_watchlist.txt') - scpcli.put(evtfile, 'event_watchlist.txt') - self.evtMonTriggered = datetime.datetime.now() + datetime.timedelta(minutes=10) - return - - def getEventData(self): - evtdate = self.newpatt.get().strip() - if len(evtdate) < 15: - tkMessageBox.showinfo("Warning", f'Need seconds in the event date field {evtdate}') - return - cmd = os.path.join(os.path.dirname(__file__), 'download_events.sh') + f' {evtdate} 1' - if ':' in cmd: - drv = cmd[0].lower() - cmd = '/mnt/' + drv + cmd[2:] - cmd = cmd.replace('\\','/') - log.warning(f'executing {cmd}') - if self.evtMonTriggered is None: - tkMessageBox.showinfo("Warning", 'Event Monitor has not been triggered') - return - if datetime.datetime.now() < self.evtMonTriggered: - tkMessageBox.showinfo("Warning", f'Wait till at least {self.evtMonTriggered.strftime("%H:%M:%S")}') - return - os.chdir(self.fb_dir) - log.info(f'getting data for {evtdate}') - procid = subprocess.Popen(('bash','-c', cmd)) - procid.wait() - tkMessageBox.showinfo("Info", 'Done') - return - - def getGMNData(self): - camlist = [line for line in os.listdir(self.dir_path) if self.correct_datafile_name(line)] - dts=[] - camids=[] - for cam in camlist: - camid,_ = os.path.splitext(cam) - spls = camid.split('_') - if camid[:2] == 'FF': - camids.append(spls[1]) - dts.append(camid[10:25]) - else: - camids.append(spls[-1][:6]) - dts.append(camid[1:16]) - dts.sort() - dtstr = f'{dts[0]}.000000' - stationlist = ','.join(map(str, camids)) - server=self.gmn_server - user=self.gmn_user - log.info('getting data from GMN') - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) - c = paramiko.SSHClient() - log.info(f'trying {user}@{server} with {self.gmn_key}') - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - command = f'source ~/anaconda3/etc/profile.d/conda.sh && conda activate wmpl && python scripts/extract_fireball.py {dtstr} {stationlist}' - log.info(f'running {command}') - - _, stdout, stderr = c.exec_command(command, timeout=900) - for line in iter(stdout.readline, ""): - log.info(line) - for line in iter(stderr.readline, ""): - print(line, end="") - scpcli = SCPClient(c.get_transport()) - log.info('done, collecting output') - indir = os.path.join(f'event_extract/{dtstr}/') - scpcli.get(indir, self.dir_path, recursive=True) - command = f'rm -Rf event_extract/{dtstr}' - log.info(f'running {command}') - _, stdout, stderr = c.exec_command(command, timeout=120) - for line in iter(stdout.readline, ""): - log.info(line) - for line in iter(stderr.readline, ""): - log.info(line) - dirs = os.listdir(os.path.join(self.dir_path, dtstr)) - for d in dirs: - srcdir = os.path.join(self.dir_path, dtstr, d) - targ = os.path.join(self.dir_path, d) - os.makedirs(targ, exist_ok=True) - for f in os.listdir(srcdir): - shutil.copy(os.path.join(srcdir, f), targ) - shutil.rmtree(os.path.join(self.dir_path, dtstr)) - tkMessageBox.showinfo("Data Collected", 'data collected from GMN') - self.update_listbox(self.get_bin_list()) - return - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("-d", "--datepatt", type=str, help="date pattern to retrieve") - args = parser.parse_args() - - dir_ = os.path.dirname(os.path.realpath(__file__)) - config_file = os.path.join(dir_, 'config.ini') - noimg_file = os.path.join(dir_, 'noimage.jpg') - - log = logging.getLogger(__name__) - log.setLevel(logging.INFO) - - logdir = os.path.join(os.getenv('TMP'), 'fbcollector') - os.makedirs(logdir, exist_ok=True) - log_file = os.path.join(logdir, log_timestamp() + '.log') - handler = logging.handlers.TimedRotatingFileHandler(log_file, when='D', interval=1) # Log to a different file each day - handler.setLevel(logging.INFO) - - formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', datefmt='%Y/%m/%d %H:%M:%S') - handler.setFormatter(formatter) - log.addHandler(handler) - - ch = logging.StreamHandler(sys.stdout) - ch.setLevel(logging.DEBUG) - formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s: %(message)s', datefmt='%Y/%m/%d %H:%M:%S') - ch.setFormatter(formatter) - log.addHandler(ch) - - # Log program start - log.info("Program start") - log.info(f'config file is {config_file}') - - - # Initialize main window - root = tk.Tk() - root.geometry('+0+0') - targdir = args.datepatt - log.info(f'patt is {targdir}') - - app = fbCollector(root, patt=targdir) - root.iconbitmap(os.path.join(dir_,'ukmda.ico')) - root.protocol('WM_DELETE_WINDOW', app.quitApplication) - - root.mainloop() +# +# UI to manage fireball data collection +# Copyright (C) 2018-2023 Mark McIntyre +# +import os +import sys +import argparse +import logging +import logging.handlers +import datetime +import configparser +import shutil +import platform +import subprocess +import glob +import xmltodict +from PIL import Image +import requests + +import boto3 +import paramiko +from scp import SCPClient + +from meteortools.ukmondb import getECSVs as getecsv +from wmpl.Formats.ECSV import loadECSVs +from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric + +import tkinter as tk +import tkinter.filedialog as tkFileDialog +import tkinter.messagebox as tkMessageBox +from tkinter.simpledialog import askstring +from tkinter import StringVar, Frame, ACTIVE, END, Listbox, Menu, Entry, Button +from tkinter.ttk import Label, Style, LabelFrame, Scrollbar + +from PIL import Image as img +from PIL import ImageTk + +from meteortools.ukmondb import getLiveJpgs, createTxtFile + + +config_file = '' +noimg_file = '' +global_bg = "Black" +global_fg = "Gray" + + +def quitApp(): + # Cleanly exits the app + root.quit() + root.destroy() + + +def log_timestamp(): + """ Returns timestamp for logging. + """ + return datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + + +class StyledButton(Button): + """ Button with style. + """ + def __init__(self, *args, **kwargs): + Button.__init__(self, *args, **kwargs) + + self.configure(foreground = global_fg, background = global_bg, borderwidth = 3) + + +class StyledEntry(Entry): + """ Entry box with style. + """ + def __init__(self, *args, **kwargs): + Entry.__init__(self, *args, **kwargs) + + self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg, disabledbackground = global_bg, disabledforeground = "DimGray") + + +class ConstrainedEntry(StyledEntry): + """ Entry box with constrained values which can be input (e.g. 0-255). + """ + def __init__(self, *args, **kwargs): + StyledEntry.__init__(self, *args, **kwargs) + self.maxvalue = 255 + vcmd = (self.register(self.on_validate), "%P") + self.configure(validate="key", validatecommand=vcmd) + # self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg) + + def disallow(self): + """ Pings a bell on values which are out of bound. + """ + self.bell() + + def update_value(self, maxvalue): + """ Updates values in the entry box. + """ + self.maxvalue = maxvalue + vcmd = (self.register(self.on_validate), "%P") + self.configure(validate="key", validatecommand=vcmd) + + def on_validate(self, new_value): + """ Checks if entered value is within bounds. + """ + try: + if new_value.strip() == "": + return True + value = int(new_value) + if value < 0 or value > self.maxvalue: + self.disallow() + return False + except ValueError: + self.disallow() + return False + + return True + + +class fbCollector(Frame): + + def __init__(self, parent, patt=None): + Frame.__init__(self, parent, bg = global_bg) + parent.configure(bg = global_bg) # Set backgound color + parent.grid_columnconfigure(0, weight=1) + parent.grid_rowconfigure(0, weight=1) + + self.grid(sticky="NSEW") # Expand frame to all directions + self.parent = parent + + self.fb_dir = '' + self.upload_bucket = '' + self.upload_folder = '' + self.live_bucket = '' + self.gmn_key = '' + self.gmn_user = '' + self.gmn_server = '' + self.wmpl_loc = '' + self.wmpl_env = '' + self.rms_loc = '' + self.rms_env = '' + self.selected = {} + self.evtMonTriggered = None + self.review_stack = False + self.soln_outputdir = None + self.log_files_to_keep = 30 + + self.readConfig() + + self.patt = patt + if patt is None: + self.dir_path = self.fb_dir.strip() + else: + self.dir_path = os.path.join(self.fb_dir, patt) + log.info(f"Fireball folder is {self.fb_dir}") + + self.initUI() + if self.dir_path != self.fb_dir: + bin_list = self.get_bin_list() + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + + # Update UI changes + parent.update_idletasks() + parent.update() + + return + + def readConfig(self): + localcfg = configparser.ConfigParser() + localcfg.read(config_file) + self.fb_dir = os.path.expanduser(localcfg['Fireballs']['basedir'].replace('$HOME','~')).replace('\\','/') + self.upload_bucket = localcfg['Fireballs']['uploadbucket'] + self.upload_folder = localcfg['Fireballs']['uploadfolder'] + self.live_bucket = localcfg['Fireballs']['livebucket'] + os.makedirs(self.fb_dir, exist_ok=True) + + try: + self.gmn_key = localcfg['gmnconnection']['gmnkey'] + self.gmn_user = localcfg['gmnconnection']['gmnuser'] + self.gmn_server = localcfg['gmnconnection']['gmnserver'] + except: + pass + + self.wmpl_loc = os.path.expanduser(localcfg['solver']['wmpl_loc'].replace('$HOME','~')).replace('\\','/') + self.wmpl_env= localcfg['solver']['wmpl_env'] + self.rms_loc = os.path.expanduser(localcfg['reduction']['rms_loc'].replace('$HOME','~')).replace('\\','/') + self.rms_env = localcfg['reduction']['rms_env'] + + self.shareloc = os.path.expanduser(localcfg['sharing']['shrfldr'].replace('$HOME','~')).replace('\\','/') + return + + def quitApplication(self): + print('quitting') + logdir = os.path.join(os.getenv('TMP'), 'fbcollector') + logfiles = os.listdir(logdir) + numtokeep = self.log_files_to_keep + if len(logfiles) > numtokeep: + logfiles.sort() + for i in range(len(logfiles)-numtokeep): + try: + os.remove(os.path.join(logdir, logfiles[i])) + except: + pass + quitApp() + + def initUI(self): + """ Initialize GUI elements. + """ + + self.parent.title("Fireball Downloader") + + # Configure the style of each element + s = Style() + s.configure("TButton", padding=(0, 5, 0, 5), font='serif 10') + s.configure('TLabelframe.Label', foreground=global_fg, background=global_bg) + s.configure('TLabelframe', foreground =global_fg, background=global_bg, padding=(3, 3, 3, 3)) + s.configure("TRadiobutton", foreground = global_fg, background = global_bg) + s.configure("TLabel", foreground = global_fg, background = global_bg) + s.configure("TCheckbutton", foreground = global_fg, background = global_bg) + s.configure("Vertical.TScrollbar", background=global_bg, troughcolor = global_bg) + + self.columnconfigure(0, pad=3) + self.columnconfigure(1, pad=3) + self.columnconfigure(2, pad=3) + self.columnconfigure(3, pad=3) + self.columnconfigure(4, pad=3) + self.columnconfigure(5, pad=3) + self.columnconfigure(6, pad=3) + self.columnconfigure(7, pad=3) + self.columnconfigure(8, pad=3) + + self.rowconfigure(0, pad=3) + self.rowconfigure(1, pad=3) + self.rowconfigure(2, pad=3) + self.rowconfigure(3, pad=3) + self.rowconfigure(4, pad=3) + self.rowconfigure(5, pad=3) + self.rowconfigure(6, pad=3) + self.rowconfigure(7, pad=3) + self.rowconfigure(8, pad=3) + self.rowconfigure(9, pad=3) + self.rowconfigure(10, pad=3) + + # Make menu + self.menuBar = Menu(self.parent) + self.parent.config(menu=self.menuBar) + + # File menu + fileMenu = Menu(self.menuBar, tearoff=0) + fileMenu.add_command(label="Load Folder", command=self.loadFolder) + fileMenu.add_command(label="Archive This Folder", command=self.archiveFolder) + fileMenu.add_command(label="Open in Explorer", command=self.openFolder) + fileMenu.add_separator() + fileMenu.add_command(label="Delete This Folder", command=self.delFolder) + fileMenu.add_separator() + fileMenu.add_command(label="Configuration", command=self.showConfig) + fileMenu.add_command(label="View Logs", command=self.viewLogs) + fileMenu.add_separator() + fileMenu.add_command(label="Exit", command=self.quitApplication) + self.menuBar.add_cascade(label="File", underline=0, menu=fileMenu) + + rawMenu = Menu(self.menuBar, tearoff=0) + rawMenu.add_command(label="Get Live Images", command=self.get_data) + rawMenu.add_command(label="Get Videos", command=self.get_vids) + rawMenu.add_command(label="Get Traj Pickle", command=self.get_trajpickle) + rawMenu.add_separator() + rawMenu.add_command(label="Get GMN Raw Data", command=self.getGMNData) + rawMenu.add_separator() + rawMenu.add_command(label="Get ECSVs", command=self.getECSVs) + self.menuBar.add_cascade(label="Raw", underline=0, menu=rawMenu) + + watchMenu = Menu(self.menuBar, tearoff=0) + watchMenu.add_command(label="Get Watchlist", command=self.getWatchlist) + watchMenu.add_command(label="View Watchlist", command=self.viewWatchlist) + watchMenu.add_command(label="Upload Watchlist", command=self.putWatchlist) + watchMenu.add_separator() + watchMenu.add_command(label="Fetch Event Data", command=self.getEventData) + self.menuBar.add_cascade(label="Watchlist", underline=0, menu=watchMenu) + + revMenu = Menu(self.menuBar, tearoff=0) + revMenu.add_command(label="Review Stacks", command=self.checkStacks) + revMenu.add_command(label="Clean Folder", command=self.clean_folder) + self.menuBar.add_cascade(label="Review", underline=0, menu=revMenu) + + solveMenu = Menu(self.menuBar, tearoff=0) + solveMenu.add_command(label="Reduce Data", command=self.reduceCamera) + solveMenu.add_command(label="Toggle Ignore", command=self.ignoreCamera) + solveMenu.add_separator() + solveMenu.add_command(label="View Raw Data", command=self.viewData) + solveMenu.add_command(label="Upload Raw Data", command=self.uploadRaw) + solveMenu.add_separator() + solveMenu.add_command(label="Solve", command=self.solveOrbit) + solveMenu.add_separator() + solveMenu.add_command(label="View Solution", command=self.viewSolution) + solveMenu.add_command(label="Delete Solution", command=self.removeSolution) + solveMenu.add_separator() + solveMenu.add_command(label="Upload Orbit", command=self.uploadOrbit) + self.menuBar.add_cascade(label="Solve", underline=0, menu=solveMenu) + # buttons + self.save_panel = LabelFrame(self, text=' Image Selection ') + self.save_panel.grid(row = 1, columnspan = 2, sticky='WE') + + self.newpatt = StringVar() + self.newpatt.set(self.patt) + + self.patt_entry = StyledEntry(self.save_panel, textvariable = self.newpatt, width = 20) + self.patt_entry.grid(row = 1, column = 1, columnspan = 2, sticky = "W") + save_bmp = StyledButton(self.save_panel, text="Get Images", width = 8, command = lambda: self.get_data()) + save_bmp.grid(row = 1, column = 3) + + save_bmp = StyledButton(self.save_panel, text="Remove", width = 8, command = lambda: self.remove_image()) + save_bmp.grid(row = 1, column = 4) + + + # Listbox + self.scrollbar = Scrollbar(self) + self.listbox = Listbox(self, width = 47, yscrollcommand=self.scrollbar.set, exportselection=0, activestyle = "none", bg = global_bg, fg = global_fg) + self.listbox.config(height = 25) # Listbox size + self.listbox.grid(row = 4, column = 0, rowspan = 7, columnspan = 2, sticky = "NS") # Listbox position + self.scrollbar.grid(row = 4, column = 2, rowspan = 7, sticky = "NS") # Scrollbar size + + self.listbox.bind('<>', self.update_image) + self.scrollbar.config(command = self.listbox.yview) + + # IMAGE + try: + # Show the TV test card image on program start + #noimage_data = img.open('noimage.bin').resize((640,360)) + noimgdata = img.open(noimg_file).resize((640,360)) + noimage = ImageTk.PhotoImage(noimgdata) + except: + noimage = None + + self.imagelabel = Label(self, image = noimage) + self.imagelabel.image = noimage + self.imagelabel.grid(row=3, column=3, rowspan = 4, columnspan = 3) + + # Timestamp label + self.timestamp_label = Label(self, text = "CCNNNN YYYY-MM-DD HH:MM.SS.mms", font=("Courier", 12)) + self.timestamp_label.grid(row = 7, column = 3, sticky = "E") + + def reduceCamera(self): + current_image = self.listbox.get(ACTIVE) + if current_image == '': + return + camid = current_image[3:9] + print('selected camera is', camid) + dirname = os.path.join(self.dir_path, camid) + tmpscr = os.path.join(os.getenv('TEMP'), 'reduce.ps1') + with open(tmpscr, 'w') as outf: + outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {dirname} -c {dirname}/.config\n') + _ = subprocess.run(['powershell.exe', tmpscr]) + frs = glob.glob(os.path.join(dirname, 'FR*.bin')) + if len(frs) > 0: + if not tkMessageBox.askyesno("Rerun", f'{len(frs)} FR files detected - rerun?'): + return + for fr in frs: + with open(tmpscr, 'w') as outf: + outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {fr} -c {dirname}/.config\n') + _ = subprocess.run(['powershell.exe', tmpscr]) + try: + os.remove(tmpscr) + except: + pass + return + + def ignoreCamera(self): + current_image = self.listbox.get(ACTIVE) + if current_image == '': + return + camid = current_image[3:9] + log.info(f'selected camera is {camid}') + dirname = os.path.join(self.dir_path, camid) + allecsvs = glob.glob(os.path.join(dirname, '*.ecsv')) + ecsvs = [e for e in allecsvs if 'REJECT' not in e.upper()] + rejs = [e for e in allecsvs if 'REJECT' in e.upper()] + if len(ecsvs) == 0 and len(rejs) == 0: + log.info('no files to reject or include') + return + elif len(ecsvs) == 1 and len(rejs) == 0: + os.rename(ecsvs[0], ecsvs[0].replace('.ecsv','_REJECT.ecsv')) + jpgname = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) + os.rename(jpgname[0], jpgname[0].replace('.jpg','_REJECT.jpg')) + elif len(ecsvs) == 0 and len(rejs) == 1: + os.rename(rejs[0], rejs[0].replace('_REJECT.ecsv','.ecsv')) + jpgname = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) + os.rename(jpgname[0], jpgname[0].replace('_REJECT.jpg','.jpg')) + else: + # more than one ECSV or REJ file to handle, urk + log.info('urk') + return + bin_list = [line for line in os.listdir(os.path.join(self.dir_path, 'jpgs')) if self.correct_datafile_name(line)] + #print(bin_list) + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + return + + def solveOrbit(self): + log.info('Using ECSV files:') + ecsv_names = [] + ecsv_paths = [] + for entry in sorted(os.walk(self.dir_path), key=lambda x: x[0]): + dir_name, _, file_names = entry + for fn in file_names: + if fn.lower().endswith(".ecsv") and 'REJECT' not in dir_name.upper() and 'REJECT' not in fn.upper(): + + # Add ECSV file, but skip duplicates + if fn not in ecsv_names: + ecsv_paths.append(os.path.join(dir_name, fn)) + ecsv_names.append(fn) + log.info(fn) + if len(ecsv_paths) < 2: + tkMessageBox.showinfo('Warning', 'Need at least two ECSV files') + return + jdt_ref, meteor_list = loadECSVs(ecsv_paths) + mcruns = 20 + max_toffset = 15.0 + velpart = None + vinitht = None + plotallspatial = True + uncertgeom = False + jacchia = False + traj = solveTrajectoryGeneric(jdt_ref, meteor_list, self.dir_path, + max_toffset=max_toffset, monte_carlo=True, mc_runs=mcruns, + geometric_uncert=uncertgeom, plot_all_spatial_residuals=plotallspatial, + show_plots=False, v_init_part=velpart, v_init_ht=vinitht, + show_jacchia=jacchia, enable_OSM_plot=True, mc_cores=8) + tkMessageBox.showinfo('Info', 'Solver Finished') + if traj is not None: + self.soln_outputdir = traj.output_dir + return + + def viewSolution(self): + self.review_stack = False + if not self.soln_outputdir: + solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') + if len(solndir) == 0: + return + solndir = os.path.join(self.dir_path, solndir[0]) + self.soln_outputdir = solndir + bin_list = [line for line in os.listdir(self.soln_outputdir) if self.correct_datafile_name(line)] + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + return + + def removeSolution(self): + if not self.soln_outputdir: + solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') + if len(solndir) == 0: + return + solndir = os.path.join(self.dir_path, solndir[0]) + self.soln_outputdir = solndir + if not tkMessageBox.askyesno("Delete file", f"delete {self.soln_outputdir}?"): + return + shutil.rmtree(os.path.join(self.dir_path, self.soln_outputdir)) + return + + def uploadOrbit(self): + pickles=[] + for path, _, files in os.walk(self.dir_path): + for name in files: + if '.pickle' in name and '_mc_' not in name: + pickles.append(os.path.join(path, name)) + + if len(pickles) == 0: + return + elif len(pickles) == 1: + pickfile = pickles[0] + else: + pickfile = tkFileDialog.askopenfilename(title='Select Orbit Pickle', defaultextension='*.pickle', + initialdir=self.dir_path, initialfile='*.pickle', + filetypes=[('pickles','*.pickle')]) + orbname = os.path.split(pickfile)[1] + tmpdir = os.path.join(self.dir_path, 'tmpzip') + os.makedirs(tmpdir, exist_ok=True) + shutil.copyfile(pickfile, os.path.join(tmpdir, orbname)) + if os.path.isdir(os.path.join(self.dir_path, 'jpgs')): + shutil.copytree(os.path.join(self.dir_path, 'jpgs'), os.path.join(tmpdir, 'jpgs')) + if os.path.isdir(os.path.join(self.dir_path, 'mp4s')): + shutil.copytree(os.path.join(self.dir_path, 'mp4s'), os.path.join(tmpdir, 'mp4s')) + for path, _, files in os.walk(self.dir_path): + for name in files: + if '_dyn_mass_fit' in name: + im = Image.open(os.path.join(path, name)).convert("RGB") + im.save(os.path.join(tmpdir,'jpgs', name[:-4] + '.jpg')) + break + zfname = os.path.join(self.dir_path, orbname[:15]) + shutil.make_archive(zfname,'zip',tmpdir) + try: + shutil.rmtree(tmpdir) + except Exception: + pass + apikey = open(os.path.expanduser('~/.ssh/fbuploadkey.txt')).readlines()[0].strip() + headers = {'Content-type': 'application/zip', 'Slug': orbname[:15], 'apikey': apikey} + url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbname[:15]}.zip' + r = requests.put(url, data=open(zfname+'.zip', 'rb'), headers=headers) #, auth=('username', 'pass')) + #print(r.text) + if r.status_code != 200: + tkMessageBox.showinfo('Warning', 'Problem with upload') + else: + tkMessageBox.showinfo('Info', 'Orbit Uploaded') + return + + def uploadRaw(self): + zfname = os.path.join(os.getenv('TMP'), os.path.basename(self.dir_path)) + log.info(f'zfname is {zfname}') + shutil.make_archive(zfname,'zip',self.dir_path) + try: + targname = os.path.join(self.shareloc, os.path.basename(zfname)+'.zip') + log.info(f'targname is {targname}') + shutil.copyfile(zfname+'.zip', targname) + tkMessageBox.showinfo('Info', 'Raw Data Uploaded to Dropbox') + subprocess.Popen(f'explorer "{self.shareloc}"') + except Exception: + tkMessageBox.showinfo('Warning', 'Problem with upload') + return + + def viewData(self): + dirpath = self.dir_path.replace('/', '\\') + log.info(f'self-dir-path {dirpath}') + subprocess.Popen(f'explorer "{dirpath}"') + return + + def getECSVs(self): + notgotlist=[] + img_list = self.get_bin_list() + for current_image in img_list: + if not self.getOneEcsv(current_image): + notgotlist.append(current_image) + if len(notgotlist) > 0: + tkMessageBox.showinfo('Info', f'No ECSVs for {notgotlist}') + return + + def getOneEcsv(self, current_image): + if current_image[:1] == 'M': + datestr = current_image[1:16] + statid = current_image[-11:-5] + elif current_image[:2] == 'FF': + statid = current_image[3:9] + datestr = current_image[10:29] + else: + return + #dtval = datetime.datetime.strptime(datestr, '%Y%m%d_%H%M%S') + #datestr = dtval.strftime('%Y-%m-%dT%H:%M:%S') + try: + lis = getecsv(statid, datestr, savefiles=True, outdir=os.path.join(self.dir_path, statid)) + for li in lis: + if 'issue getting data' in li: + return False + return True + except Exception: + return False + + def get_bin_list(self): + """ Get a list of image files in a given directory. + """ + if self.dir_path is None: + dirname = tkFileDialog.askdirectory(parent=root,initialdir=self.fb_dir, + title='Please select a directory') + _, thispatt = os.path.split(dirname) + self.dir_path = dirname + self.patt = thispatt + self.newpatt.set(self.patt) + + if self.review_stack is False: + targdir = os.path.join(self.dir_path, 'jpgs') + else: + targdir = os.path.join(self.dir_path, 'stacks') + if os.path.isdir(targdir): + bin_list = [line for line in os.listdir(targdir) if self.correct_datafile_name(line)] + else: + log.info('no jpgs available') + bin_list = [] + return bin_list + + def update_listbox(self, bin_list): + """ Updates the listbox with the current entries. + """ + self.listbox.delete(0, END) + for line in sorted(bin_list): + self.listbox.insert(END, line) + if line not in self.selected: + self.selected[line] = (0, '') + if self.selected[line][0] == 1: + self.listbox.itemconfig(END, fg = 'green') + + def checkStacks(self): + self.review_stack = True + bin_list = self.get_bin_list() + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + + + def loadFolder(self): + self.review_stack = False + self.dir_path = None + self.soln_outputdir = None + bin_list = self.get_bin_list() + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + + def openFolder(self): + dir_path = self.dir_path.replace("/","\\") + os.system(f'explorer.exe {dir_path}') + + def viewLogs(self): + logdir = os.path.join(os.getenv('TMP'), 'fbcollector') + os.system(f'explorer.exe {logdir}') + + def archiveFolder(self): + noimgdata = img.open(noimg_file).resize((640,360)) + noimage = ImageTk.PhotoImage(noimgdata) + self.imagelabel.configure(image = noimage) + self.imagelabel.image = noimage + if self.dir_path == self.fb_dir.strip(): + self.loadFolder() + if self.dir_path is not None and self.dir_path != self.fb_dir: + try: + _, fldr = os.path.split(os.path.normpath(self.dir_path)) + yr = fldr[:4] + archdir = os.path.join(self.fb_dir, yr) + os.makedirs(archdir, exist_ok=True) + zfname = os.path.join(archdir, fldr) + shutil.make_archive(zfname,'zip',self.fb_dir, fldr) + except Exception as e: + log.warning(f'unable to create archive {self.dir_path}') + log.warning(e) + try: + os.chdir(self.fb_dir) + shutil.rmtree(self.dir_path) + except Exception as e: + log.warning(f'unable to remove folder, please do it manually {self.dir_path}') + log.warning(e) + self.dir_path = self.fb_dir + + def delFolder(self): + noimgdata = img.open(noimg_file).resize((640,360)) + noimage = ImageTk.PhotoImage(noimgdata) + self.imagelabel.configure(image = noimage) + self.imagelabel.image = noimage + if self.dir_path is not None and self.dir_path != self.fb_dir: + try: + shutil.rmtree(self.dir_path) + self.dir_path = self.fb_dir + except Exception as e: + log.warning(f'unable to remove {self.dir_path}') + log.warning(e) + + def correct_datafile_name(self, line): + if ('.jpg' in line or '.png' in line) and 'noimage' not in line: + return True + return False + + def removeRelated(self, imgname): + camid = imgname[:6] + mp4s = os.listdir(os.path.join(self.dir_path, 'mp4s')) + badones = [x for x in mp4s if camid in x] + for ba in badones: + os.remove(os.path.join(self.dir_path, 'mp4s', ba)) + jpgs = os.listdir(os.path.join(self.dir_path, 'jpgs')) + badones = [x for x in jpgs if camid in x] + for ba in badones: + os.remove(os.path.join(self.dir_path, 'jpgs', ba)) + fldrs = os.listdir(self.dir_path) + badones = [x for x in fldrs if camid in x] + for ba in badones: + try: + shutil.rmtree(os.path.join(self.dir_path, ba)) + except Exception: + os.remove(os.path.join(self.dir_path, ba)) + os.remove(os.path.join(self.dir_path, 'stacks', imgname)) + + def remove_image(self): + """ Remove the selected image from disk + """ + current_image = self.listbox.get(ACTIVE) + if current_image == '': + return + if not tkMessageBox.askyesno("Delete file", f"delete {current_image}?"): + return + log.info(f'removing {current_image}') + if self.review_stack: + self.removeRelated(current_image) + else: + try: + os.remove(os.path.join(self.dir_path, 'jpgs', current_image)) + except Exception: + pass + xmlf = current_image.replace('P.jpg', '.xml') + try: + os.remove(os.path.join(self.dir_path, 'jpgs', xmlf)) + except Exception: + pass + self.selected[current_image] = (0,'') + self.update_listbox(self.get_bin_list()) + + def update_image(self, thing): + """ When selected, load a new image + """ + try: + # Check if the list is empty. If it is, do nothing. + if self.review_stack: + self.current_image = os.path.join(self.dir_path, 'stacks', self.listbox.get(self.listbox.curselection()[0])) + elif self.soln_outputdir: + self.current_image = os.path.join(self.soln_outputdir, self.listbox.get(self.listbox.curselection()[0])) + else: + self.current_image = os.path.join(self.dir_path, 'jpgs', self.listbox.get(self.listbox.curselection()[0])) + except: + return 0 + + with img.open(self.current_image).resize((640,360)) as imgdata: + thisimage = ImageTk.PhotoImage(imgdata) + self.imagelabel.configure(image = thisimage) + self.imagelabel.image = thisimage + + self.timestamp_label.configure(text = os.path.split(self.current_image)[1]) + return + + def save_image(self): + """ Marks the image as of interest + """ + current_image = self.listbox.get(ACTIVE) + if current_image == '': + return + log.info(f'marking {current_image}') + srcfile = createTxtFile(current_image, self.dir_path) + _, targfile = os.path.split(srcfile) + s3 = boto3.client('s3') + s3.upload_file(srcfile, self.upload_bucket, f'{self.upload_folder}/{targfile}') + cur_index = int(self.listbox.curselection()[0]) + self.listbox.itemconfig(cur_index, fg = 'green') + self.selected[current_image] = (1, srcfile) + + def clean_folder(self): + stacklist = os.listdir(os.path.join(self.dir_path, 'stacks')) + camlist = [x[:6] for x in stacklist if 'stack.jpg' in x] + datalist = os.listdir(self.dir_path) + datalist = [x for x in datalist if 'jpgs' not in x and 'mp4s' not in x and 'stacks' not in x] + for d in datalist: + keep = False + for c in camlist: + if c in d: + keep = True + if keep is False: + try: + os.remove(d) + except Exception: + pass + return + + def get_data(self): + thispatt = self.newpatt.get().strip() + self.patt = thispatt.ljust(15,'0') + self.dir_path = os.path.join(self.fb_dir, self.newpatt.get().strip()) + log.info(f'getting data matching {thispatt}') + os.makedirs(os.path.join(self.dir_path, 'jpgs'), exist_ok=True) + reqdate = datetime.datetime.strptime(self.patt, '%Y%m%d_%H%M%S') + reqdate = reqdate + datetime.timedelta(seconds=-30) + getLiveJpgs(reqdate.strftime('%Y%m%d_%H%M%S'), outdir=os.path.join(self.dir_path, 'jpgs')) + self.renameImages(self.dir_path) + self.update_listbox(self.get_bin_list()) + + def get_trajpickle(self): + basepatt = os.path.split(self.dir_path)[1] + ymd = basepatt[:8] + fullpatt = askstring('Trajectory Name', 'eg 20240101_010203.345_UK', initialvalue=basepatt) + trajpick = f'{basepatt}_trajectory.pickle' + url = f'https://archive.ukmeteors.co.uk/reports/{ymd[:4]}/orbits/{ymd[:6]}/{ymd}/{fullpatt}/{trajpick}' + log.info(f'{url}') + get_response = requests.get(url, stream=True) + if get_response.status_code == 200: + log.info(f'retrieved {trajpick}') + with open(os.path.join(self.dir_path, trajpick), 'wb') as f: + for chunk in get_response.iter_content(chunk_size=4096): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + else: + log.info(f'unable to retrieve {trajpick}, {get_response.status_code}') + return + + def get_vids(self): + jpglist = glob.glob1(os.path.join(self.dir_path,'jpgs'), 'FF*.jpg') + os.makedirs(os.path.join(self.dir_path, 'mp4s'), exist_ok=True) + count = 0 + for jpg in jpglist: + mp4 = jpg.replace('.jpg','.mp4') + ym = mp4[10:16] + url = f'https://archive.ukmeteors.co.uk/img/mp4/{ym[:4]}/{ym}/{mp4}' + get_response = requests.get(url, stream=True) + if get_response.status_code == 200: + log.info(f'retrieved {mp4}') + count += 1 + with open(os.path.join(self.dir_path, 'mp4s', mp4), 'wb') as f: + for chunk in get_response.iter_content(chunk_size=4096): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + log.info(f'retrieved {count} videos') + tkMessageBox.showinfo("Info", f'Retrieved {count} videos') + return + + def renameImages(self, dir_path): + xmllist = glob.glob(os.path.join(dir_path,'jpgs', 'M*.xml')) + for xmlf in xmllist: + jpgf = xmlf.replace('.xml','P.jpg') + if os.path.isfile(jpgf): + xmld = xmltodict.parse(open(xmlf).read()) + realfname = xmld['ufocapture_record']['@cap'].replace('.fits','.jpg') + if not os.path.isfile(os.path.join(dir_path, 'jpgs', realfname)): + os.rename(jpgf, os.path.join(dir_path, 'jpgs', realfname)) + else: + os.remove(jpgf) + os.remove(xmlf) + return + + def viewWatchlist(self): + evtfile = os.path.join(self.fb_dir,'event_watchlist.txt') + if platform.system() == 'Darwin': # macOS + procid = subprocess.Popen(('open', evtfile)) + elif platform.system() == 'Windows': # Windows + procid = subprocess.Popen(('cmd','/c',evtfile)) + else: # linux variants + procid = subprocess.Popen(('xdg-open', evtfile)) + procid.wait() + if not tkMessageBox.askyesno("Upload File", "Upload event watchlist?"): + return + else: + self.putWatchlist() + + def showConfig(self): + if platform.system() == 'Darwin': # macOS + procid = subprocess.Popen(('open', config_file)) + elif platform.system() == 'Windows': # Windows + procid = subprocess.Popen(('cmd','/c',config_file)) + else: # linux variants + procid = subprocess.Popen(('xdg-open', config_file)) + procid.wait() + + def getWatchlist(self): + k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) + c = paramiko.SSHClient() + server=self.gmn_server + user=self.gmn_user + log.info(f'trying {user}@{server} with {self.gmn_key}') + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(hostname = server, username = user, pkey = k) + scpcli = SCPClient(c.get_transport()) + log.info('getting Watchlist') + scpcli.get('./event_watchlist.txt', self.fb_dir) + self.viewWatchlist() + + def putWatchlist(self): + k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) + c = paramiko.SSHClient() + server=self.gmn_server + user=self.gmn_user + log.info(f'trying {user}@{server} with {self.gmn_key}') + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(hostname = server, username = user, pkey = k) + scpcli = SCPClient(c.get_transport()) + log.info('Uploading watchlist') + evtfile = os.path.join(self.fb_dir,'event_watchlist.txt') + scpcli.put(evtfile, 'event_watchlist.txt') + self.evtMonTriggered = datetime.datetime.now() + datetime.timedelta(minutes=10) + return + + def getEventData(self): + evtdate = self.newpatt.get().strip() + if len(evtdate) < 15: + tkMessageBox.showinfo("Warning", f'Need seconds in the event date field {evtdate}') + return + cmd = os.path.join(os.path.dirname(__file__), 'download_events.sh') + f' {evtdate} 1' + if ':' in cmd: + drv = cmd[0].lower() + cmd = '/mnt/' + drv + cmd[2:] + cmd = cmd.replace('\\','/') + log.warning(f'executing {cmd}') + if self.evtMonTriggered is None: + tkMessageBox.showinfo("Warning", 'Event Monitor has not been triggered') + return + if datetime.datetime.now() < self.evtMonTriggered: + tkMessageBox.showinfo("Warning", f'Wait till at least {self.evtMonTriggered.strftime("%H:%M:%S")}') + return + os.chdir(self.fb_dir) + log.info(f'getting data for {evtdate}') + procid = subprocess.Popen(('bash','-c', cmd)) + procid.wait() + tkMessageBox.showinfo("Info", 'Done') + return + + def getGMNData(self): + camlist = [line for line in os.listdir(self.dir_path) if self.correct_datafile_name(line)] + dts=[] + camids=[] + for cam in camlist: + camid,_ = os.path.splitext(cam) + spls = camid.split('_') + if camid[:2] == 'FF': + camids.append(spls[1]) + dts.append(camid[10:25]) + else: + camids.append(spls[-1][:6]) + dts.append(camid[1:16]) + dts.sort() + dtstr = f'{dts[0]}.000000' + stationlist = ','.join(map(str, camids)) + server=self.gmn_server + user=self.gmn_user + log.info('getting data from GMN') + k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) + c = paramiko.SSHClient() + log.info(f'trying {user}@{server} with {self.gmn_key}') + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + c.connect(hostname = server, username = user, pkey = k) + command = f'source ~/anaconda3/etc/profile.d/conda.sh && conda activate wmpl && python scripts/extract_fireball.py {dtstr} {stationlist}' + log.info(f'running {command}') + + _, stdout, stderr = c.exec_command(command, timeout=900) + for line in iter(stdout.readline, ""): + log.info(line) + for line in iter(stderr.readline, ""): + print(line, end="") + scpcli = SCPClient(c.get_transport()) + log.info('done, collecting output') + indir = os.path.join(f'event_extract/{dtstr}/') + scpcli.get(indir, self.dir_path, recursive=True) + command = f'rm -Rf event_extract/{dtstr}' + log.info(f'running {command}') + _, stdout, stderr = c.exec_command(command, timeout=120) + for line in iter(stdout.readline, ""): + log.info(line) + for line in iter(stderr.readline, ""): + log.info(line) + dirs = os.listdir(os.path.join(self.dir_path, dtstr)) + for d in dirs: + srcdir = os.path.join(self.dir_path, dtstr, d) + targ = os.path.join(self.dir_path, d) + os.makedirs(targ, exist_ok=True) + for f in os.listdir(srcdir): + shutil.copy(os.path.join(srcdir, f), targ) + shutil.rmtree(os.path.join(self.dir_path, dtstr)) + tkMessageBox.showinfo("Data Collected", 'data collected from GMN') + self.update_listbox(self.get_bin_list()) + return + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--datepatt", type=str, help="date pattern to retrieve") + args = parser.parse_args() + + dir_ = os.path.dirname(os.path.realpath(__file__)) + config_file = os.path.join(dir_, 'config.ini') + noimg_file = os.path.join(dir_, 'noimage.jpg') + + log = logging.getLogger(__name__) + log.setLevel(logging.INFO) + + logdir = os.path.join(os.getenv('TMP'), 'fbcollector') + os.makedirs(logdir, exist_ok=True) + log_file = os.path.join(logdir, log_timestamp() + '.log') + handler = logging.handlers.TimedRotatingFileHandler(log_file, when='D', interval=1) # Log to a different file each day + handler.setLevel(logging.INFO) + + formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', datefmt='%Y/%m/%d %H:%M:%S') + handler.setFormatter(formatter) + log.addHandler(handler) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s: %(message)s', datefmt='%Y/%m/%d %H:%M:%S') + ch.setFormatter(formatter) + log.addHandler(ch) + + # Log program start + log.info("Program start") + log.info(f'config file is {config_file}') + + + # Initialize main window + root = tk.Tk() + root.geometry('+0+0') + targdir = args.datepatt + log.info(f'patt is {targdir}') + + app = fbCollector(root, patt=targdir) + root.iconbitmap(os.path.join(dir_,'ukmda.ico')) + root.protocol('WM_DELETE_WINDOW', app.quitApplication) + + root.mainloop() diff --git a/fbCollector/requirements.txt b/fbCollector/requirements.txt index 26e2479d..e3cff877 100644 --- a/fbCollector/requirements.txt +++ b/fbCollector/requirements.txt @@ -1,8 +1,8 @@ -# requirements for the fbCollector tool -# Copyright (C) 2018-2023 Mark McIntyre -boto3 -Pillow -paramiko -meteortools -scp -pyqtgraph +# requirements for the fbCollector tool +# Copyright (C) 2018-2023 Mark McIntyre +boto3 +Pillow +paramiko +meteortools +scp +pyqtgraph From 1782b6379b4a8e2d50e2a9c7f94027f7fe7ac96f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 27 Aug 2025 22:59:29 +0100 Subject: [PATCH 24/83] moving stuff around --- utils/addVidorImg.ps1 | 100 +++++++-------- utils/analysis.ini | 58 ++++----- utils/getBrightest.ps1 | 80 ++++++------ utils/helperfunctions.ps1 | 248 +++++++++++++++++++------------------- utils/uploadOrbit.ps1 | 140 ++++++++++----------- 5 files changed, 313 insertions(+), 313 deletions(-) diff --git a/utils/addVidorImg.ps1 b/utils/addVidorImg.ps1 index 933c75e4..7b03fa64 100644 --- a/utils/addVidorImg.ps1 +++ b/utils/addVidorImg.ps1 @@ -1,50 +1,50 @@ -# Copyright (C) 2018-2023 Mark McIntyre -# -# powershell script to upload an extra video or image of an event, given a trajectory ID - -$loc = get-location -if ($args.count -lt 1) { - write-output "Usage: addVideoorImg.ps1 trajpath" - write-output " First create a folder for the trajectory eg 20241113_192754.345_UK" - write-output " Copy any additional mp4, mov or jpgs into it, then run this script" - exit -}else { - $pth = $args[0] -} -set-location $PSScriptRoot -# load the helper functions -. .\helperfunctions.ps1 -$ini=get-inicontent analysis.ini -$fbfolder = $ini['localdata']['fbfolder'].replace('$HOME',$home) -$fbfolder = $fbfolder + "/fireballs" -set-location $fbfolder -$traj = (split-path -leaf $pth) - -$yr=$traj.Substring(0,4) -$ym=$traj.Substring(0,6) -$ymd=$traj.Substring(0,8) -$pick=$traj.substring(0,15) + '_trajectory.pickle' -$zipf=$traj.substring(0,15) + '.zip' - -wget https://archive.ukmeteors.co.uk/reports/${yr}/orbits/${ym}/${ymd}/${traj}/${pick} -O ${pth}\${pick} - -if ((test-path $pth\jpgs) -eq $false) { mkdir $pth\jpgs } -if ((test-path $pth\mp4s) -eq $false) { mkdir $pth\mp4s } - -move-item $pth\*.jpg $pth\jpgs -move-item $pth\*.mp4 $pth\mp4s - -compress-archive -path $pth\* -DestinationPath $pth\$zipf -force - -$apikey=(Get-Content ~/.ssh/fbuploadkey.txt) -$headers = @{ - 'Content-type' = 'application/zip' - 'Slug' = ${zipf} - 'apikey'= ${apikey} -} -$url = "https://api.ukmeteors.co.uk/fireballfiles?orbitfile=${zipf}" - -invoke-webrequest -uri $url -infile $pth\$zipf -Method PUT -Headers $headers - -Write-Output "uploaded $zipf at $(get-date)" -set-location $loc +# Copyright (C) 2018-2023 Mark McIntyre +# +# powershell script to upload an extra video or image of an event, given a trajectory ID + +$loc = get-location +if ($args.count -lt 1) { + write-output "Usage: addVideoorImg.ps1 trajpath" + write-output " First create a folder for the trajectory eg 20241113_192754.345_UK" + write-output " Copy any additional mp4, mov or jpgs into it, then run this script" + exit +}else { + $pth = $args[0] +} +set-location $PSScriptRoot +# load the helper functions +. .\helperfunctions.ps1 +$ini=get-inicontent analysis.ini +$fbfolder = $ini['localdata']['fbfolder'].replace('$HOME',$home) +$fbfolder = $fbfolder + "/fireballs" +set-location $fbfolder +$traj = (split-path -leaf $pth) + +$yr=$traj.Substring(0,4) +$ym=$traj.Substring(0,6) +$ymd=$traj.Substring(0,8) +$pick=$traj.substring(0,15) + '_trajectory.pickle' +$zipf=$traj.substring(0,15) + '.zip' + +wget https://archive.ukmeteors.co.uk/reports/${yr}/orbits/${ym}/${ymd}/${traj}/${pick} -O ${pth}\${pick} + +if ((test-path $pth\jpgs) -eq $false) { mkdir $pth\jpgs } +if ((test-path $pth\mp4s) -eq $false) { mkdir $pth\mp4s } + +move-item $pth\*.jpg $pth\jpgs +move-item $pth\*.mp4 $pth\mp4s + +compress-archive -path $pth\* -DestinationPath $pth\$zipf -force + +$apikey=(Get-Content ~/.ssh/fbuploadkey.txt) +$headers = @{ + 'Content-type' = 'application/zip' + 'Slug' = ${zipf} + 'apikey'= ${apikey} +} +$url = "https://api.ukmeteors.co.uk/fireballfiles?orbitfile=${zipf}" + +invoke-webrequest -uri $url -infile $pth\$zipf -Method PUT -Headers $headers + +Write-Output "uploaded $zipf at $(get-date)" +set-location $loc diff --git a/utils/analysis.ini b/utils/analysis.ini index 5be0f402..3d0d32ee 100644 --- a/utils/analysis.ini +++ b/utils/analysis.ini @@ -1,29 +1,29 @@ -# Parameters that define your local configuration -# -# -[pylib] -pylib=$HOME/OneDrive/dev/ukmda-dataprocessing/archive/ukmon_pylib - -[localdata] -FBFOLDER=$HOME/OneDrive/pictures/meteors -MTHLYFOLDER=$HOME/OneDrive/pictures/meteors/mthlystacks - -[rms] -RMS_INSTALLED=1 -RMS_LOC=$HOME/OneDrive/dev/RMS -RMS_ENV=RMS - -[wmpl] -WMPL_INSTALLED=1 -WMPL_LOC=$HOME/OneDrive/dev/WesternMeteorPyLib -WMPL_ENV=wmpl - -[gmn] -IDFILE=~/.ssh/gmnanalysis - -# camid to hostname mappings, yours may be different -[stations] -UK0006=uk0006 -UK000F=uk000f -UK002F=uk002f -UK001L=uk001l +# Parameters that define your local configuration +# +# +[pylib] +pylib=$HOME/OneDrive/dev/ukmda-dataprocessing/archive/ukmon_pylib + +[localdata] +FBFOLDER=$HOME/OneDrive/pictures/meteors +MTHLYFOLDER=$HOME/OneDrive/pictures/meteors/mthlystacks + +[rms] +RMS_INSTALLED=1 +RMS_LOC=$HOME/OneDrive/dev/RMS +RMS_ENV=RMS + +[wmpl] +WMPL_INSTALLED=1 +WMPL_LOC=$HOME/OneDrive/dev/WesternMeteorPyLib +WMPL_ENV=wmpl + +[gmn] +IDFILE=~/.ssh/gmnanalysis + +# camid to hostname mappings, yours may be different +[stations] +UK0006=uk0006 +UK000F=uk000f +UK002F=uk002f +UK001L=uk001l diff --git a/utils/getBrightest.ps1 b/utils/getBrightest.ps1 index 1cbe529e..ba9933b2 100644 --- a/utils/getBrightest.ps1 +++ b/utils/getBrightest.ps1 @@ -1,41 +1,41 @@ -# powershell script to get the 30 best detections from the previous night -# for sharing on social media -# note that you will have to go through and delete non-meteors from the images - -# requirements - you must clone WesternMeteorPyLib and ukmda-dataprocessing from github -# to your local development space. I use onedrive\dev for my code - set this location in $codeloc - - -# copyright (c) Mark McIntyre, 2025- - -set-location $PSScriptRoot - -# load the helper functions -. .\helperfunctions.ps1 -$ini=get-inicontent analysis.ini -$bdir = $ini['localdata']['fbfolder'].replace('$HOME',$home) -$bdir = $bdir + "/brightest" -set-location $bdir - -$outdir = $bdir.replace('\','/') - -$wmplloc = $ini['wmpl']['wmpl_loc'].replace('$HOME',$home) -$repdir = $ini['pylib']['pylib'].replace('$HOME',$home) - -$env:pythonpath="$wmplloc" -Push-Location $repdir - -conda activate ukmon-shared - -Write-Output "working..." -if ($args.count -eq 0) { - python -c "from reports.findBestMp4s import getBestNSingles;getBestNSingles(numtoget=30,outdir='$outdir')" -}else{ - $reqdate = $args[0] - python -c "from reports.findBestMp4s import getBestNSingles;getBestNSingles(numtoget=30,outdir='$outdir', reqdate='$reqdate')" -} - -$outdirw = $outdir.replace('/','\') -explorer "$outdirw" - +# powershell script to get the 30 best detections from the previous night +# for sharing on social media +# note that you will have to go through and delete non-meteors from the images + +# requirements - you must clone WesternMeteorPyLib and ukmda-dataprocessing from github +# to your local development space. I use onedrive\dev for my code - set this location in $codeloc + + +# copyright (c) Mark McIntyre, 2025- + +set-location $PSScriptRoot + +# load the helper functions +. .\helperfunctions.ps1 +$ini=get-inicontent analysis.ini +$bdir = $ini['localdata']['fbfolder'].replace('$HOME',$home) +$bdir = $bdir + "/brightest" +set-location $bdir + +$outdir = $bdir.replace('\','/') + +$wmplloc = $ini['wmpl']['wmpl_loc'].replace('$HOME',$home) +$repdir = $ini['pylib']['pylib'].replace('$HOME',$home) + +$env:pythonpath="$wmplloc" +Push-Location $repdir + +conda activate ukmon-shared + +Write-Output "working..." +if ($args.count -eq 0) { + python -c "from reports.findBestMp4s import getBestNSingles;getBestNSingles(numtoget=30,outdir='$outdir')" +}else{ + $reqdate = $args[0] + python -c "from reports.findBestMp4s import getBestNSingles;getBestNSingles(numtoget=30,outdir='$outdir', reqdate='$reqdate')" +} + +$outdirw = $outdir.replace('/','\') +explorer "$outdirw" + Pop-Location \ No newline at end of file diff --git a/utils/helperfunctions.ps1 b/utils/helperfunctions.ps1 index 4f1c77a8..b914e30d 100644 --- a/utils/helperfunctions.ps1 +++ b/utils/helperfunctions.ps1 @@ -1,125 +1,125 @@ -# Copyright (C) 2018-2023 Mark McIntyre - -# helper functions - -Function Get-IniContent { - <# - .Synopsis - Gets the content of an INI file - - .Description - Gets the content of an INI file and returns it as a hashtable - - .Notes - Author : Oliver Lipkau - Blog : http://oliver.lipkau.net/blog/ - Source : https://github.com/lipkau/PsIni - http://gallery.technet.microsoft.com/scriptcenter/ea40c1ef-c856-434b-b8fb-ebd7a76e8d91 - Version : 1.0 - 2010/03/12 - Initial release - 1.1 - 2014/12/11 - Typo (Thx SLDR) - Typo (Thx Dave Stiff) - - #Requires -Version 2.0 - - .Inputs - System.String - - .Outputs - System.Collections.Hashtable - - .Parameter FilePath - Specifies the path to the input file. - - .Example - $FileContent = Get-IniContent "C:\myinifile.ini" - ----------- - Description - Saves the content of the c:\myinifile.ini in a hashtable called $FileContent - - .Example - $inifilepath | $FileContent = Get-IniContent - ----------- - Description - Gets the content of the ini file passed through the pipe into a hashtable called $FileContent - - .Example - C:\PS>$FileContent = Get-IniContent "c:\settings.ini" - C:\PS>$FileContent["Section"]["Key"] - ----------- - Description - Returns the key "Key" of the section "Section" from the C:\settings.ini file - - .Link - Out-IniFile - #> - - [CmdletBinding()] - Param( - [ValidateNotNullOrEmpty()] - [ValidateScript({(Test-Path $_) -and ((Get-Item $_).Extension -eq ".ini")})] - [Parameter(ValueFromPipeline=$True,Mandatory=$True)] - [string]$FilePath - ) - - Begin - {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function started"} - - Process - { - Write-Verbose "$($MyInvocation.MyCommand.Name):: Processing file: $Filepath" - - $ini = @{} - switch -regex -file $FilePath - { - "^\[(.+)\]$" # Section - { - $section = $matches[1] - $ini[$section] = @{} - $CommentCount = 0 - } - "^(;.*)$" # Comment - { - if (!($section)) - { - $section = "No-Section" - $ini[$section] = @{} - } - $value = $matches[1] - $CommentCount = $CommentCount + 1 - $name = "Comment" + $CommentCount - $ini[$section][$name] = $value - } - "(.+?)\s*=\s*(.*)" # Key - { - if (!($section)) - { - $section = "No-Section" - $ini[$section] = @{} - } - $name,$value = $matches[1..2] - $ini[$section][$name] = $value - } - } - Write-Verbose "$($MyInvocation.MyCommand.Name):: Finished Processing file: $FilePath" - Return $ini - } - - End - {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function ended"} -} - -Function Test-CommandExists -{ - Param ($command) - $oldPreference = $ErrorActionPreference - $ErrorActionPreference = ‘stop’ - try {if(Get-Command $command){RETURN $true}} - Catch {Write-Host “$command does not exist”; RETURN $false} - Finally {$ErrorActionPreference=$oldPreference} -} #end function test-CommandExists - -Function New-TemporaryFolder { - # Make a new folder based upon a TempFileName - $T="$($Env:temp)\tmp$([convert]::tostring((get-random 65535),16).padleft(4,'0')).tmp" - New-Item -ItemType Directory -Path $T +# Copyright (C) 2018-2023 Mark McIntyre + +# helper functions + +Function Get-IniContent { + <# + .Synopsis + Gets the content of an INI file + + .Description + Gets the content of an INI file and returns it as a hashtable + + .Notes + Author : Oliver Lipkau + Blog : http://oliver.lipkau.net/blog/ + Source : https://github.com/lipkau/PsIni + http://gallery.technet.microsoft.com/scriptcenter/ea40c1ef-c856-434b-b8fb-ebd7a76e8d91 + Version : 1.0 - 2010/03/12 - Initial release + 1.1 - 2014/12/11 - Typo (Thx SLDR) + Typo (Thx Dave Stiff) + + #Requires -Version 2.0 + + .Inputs + System.String + + .Outputs + System.Collections.Hashtable + + .Parameter FilePath + Specifies the path to the input file. + + .Example + $FileContent = Get-IniContent "C:\myinifile.ini" + ----------- + Description + Saves the content of the c:\myinifile.ini in a hashtable called $FileContent + + .Example + $inifilepath | $FileContent = Get-IniContent + ----------- + Description + Gets the content of the ini file passed through the pipe into a hashtable called $FileContent + + .Example + C:\PS>$FileContent = Get-IniContent "c:\settings.ini" + C:\PS>$FileContent["Section"]["Key"] + ----------- + Description + Returns the key "Key" of the section "Section" from the C:\settings.ini file + + .Link + Out-IniFile + #> + + [CmdletBinding()] + Param( + [ValidateNotNullOrEmpty()] + [ValidateScript({(Test-Path $_) -and ((Get-Item $_).Extension -eq ".ini")})] + [Parameter(ValueFromPipeline=$True,Mandatory=$True)] + [string]$FilePath + ) + + Begin + {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function started"} + + Process + { + Write-Verbose "$($MyInvocation.MyCommand.Name):: Processing file: $Filepath" + + $ini = @{} + switch -regex -file $FilePath + { + "^\[(.+)\]$" # Section + { + $section = $matches[1] + $ini[$section] = @{} + $CommentCount = 0 + } + "^(;.*)$" # Comment + { + if (!($section)) + { + $section = "No-Section" + $ini[$section] = @{} + } + $value = $matches[1] + $CommentCount = $CommentCount + 1 + $name = "Comment" + $CommentCount + $ini[$section][$name] = $value + } + "(.+?)\s*=\s*(.*)" # Key + { + if (!($section)) + { + $section = "No-Section" + $ini[$section] = @{} + } + $name,$value = $matches[1..2] + $ini[$section][$name] = $value + } + } + Write-Verbose "$($MyInvocation.MyCommand.Name):: Finished Processing file: $FilePath" + Return $ini + } + + End + {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function ended"} +} + +Function Test-CommandExists +{ + Param ($command) + $oldPreference = $ErrorActionPreference + $ErrorActionPreference = ‘stop’ + try {if(Get-Command $command){RETURN $true}} + Catch {Write-Host “$command does not exist”; RETURN $false} + Finally {$ErrorActionPreference=$oldPreference} +} #end function test-CommandExists + +Function New-TemporaryFolder { + # Make a new folder based upon a TempFileName + $T="$($Env:temp)\tmp$([convert]::tostring((get-random 65535),16).padleft(4,'0')).tmp" + New-Item -ItemType Directory -Path $T } \ No newline at end of file diff --git a/utils/uploadOrbit.ps1 b/utils/uploadOrbit.ps1 index db92db60..a512a129 100644 --- a/utils/uploadOrbit.ps1 +++ b/utils/uploadOrbit.ps1 @@ -1,70 +1,70 @@ -# Copyright (C) 2018-2023 Mark McIntyre -# -# powershell script to create a zip file of a solution and upload it - -$loc = get-location -if ($args.count -lt 1) { - $curdir = get-location -}else { - $curdir = resolve-path $args[0] -} -set-location $PSScriptRoot -# load the helper functions -. .\helperfunctions.ps1 -$ini=get-inicontent analysis.ini -Set-Location $Loc - -Add-Type -AssemblyName System.Windows.Forms - -conda activate $ini['wmpl']['wmpl_env'] -$wmplloc = $ini['wmpl']['wmpl_loc'].replace('$HOME',$home) -$env:pythonpath="$wmplloc;$env:pythonpath" - -write-output "Find the pickle" -$picklefile= (Get-ChildItem $curdir\*.pickle -r).fullname -if ($picklefile -is [Array] -or $picklefile.length -eq 0) { - $FileBrowser = New-Object System.Windows.Forms.OpenFileDialog -Property @{ InitialDirectory = $curdir; Filter = 'pickles (*.pickle)|*.pickle'; Title='Select orbit pickle' } - $null = $FileBrowser.ShowDialog() - $picklefile = $filebrowser.filename - if ( $picklefile -eq "" ) { - Write-Output "Cancelled" - set-location $loc - exit - } -} -write-output "find the orbit full name" -$srcdir = ((get-item $picklefile).directoryname).replace('\','/') -$pickname = (get-item $picklefile).name -$origdir = (python -c "from wmpl.Utils.Pickling import loadPickle;pick = loadPickle('${srcdir}','${pickname}');print(pick.output_dir)") -$orbname = split-path $origdir -leaf -if ((test-path $curdir\$orbname) -eq 0 ) -{ - mkdir -force $curdir\$orbname - move-item $picklefile $curdir\$orbname\ -} -$picklefile = (Get-ChildItem $picklefile -r).fullname - -write-output "look for jpgs and mp4s" -mkdir -force $curdir\jpgs -$jpgdir = resolve-path $curdir\jpgs -Move-Item $curdir\*.jpg $jpgdir -mkdir -force $curdir\mp4s -$mp4dir = resolve-path $curdir\mp4s -Move-Item $curdir\*.mp4 $mp4dir - -write-output "Creating zip file" -if ((test-path $env:temp\$orbname\mp4s) -eq 0 ) {mkdir $env:temp\$orbname\mp4s | out-null} -if ((test-path $env:temp\$orbname\jpgs) -eq 0 ) {mkdir $env:temp\$orbname\jpgs | out-null} -copy-item $picklefile $env:temp\$orbname -copy-item $jpgdir\*.jpg $env:temp\$orbname\jpgs -copy-item $mp4dir\*.mp4 $env:temp\$orbname\mp4s -if ((test-path $env:temp\$orbname.zip) -eq 1 ) { remove-item $env:temp\$orbname.zip} -compress-archive -destinationpath "$env:temp\$orbname.zip" -path "$env:temp\$orbname\*" -remove-item $env:temp\$orbname\* -Recurse - -write-output "Uploading $orbname.zip" -curl -X PUT -H "Content-Type:application/zip" --data-binary "@$env:temp\$orbname.zip" "https://api.ukmeteors.co.uk/fireballfiles?orbitfile=$orbname.zip" -move-item $env:temp\$orbname.zip $curdir -Force -Write-Output "uploaded at $(get-date)" -set-location $loc -conda deactivate +# Copyright (C) 2018-2023 Mark McIntyre +# +# powershell script to create a zip file of a solution and upload it + +$loc = get-location +if ($args.count -lt 1) { + $curdir = get-location +}else { + $curdir = resolve-path $args[0] +} +set-location $PSScriptRoot +# load the helper functions +. .\helperfunctions.ps1 +$ini=get-inicontent analysis.ini +Set-Location $Loc + +Add-Type -AssemblyName System.Windows.Forms + +conda activate $ini['wmpl']['wmpl_env'] +$wmplloc = $ini['wmpl']['wmpl_loc'].replace('$HOME',$home) +$env:pythonpath="$wmplloc;$env:pythonpath" + +write-output "Find the pickle" +$picklefile= (Get-ChildItem $curdir\*.pickle -r).fullname +if ($picklefile -is [Array] -or $picklefile.length -eq 0) { + $FileBrowser = New-Object System.Windows.Forms.OpenFileDialog -Property @{ InitialDirectory = $curdir; Filter = 'pickles (*.pickle)|*.pickle'; Title='Select orbit pickle' } + $null = $FileBrowser.ShowDialog() + $picklefile = $filebrowser.filename + if ( $picklefile -eq "" ) { + Write-Output "Cancelled" + set-location $loc + exit + } +} +write-output "find the orbit full name" +$srcdir = ((get-item $picklefile).directoryname).replace('\','/') +$pickname = (get-item $picklefile).name +$origdir = (python -c "from wmpl.Utils.Pickling import loadPickle;pick = loadPickle('${srcdir}','${pickname}');print(pick.output_dir)") +$orbname = split-path $origdir -leaf +if ((test-path $curdir\$orbname) -eq 0 ) +{ + mkdir -force $curdir\$orbname + move-item $picklefile $curdir\$orbname\ +} +$picklefile = (Get-ChildItem $picklefile -r).fullname + +write-output "look for jpgs and mp4s" +mkdir -force $curdir\jpgs +$jpgdir = resolve-path $curdir\jpgs +Move-Item $curdir\*.jpg $jpgdir +mkdir -force $curdir\mp4s +$mp4dir = resolve-path $curdir\mp4s +Move-Item $curdir\*.mp4 $mp4dir + +write-output "Creating zip file" +if ((test-path $env:temp\$orbname\mp4s) -eq 0 ) {mkdir $env:temp\$orbname\mp4s | out-null} +if ((test-path $env:temp\$orbname\jpgs) -eq 0 ) {mkdir $env:temp\$orbname\jpgs | out-null} +copy-item $picklefile $env:temp\$orbname +copy-item $jpgdir\*.jpg $env:temp\$orbname\jpgs +copy-item $mp4dir\*.mp4 $env:temp\$orbname\mp4s +if ((test-path $env:temp\$orbname.zip) -eq 1 ) { remove-item $env:temp\$orbname.zip} +compress-archive -destinationpath "$env:temp\$orbname.zip" -path "$env:temp\$orbname\*" +remove-item $env:temp\$orbname\* -Recurse + +write-output "Uploading $orbname.zip" +curl -X PUT -H "Content-Type:application/zip" --data-binary "@$env:temp\$orbname.zip" "https://api.ukmeteors.co.uk/fireballfiles?orbitfile=$orbname.zip" +move-item $env:temp\$orbname.zip $curdir -Force +Write-Output "uploaded at $(get-date)" +set-location $loc +conda deactivate From ac9faf0eb40aa77b4b7ac03873b4c743bf66e8aa Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 28 Aug 2025 13:55:16 +0100 Subject: [PATCH 25/83] getExtrafilesv2: update docker tag and wmpl refresh proces --- .../getExtraFilesV2/template.yaml | 2 +- .../getExtraFilesV2/update_wmpl.sh | 1 + .../getExtraFilesV2/wmpl__init__.py_fixed.py | 45 +++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py diff --git a/archive/samfunctions/getExtraFilesV2/template.yaml b/archive/samfunctions/getExtraFilesV2/template.yaml index a2d87dfe..54dd9924 100644 --- a/archive/samfunctions/getExtraFilesV2/template.yaml +++ b/archive/samfunctions/getExtraFilesV2/template.yaml @@ -38,4 +38,4 @@ Resources: Metadata: Dockerfile: Dockerfile DockerContext: ./pythoncode - DockerTag: python3.9-v1 + DockerTag: python3.13-v1 diff --git a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh b/archive/samfunctions/getExtraFilesV2/update_wmpl.sh index e06a917b..57d3fb6f 100644 --- a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh +++ b/archive/samfunctions/getExtraFilesV2/update_wmpl.sh @@ -14,3 +14,4 @@ rm -Rf ./WesternMeteorPyLib/wmpl/MetSim rm -Rf ./WesternMeteorPyLib/wmpl/CAMO rm -f ./WesternMeteorPyLib/wmpl/Utils/DynamicMassFit.py rm -f ./WesternMeteorPyLib/wmpl/Utils/PlotMap_OSM.py +cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py diff --git a/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py b/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py new file mode 100644 index 00000000..dc34c35d --- /dev/null +++ b/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py @@ -0,0 +1,45 @@ +# This is a fixed-up version of WesternMeteorPyLib/wmpl/__init__.py which is python 3.12+ compatible + +""" Import all submodules. """ + +import importlib +import pkgutil +import sys + +# Excluded packages +exclude = ["MetSim.ML", "GUI"] + +__all__ = [] +for loader, module_name, is_pkg in pkgutil.walk_packages(__path__): + + # Skip the config module + if 'Config' in module_name: + continue + + # Skip Supracenter + if 'Supracenter' in module_name: + continue + + + ### Skip exluded packages ### + skip_package = False + for exclude_test in exclude: + if exclude_test in module_name: + skip_package = True + break + + if skip_package: + continue + + ### ### + + + __all__.append(module_name) + if sys.version_info.major > 2 and sys.version_info.minor > 12: + module_spec = loader.find_spec(module_name) + if module_spec is not None: + module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(module) + else: + module = loader.find_module(module_name).load_module(module_name) + exec('%s = module' % module_name) From 1502a4c26356386092b993583049e0a7daf9faae Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 28 Aug 2025 15:11:54 +0100 Subject: [PATCH 26/83] upgrade to newer base image --- archive/containers/trajsolver/Dockerfile | 12 +++++++----- archive/containers/trajsolver/build.ps1 | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/archive/containers/trajsolver/Dockerfile b/archive/containers/trajsolver/Dockerfile index 276e5a11..ff8b6ff5 100644 --- a/archive/containers/trajsolver/Dockerfile +++ b/archive/containers/trajsolver/Dockerfile @@ -1,18 +1,20 @@ # Copyright (C) 2018-2023 Mark McIntyre -FROM continuumio/miniconda3:4.11.0 +#FROM continuumio/miniconda3:4.11.0 +FROM continuumio/miniconda3:24.7.1-0 -RUN conda install -y -c conda-forge "numpy<1.25.0" scipy cython pytz pyqt +#RUN conda install -y -c conda-forge "numpy<1.25.0" scipy cython pytz pyqt +RUN conda install -y -c conda-forge "numpy<2" scipy cython pytz pyqt RUN conda install -y -c conda-forge jplephem ephem RUN conda install -y -c conda-forge basemap basemap-data-hires RUN conda install -y -c conda-forge pandas cartopy -RUN conda install -y matplotlib=3.4.3 +RUN conda install -y matplotlib RUN pip install boto3 RUN apt-get update && apt-get -y upgrade RUN conda install -y -c conda-forge numba COPY WesternMeteorPyLib/ ./WesternMeteorPyLib -ENV PYTHONPATH ./WesternMeteorPyLib -ENV PROJ_LIB ./ +ENV PYTHONPATH=./WesternMeteorPyLib +ENV PROJ_LIB=./ COPY *.py ./ #ENV NUMPY_EXPERIMENTAL_DTYPE_API 1 diff --git a/archive/containers/trajsolver/build.ps1 b/archive/containers/trajsolver/build.ps1 index 8d63c887..4a4dac04 100644 --- a/archive/containers/trajsolver/build.ps1 +++ b/archive/containers/trajsolver/build.ps1 @@ -17,7 +17,7 @@ write-output "building $imagename in $accid" $yn=read-host -prompt "update WMPL?" if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" } -if ($env -eq "test") { copy-item awskeys.test awskeys} +if ($env -eq "test") { move-item awskeys.test awskeys} docker build . -t ${imagename} if (! $?) @@ -26,7 +26,7 @@ if (! $?) exit } set-location $loc -if ($env -eq "test") { Remove-Item awskeys} +if ($env -eq "test") { move-item awskeys awskeys.test } $yn=read-host -prompt "upload to ECR?" if ($yn.tolower() -eq "y") { From 6fa06e3c94684e200f4b0ac295bfc5d6f277f4ef Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 28 Aug 2025 15:13:11 +0100 Subject: [PATCH 27/83] remove legacy and deprecated code dont copy unused files to website --- archive/containers/trajsolver/trajsolver.py | 50 +++++++++++---------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py index 7155a235..ff1ce8f2 100644 --- a/archive/containers/trajsolver/trajsolver.py +++ b/archive/containers/trajsolver/trajsolver.py @@ -36,7 +36,7 @@ def runCorrelator(dir_path, time_beg, time_end): trajectory_constraints.geometric_uncert = not uncerttime # Clock for measuring script time - t1 = datetime.datetime.utcnow() + t1 = datetime.datetime.now(datetime.timezone.utc) # If auto run is enabled, compute the time range to use event_time_range = None @@ -115,7 +115,7 @@ def runCorrelator(dir_path, time_beg, time_end): tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, distribute=distribute, enableOSM=True) tc.run(event_time_range=event_time_range) - print("Total run time: {:s}".format(str(datetime.datetime.utcnow() - t1))) + print("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1))) return @@ -136,9 +136,7 @@ def getSourceAndTargets(): webbucket = webpth[:webpth.find('/')] webpth = webpth[webpth.find('/')+1:] - oldoutb = os.getenv('OLDOUTB', default=None) - oldwebb = os.getenv('OLDWEBB', default=None) - return srcbucket, srcpth, outbucket, outpth, webbucket, webpth, oldwebb, oldoutb + return srcbucket, srcpth, outbucket, outpth, webbucket, webpth # extra args for setting the MIME type when uploading to S3. @@ -168,10 +166,30 @@ def getExtraArgs(fname): return extraargs +def checkIfFileNeeded(filename): + + # update this if there's an additional file created by WMPL that we want + # to use on the website + if 'orbit_top.png' in filename or 'orbit_side.png' in filename or 'ground_track.png' in filename: + return True + if 'velocities.png' in filename or 'lengths.png' in filename or 'lags_all.png' in filename: + return True + if 'abs_mag.png' in filename or 'abs_mag_ht.png' in filename or 'report.txt' in filename: + return True + if 'all_angular_residuals.png' in filename or 'all_spatial_total_residuals_height.png' in filename: + return True + if 'trajectory.pickle' in filename: + return True + return False + + #push solutions to the website, and push the pickle and report to shared bucket -def pushToWebsite(s3, localfldr, webbucket, webfldr, outbucket, outpth, oldwebb=None, oldoutb=None): +def pushToWebsite(s3, localfldr, webbucket, webfldr, outbucket, outpth): for root, _, files in os.walk(localfldr): for fil in files: + if not checkIfFileNeeded(fil): + print(f'skipping {fil}') + continue fullname = os.path.join(localfldr, root, fil) fname = os.path.split(fil)[1] orbname = os.path.split(root)[1] @@ -181,26 +199,10 @@ def pushToWebsite(s3, localfldr, webbucket, webfldr, outbucket, outpth, oldwebb= targkey = f'{webfldr}/{yr}/orbits/{ym}/{ymd}/{orbname}/{fname}' print(f'uploading {fname} to s3://{webbucket}/{targkey}') s3.meta.client.upload_file(fullname, webbucket, targkey, ExtraArgs = getExtraArgs(fname)) - if oldwebb: - print(f' and uploading to {oldwebb}') - try: - s3.meta.client.upload_file(fullname, oldwebb, targkey, ExtraArgs = getExtraArgs(fname)) - except: - print('unable to push to old website') - else: - print('not pushing to old website') if 'report' in fname or 'pickle' in fname: targkey = f'{outpth}/trajectories/{yr}/{ym}/{ymd}/{orbname}/{fname}' print(f'uploading {fname} to s3://{outbucket}/{targkey}') s3.meta.client.upload_file(fullname, outbucket, targkey, ExtraArgs = getExtraArgs(fname)) - if oldoutb: - print(f' and uploading to {oldoutb}') - try: - s3.meta.client.upload_file(fullname, oldoutb, targkey, ExtraArgs = getExtraArgs(fname)) - except: - print('unable to push to old share') - else: - print('not pushing to old share') return @@ -238,7 +240,7 @@ def startup(srcfldr, startdt, enddt, isTest=False): os.makedirs(canddir, exist_ok = True) s3 = getS3Client() - srcbucket, srcpth, outbucket, outpth, webbucket, webpth, oldwebb, oldoutb = getSourceAndTargets() + srcbucket, srcpth, outbucket, outpth, webbucket, webpth = getSourceAndTargets() if isTest is True: outpth = os.path.join(outpth, 'test') @@ -263,7 +265,7 @@ def startup(srcfldr, startdt, enddt, isTest=False): # reacquire tokens just in case the 1hour time limit on chained roles is exceeded s3 = getS3Client() - pushToWebsite(s3, trajfldr, webbucket, webpth, outbucket, outpth, oldwebb=oldwebb, oldoutb=oldoutb) + pushToWebsite(s3, trajfldr, webbucket, webpth, outbucket, outpth) fname = f'{srcfldr}.json' jsonfile = os.path.join(localfldr, 'processed_trajectories.json') From 8d028302d87497542e07dcb757e268a09f6f9ddd Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 28 Aug 2025 15:56:31 +0100 Subject: [PATCH 28/83] remove ref to old EE account --- archive/ukmon_pylib/traj/README.md | 2 +- archive/ukmon_pylib/traj/distributeCandidates.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/archive/ukmon_pylib/traj/README.md b/archive/ukmon_pylib/traj/README.md index 8b6280a2..7cb699d7 100644 --- a/archive/ukmon_pylib/traj/README.md +++ b/archive/ukmon_pylib/traj/README.md @@ -12,7 +12,7 @@ routines associated with trajectory solving and analysis of results * plotOSMGroundTrack.py - creates an OS-style ground map from a trajectory pickle. * showerAssociation.py - creates shower association information from a trajectory pickle. -* clusdetails-ee.txt - files autogenerated by Terraform containing cluster details for the distributed solver +* clusdetails-mda.txt - files autogenerated by Terraform containing cluster details for the solver * clusdetails-mm.txt * clusdetails.txt diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py index 1c025408..3030c9b0 100644 --- a/archive/ukmon_pylib/traj/distributeCandidates.py +++ b/archive/ukmon_pylib/traj/distributeCandidates.py @@ -19,9 +19,7 @@ def getClusterDetails(templdir): sts = boto3.client('sts') accid = sts.get_caller_identity()['Account'] - if accid == '822069317839': - clusdetails = os.path.join(templdir, 'clusdetails-ee.txt') - elif accid == '183798037734': + if accid == '183798037734': clusdetails = os.path.join(templdir, 'clusdetails-mda.txt') else: clusdetails = os.path.join(templdir, 'clusdetails-mm.txt') From ac92d54f483535611f737d4b95b1dfdd8e85047c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 28 Aug 2025 23:23:33 +0100 Subject: [PATCH 29/83] add code to remove unused files from website to save space --- .../maintenance/compressOldArchiveData.py | 385 +++++++++++------- 1 file changed, 233 insertions(+), 152 deletions(-) diff --git a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py index efa3fce9..cf8b449f 100644 --- a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py +++ b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py @@ -1,152 +1,233 @@ -# copyright 2023- Mark McIntyre -# all rights reserved - -""" - a function to compress the ukmda-shared/archive data from before the current year - -usage: - 1) python compressOldArchiveData.py - this will scan and compress ALL data from prior years - 2) python compressOldArchiveData.py "archive/somefolder" will scan and compress only the named location folder - -note that any MP4 or AVI files that have been uploaded to this area are removed by this process. The videos should be -on the website in ukmda-website/img/mp4s in mp4 format -""" - -import boto3 -import zipfile -import io -import sys -import datetime - -maxyear = datetime.datetime.now().year - 1 - -s3 = boto3.client('s3') -s3res = boto3.resource('s3') - - -def listLocations(srcbucket='ukmda-shared', prefixstr='archive/'): - """ - list all top level folders in a particular prefix - parameters: - srcbucket: [string] bucket name, default ukmda-shared - prefixstr: [string] prefix, default archive/ - note must have terminal slash - - returns: - list of prefixes - """ - response = s3.list_objects_v2(Bucket=srcbucket, Prefix=prefixstr, Delimiter='/') - if 'CommonPrefixes' in response: - return response['CommonPrefixes'] - else: - return [] - - -def listCameras(srcbucket='ukmda-shared', location='archive/Tackley/'): - response = s3.list_objects_v2(Bucket=srcbucket, Prefix=location, Delimiter='/') - if 'CommonPrefixes' in response: - return response['CommonPrefixes'] - else: - return [] - - -def listYears(srcbucket='ukmda-shared', camid='archive/Tackley/c1/'): - response = s3.list_objects_v2(Bucket=srcbucket, Prefix=camid, Delimiter='/') - if 'CommonPrefixes' in response: - return response['CommonPrefixes'] - else: - return [] - - -def getAllS3Objects(s3, **base_kwargs): - continuation_token = None - while True: - list_kwargs = dict(MaxKeys=1000, **base_kwargs) - if continuation_token: - list_kwargs['ContinuationToken'] = continuation_token - response = s3.list_objects_v2(**list_kwargs) - yield from response.get('Contents', []) - if not response.get('IsTruncated'): # At the end of the list? - break - continuation_token = response.get('NextContinuationToken') - - -def deleteFiles(flist, srcbucket='ukmda-shared'): - chunk_size = 900 - chunked_list = [flist[i:i + chunk_size] for i in range(0, len(flist), chunk_size)] - for ch in chunked_list: - delete_keys = {'Objects': []} - delete_keys['Objects'] = [{'Key': k} for k in ch] - s3.delete_objects(Bucket=srcbucket, Delete=delete_keys) - return - - -def compressObjects(srcbucket='ukmda-shared', camprefix=None): - lcmprefixes = [] - if camprefix is None: - locations = listLocations() - for location in locations: - camids = listCameras(location=location['Prefix']) - for camid in camids: - years = listYears(camid=camid['Prefix']) - for year in years: - lcmprefix = year['Prefix'][:-1] - lcmprefixes.append(lcmprefix) - else: - spls = camprefix[:-1].split('/') - print(spls) - if len(spls) < 3: - camids = listCameras(location=camprefix) - for camid in camids: - years = listYears(camid=camid['Prefix']) - for year in years: - lcmprefix = year['Prefix'][:-1] - lcmprefixes.append(lcmprefix) - else: - years = listYears(camid=camprefix) - for year in years: - lcmprefix = year['Prefix'][:-1] - lcmprefixes.append(lcmprefix) - - for lcmprefix in lcmprefixes: - spls = lcmprefix.split('/') - zfname = f'{spls[0]}/{spls[1]}/{spls[1]}_{spls[2]}_{spls[3]}.zip' - try: - if int(spls[3]) > maxyear or 'miscellan' in spls[3] or 'profiles' in spls[3]: - continue - except Exception: - pass - if 'testpi' in lcmprefix: - continue - bucket = s3res.Bucket(srcbucket) - files = [os.key for os in bucket.objects.filter(Prefix=lcmprefix)] - mp4files = [file for file in files if '.mp4' in file] - deleteFiles(mp4files) - avifiles = [file for file in files if '.avi' in file] - deleteFiles(avifiles) - zip_buffer = io.BytesIO() - with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zipf: - for key in files: - if '.mp4' not in key and '.avi' not in key: - print(key) - data = s3.get_object(Bucket=srcbucket, Key=key) - content = data['Body'].read() - zipf.writestr(key, content) - - # Upload the zip file to the target bucket - zip_buffer.seek(0) - print(f'uploading {zfname}') - s3.upload_fileobj(zip_buffer, srcbucket, zfname) - # now delete the files - print(f'deleting files from {lcmprefix}') - deleteFiles(files) - - -if __name__ == '__main__': - source_bucket = 'ukmda-shared' - prefix_str = None - if len(sys.argv) > 1: - prefix_str = sys.argv[1] - if prefix_str[-1] != '/': - prefix_str = prefix_str + '/' - print(f'compressing {prefix_str}') - compressObjects(source_bucket, prefix_str) +# copyright 2023- Mark McIntyre +# all rights reserved + +""" + a set of functions to + - compress the ukmda-shared/archive data from before the current year + - remove unused but easily recreated files from the website + - remove links to the old downloadable zip file + +usage: + 1) python compressOldArchiveData.py - will scan and compress ALL data from prior years + 2) python compressOldArchiveData.py "archive/somefolder" - will scan and compress only the named location folder + 3) python compressOldArchiveData.py prune_website - will scan and remove unused files from the website + 3) python compressOldArchiveData.py prune_website reports/2024/202401 - will scan and remove unused files + from the named location and below + +note that any MP4 or AVI files that have been uploaded to this area are removed by this process. The videos should be +on the website in ukmda-website/img/mp4s in mp4 format +""" + +import boto3 +import zipfile +import io +import os +import datetime +import argparse + +maxyear = datetime.datetime.now().year - 1 + +s3 = boto3.client('s3') +s3res = boto3.resource('s3') + + +def listLocations(srcbucket='ukmda-shared', prefixstr='archive/'): + """ + list all top level folders in a particular prefix + parameters: + srcbucket: [string] bucket name, default ukmda-shared + prefixstr: [string] prefix, default archive/ - note must have terminal slash + + returns: + list of prefixes + """ + response = s3.list_objects_v2(Bucket=srcbucket, Prefix=prefixstr, Delimiter='/') + if 'CommonPrefixes' in response: + return response['CommonPrefixes'] + else: + return [] + + +def listCameras(srcbucket='ukmda-shared', location='archive/Tackley/'): + response = s3.list_objects_v2(Bucket=srcbucket, Prefix=location, Delimiter='/') + if 'CommonPrefixes' in response: + return response['CommonPrefixes'] + else: + return [] + + +def listYears(srcbucket='ukmda-shared', camid='archive/Tackley/c1/'): + response = s3.list_objects_v2(Bucket=srcbucket, Prefix=camid, Delimiter='/') + if 'CommonPrefixes' in response: + return response['CommonPrefixes'] + else: + return [] + + +""" +Not used +def getAllS3Objects(s3, **base_kwargs): + continuation_token = None + while True: + list_kwargs = dict(MaxKeys=1000, **base_kwargs) + if continuation_token: + list_kwargs['ContinuationToken'] = continuation_token + response = s3.list_objects_v2(**list_kwargs) + yield from response.get('Contents', []) + if not response.get('IsTruncated'): # At the end of the list? + break + continuation_token = response.get('NextContinuationToken') +""" + +def deleteFiles(flist, srcbucket='ukmda-shared'): + chunk_size = 900 + chunked_list = [flist[i:i + chunk_size] for i in range(0, len(flist), chunk_size)] + for ch in chunked_list: + delete_keys = {'Objects': []} + delete_keys['Objects'] = [{'Key': k} for k in ch] + s3.delete_objects(Bucket=srcbucket, Delete=delete_keys) + return + + +def compressObjects(srcbucket='ukmda-shared', camprefix=None): + lcmprefixes = [] + if camprefix is None: + locations = listLocations() + for location in locations: + camids = listCameras(location=location['Prefix']) + for camid in camids: + years = listYears(camid=camid['Prefix']) + for year in years: + lcmprefix = year['Prefix'][:-1] + lcmprefixes.append(lcmprefix) + else: + spls = camprefix[:-1].split('/') + print(spls) + if len(spls) < 3: + camids = listCameras(location=camprefix) + for camid in camids: + years = listYears(camid=camid['Prefix']) + for year in years: + lcmprefix = year['Prefix'][:-1] + lcmprefixes.append(lcmprefix) + else: + years = listYears(camid=camprefix) + for year in years: + lcmprefix = year['Prefix'][:-1] + lcmprefixes.append(lcmprefix) + + for lcmprefix in lcmprefixes: + spls = lcmprefix.split('/') + zfname = f'{spls[0]}/{spls[1]}/{spls[1]}_{spls[2]}_{spls[3]}.zip' + try: + if int(spls[3]) > maxyear or 'miscellan' in spls[3] or 'profiles' in spls[3]: + continue + except Exception: + pass + if 'testpi' in lcmprefix: + continue + bucket = s3res.Bucket(srcbucket) + files = [os.key for os in bucket.objects.filter(Prefix=lcmprefix)] + mp4files = [file for file in files if '.mp4' in file] + deleteFiles(mp4files) + avifiles = [file for file in files if '.avi' in file] + deleteFiles(avifiles) + zip_buffer = io.BytesIO() + with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zipf: + for key in files: + if '.mp4' not in key and '.avi' not in key: + print(key) + data = s3.get_object(Bucket=srcbucket, Key=key) + content = data['Body'].read() + zipf.writestr(key, content) + + # Upload the zip file to the target bucket + zip_buffer.seek(0) + print(f'uploading {zfname}') + s3.upload_fileobj(zip_buffer, srcbucket, zfname) + # now delete the files + print(f'deleting files from {lcmprefix}') + deleteFiles(files) + + +def updateIndexes(idxs, source_bucket): + for idx in idxs: + if len(idx.split('/')) < 7: + continue + s3.download_file(source_bucket, idx, f'/tmp/oldindex.html') + lis = open('/tmp/oldindex.html', 'r').readlines() + with open('/tmp/newindex.html', 'w') as outf: + for li in lis: + if 'download a zip of the' in li: + continue + outf.write(li) + extraargs = {'ContentType': 'text/html'} + s3.upload_file('/tmp/newindex.html', source_bucket, idx, ExtraArgs=extraargs) + return + + +def pruneObjects(source_bucket, prefix_str): + spls = prefix_str.split('/') + bucket = s3res.Bucket(source_bucket) + if len(spls) < 3: + years = listYears(source_bucket, prefix_str) + years = [x['Prefix'] for x in years if 'reports/20' in x['Prefix']] + else: + years = [f'reports/{spls[1]}/'] + for yr in years: + print(f'processing {yr}') + yr_prefix = f'{yr}orbits/' + mths = listYears(source_bucket, yr_prefix) + mths = [x['Prefix'] for x in mths if 'csv/' not in x['Prefix'] and 'plots/' not in x['Prefix']] + for mth in mths: + print(f'processing {mth}') + files = [os.key for os in bucket.objects.filter(Prefix=mth)] + print('purging zip files') + zipfs = [file for file in files if '.zip' in file] + deleteFiles(zipfs, source_bucket) + if len(zipfs) > 0: + print('updating indexes') + idxs = [file for file in files if '/index.html' in file] + updateIndexes(idxs, source_bucket) + print('purging pngs') + spatres = [file for file in files if '_spatial_residuals.png' in file and '_all' not in file] + deleteFiles(spatres, source_bucket) + print('purging kml and ftpdetect') + others = [file for file in files if '.kml' in file or 'FTPdetectinfo' in file] + deleteFiles(others, source_bucket) + print('purging csv') + csvs = [file for file in files if '.csv' in file] + deleteFiles(csvs, source_bucket) + return + + +if __name__ == '__main__': + prefix_str = None + + arg_parser = argparse.ArgumentParser(description='Compress, prune or clear down older data') + + arg_parser.add_argument('command_str', nargs=1, metavar='COMMAND_STR', type=str, + help='action to take. Options are COMPRESS or PRUNE to compress archive files or prune the website.') + + arg_parser.add_argument('-f', '--folder', help="""area to act on, eg "Tackley" or "2025/202504".""") + + args = arg_parser.parse_args() + + if args.command_str[0].upper() == 'COMPRESS': + source_bucket = 'ukmda-shared' + if args.folder: + prefix_str = 'archive/' + prefix_str + prefix_str = args.folder + if prefix_str[-1] != '/': + prefix_str = prefix_str + '/' + print(f'compressing {prefix_str}') + compressObjects(source_bucket, prefix_str) + + if args.command_str[0].upper() == 'PRUNE': + source_bucket = 'ukmda-website' + if args.folder: + prefix_str = 'reports/' + args.folder + if prefix_str[-1] != '/': + prefix_str = prefix_str + '/' + print(f'pruning {prefix_str}') + pruneObjects(source_bucket, prefix_str) From 104a2b8f81cdd2560f9c39375b446b8df309ebae Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 30 Aug 2025 15:11:16 +0100 Subject: [PATCH 30/83] set sensible timeperiod for transition to Glacier_IR --- archive/terraform/ukmda/ukmdashared_bucket.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/terraform/ukmda/ukmdashared_bucket.tf b/archive/terraform/ukmda/ukmdashared_bucket.tf index 692de9e9..26a0c013 100644 --- a/archive/terraform/ukmda/ukmdashared_bucket.tf +++ b/archive/terraform/ukmda/ukmdashared_bucket.tf @@ -106,7 +106,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" { } transition { - days = 45 + days = 180 storage_class = "GLACIER_IR" } } From 32500a5f2e67263b59236c975d21ccc29d0c4aaf Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 30 Aug 2025 15:11:35 +0100 Subject: [PATCH 31/83] more support functions to clear down and compress and reorganize old data --- .../maintenance/compressOldArchiveData.py | 122 +++++++++++++++++- 1 file changed, 116 insertions(+), 6 deletions(-) diff --git a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py index cf8b449f..bbed0b0f 100644 --- a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py +++ b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py @@ -150,12 +150,114 @@ def compressObjects(srcbucket='ukmda-shared', camprefix=None): deleteFiles(files) +def moveJpgsAndMp4s(source_bucket, yr, ym): + print(f'moving jpgs and mp4s from {ym}') + bucket = s3res.Bucket(source_bucket) + files = [os.key for os in bucket.objects.filter(Prefix=f'reports/{yr}/orbits/{ym}/')] + jpgs = [x for x in files if '.jpg' in x or '.mp4' in x] + print(f'moving {len(jpgs)} jpgs or mp4s') + for jpg in jpgs: + print(jpg) + keyname = os.path.split(jpg)[1] + if '.jpg' in keyname: + newkey = f'img/single/{yr}/{ym}/{keyname}' + else: + newkey = f'img/mp4/{yr}/{ym}/{keyname}' + s3.copy_object(Bucket=source_bucket, + CopySource={'Bucket':source_bucket,'Key':jpg}, + Key=newkey, MetadataDirective='COPY') + s3.delete_object(Bucket=source_bucket, Key=jpg) + jpgfldrs = [os.path.split(x)[0] for x in jpgs] + jpgfldrs = list(set(jpgfldrs)) + jpgfldrs.sort() + print(f'updating {len(jpgfldrs)} indexes') + for fldr in jpgfldrs: + print(fldr) + idx = f'{fldr}/index.html' + rootdir = os.path.split(idx)[0] + try: + s3.download_file(source_bucket, idx, f'/tmp/oldindex.html') + except Exception: + continue + lis = open('/tmp/oldindex.html', 'r').readlines() + # skip files that dont have the line in them + mtch = [li for li in lis if '.jpg' in li or '.mp4' in li] + if len(mtch) == 0: + continue + with open('/tmp/newindex.html', 'w') as outf: + for li in lis: + if '.jpg' in li and '/img/single/' not in li: + newpth=f'/img/single/{yr}/{ym}/' + li = li.replace('a href="', f'a href="{newpth}') + li = li.replace('img src="', f'img src="{newpth}') + if '.mp4' in li and '/img/mp4/' not in li: + newpth=f'/img/mp4/{yr}/{ym}/' + li = li.replace('a href="', f'a href="{newpth}') + li = li.replace('source src="', f'source src="{newpth}') + li = li.replace(rootdir, '') + li = li.replace('//', '/') + li = li.replace('//', '/') + outf.write(li) + extraargs = {'ContentType': 'text/html'} + s3.upload_file('/tmp/newindex.html', source_bucket, idx, ExtraArgs=extraargs) + print('done') + +def fixBrokenIndexes(source_bucket, yr, ym): + bucket = s3res.Bucket(source_bucket) + files = [os.key for os in bucket.objects.filter(Prefix=f'reports/{yr}/orbits/{ym}/')] + idxs = [x for x in files if 'index.html' in x] + for idx in idxs: + print(idx) + s3.download_file(source_bucket, idx, f'/tmp/oldindex.html') + lis = open('/tmp/oldindex.html', 'r').readlines() + rootdir = os.path.split(idx)[0] + # skip files that dont have the line in them + mtch = [li for li in lis if '.jpg' in li or '.mp4' in li] + if len(mtch) == 0: + continue + with open('/tmp/newindex.html', 'w') as outf: + for li in lis: + if '.jpg' in li and '/img/single/' not in li: + newpth=f'/img/single/{yr}/{ym}/' + li = li.replace('a href="', f'a href="{newpth}') + li = li.replace('img src="', f'img src="{newpth}') + if '.mp4' in li and '/img/mp4/' not in li: + newpth=f'/img/mp4/{yr}/{ym}/' + li = li.replace('a href="', f'a href="{newpth}') + li = li.replace('source src="', f'source src="{newpth}') + li = li.replace(rootdir, '') + li = li.replace('//', '/') + li = li.replace('//', '/') + outf.write(li) +# for li in lis: +# if '.jpg' in li: +# oldpth = f'/img/single/{yr}/{ym}' +# newpth = f'/img/single/{yr}/{ym}/' +# li = li.replace(f'a href="{oldpth}', f'a href="{newpth}') +# li = li.replace(f'img src="{oldpth}', f'img src="{newpth}') +# if '.mp4' in li: +# oldpth = f'/img/single/{yr}/{ym}' +# newpth=f'/img/mp4/{yr}/{ym}/' +# li = li.replace(f'a href="{oldpth}', f'a href="{newpth}') +# li = li.replace(f'source src="{oldpth}', f'source src="{newpth}') +# li = li.replace(rootdir, '') +# li = li.replace('//', '/') +# li = li.replace('//', '/') +# outf.write(li) + extraargs = {'ContentType': 'text/html'} + s3.upload_file('/tmp/newindex.html', source_bucket, idx, ExtraArgs=extraargs) + + def updateIndexes(idxs, source_bucket): for idx in idxs: if len(idx.split('/')) < 7: continue s3.download_file(source_bucket, idx, f'/tmp/oldindex.html') lis = open('/tmp/oldindex.html', 'r').readlines() + # skip files that dont have the line in them + mtch = [li for li in lis if 'download a zip of the' in li] + if len(mtch) == 0: + continue with open('/tmp/newindex.html', 'w') as outf: for li in lis: if 'download a zip of the' in li: @@ -166,7 +268,7 @@ def updateIndexes(idxs, source_bucket): return -def pruneObjects(source_bucket, prefix_str): +def pruneObjects(source_bucket, prefix_str, force_reindex=False): spls = prefix_str.split('/') bucket = s3res.Bucket(source_bucket) if len(spls) < 3: @@ -177,15 +279,18 @@ def pruneObjects(source_bucket, prefix_str): for yr in years: print(f'processing {yr}') yr_prefix = f'{yr}orbits/' - mths = listYears(source_bucket, yr_prefix) - mths = [x['Prefix'] for x in mths if 'csv/' not in x['Prefix'] and 'plots/' not in x['Prefix']] + if len(spls) > 3: + mths = [f'{yr_prefix}{spls[2]}/'] + else: + mths = listYears(source_bucket, yr_prefix) + mths = [x['Prefix'] for x in mths if 'csv/' not in x['Prefix'] and 'plots/' not in x['Prefix']] for mth in mths: print(f'processing {mth}') files = [os.key for os in bucket.objects.filter(Prefix=mth)] print('purging zip files') zipfs = [file for file in files if '.zip' in file] deleteFiles(zipfs, source_bucket) - if len(zipfs) > 0: + if len(zipfs) > 0 or force_reindex: print('updating indexes') idxs = [file for file in files if '/index.html' in file] updateIndexes(idxs, source_bucket) @@ -211,6 +316,8 @@ def pruneObjects(source_bucket, prefix_str): arg_parser.add_argument('-f', '--folder', help="""area to act on, eg "Tackley" or "2025/202504".""") + arg_parser.add_argument('-i', '--reindex', action='store_true', help="""force recreation of indexes".""") + args = arg_parser.parse_args() if args.command_str[0].upper() == 'COMPRESS': @@ -229,5 +336,8 @@ def pruneObjects(source_bucket, prefix_str): prefix_str = 'reports/' + args.folder if prefix_str[-1] != '/': prefix_str = prefix_str + '/' - print(f'pruning {prefix_str}') - pruneObjects(source_bucket, prefix_str) + force_reindex = False + if args.reindex: + force_reindex = True + print(f'pruning {prefix_str} and forcing reindex') + pruneObjects(source_bucket, prefix_str, force_reindex) From c0d582b169176702f340c40103d47d4dab152091 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 30 Aug 2025 15:11:59 +0100 Subject: [PATCH 32/83] remove zip file link from Orbit pages --- .../getExtraFilesV2/pythoncode/createOrbitPageIndex.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py b/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py index 35f5497a..8d0cd2cb 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py +++ b/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py @@ -31,10 +31,10 @@ def createOrbitPageIndex(fldr, websitebucket, s3): lis = sumf.readlines() idxf.writelines(lis) - zipf = orbitname + '.zip' - if os.path.isfile(os.path.join(fldr, zipf)): - idxf.write(f"Click here to download a zip of the raw and processed data.\n") - idxf.write("\n") + #zipf = orbitname + '.zip' + #if os.path.isfile(os.path.join(fldr, zipf)): + # idxf.write(f"Click here to download a zip of the raw and processed data.\n") + #idxf.write("\n") idxf.write("

Detailed report below graphs

\n") idxf.write("

Click on an image to see a larger view

\n") From c0d2cddccbe38d79050483bd5330ac7451003382 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 30 Aug 2025 20:30:07 +0100 Subject: [PATCH 33/83] Update calcengine to c8g and add DNS tags and IMDSv2 metadata option --- archive/terraform/ukmda/ec2.tf | 70 +++++++++++++++++----------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/ec2.tf index f91de92c..97b2a567 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/ec2.tf @@ -1,13 +1,16 @@ # Copyright (C) 2018-2023 Mark McIntyre resource "aws_instance" "calc_server" { - ami = "ami-0df2d8f6def0bd716" - instance_type = "c6g.4xlarge" - iam_instance_profile = aws_iam_instance_profile.calcserverrole.name - key_name = aws_key_pair.marks_key.key_name + ami = "ami-0df2d8f6def0bd716" + instance_type = "c8g.4xlarge" + iam_instance_profile = aws_iam_instance_profile.calcserverrole.name + key_name = aws_key_pair.marks_key.key_name + force_destroy = false tags = { - "Name" = "Calcengine" - "billingtag" = "ukmda" + "Name" = "Calcengine" + "billingtag" = "ukmda" + "Route53FQDN" = "calcengine.ukmeteors.co.uk" + "DNSRecordType" = "A" } root_block_device { tags = { @@ -16,9 +19,12 @@ resource "aws_instance" "calc_server" { } volume_size = 100 } - network_interface { + primary_network_interface { network_interface_id = aws_network_interface.calcserver_if.id - device_index = 0 + } + + metadata_options { + http_tokens = "required" } } @@ -33,8 +39,6 @@ resource "aws_network_interface" "calcserver_if" { tags = { "Name" = "calcengine" "billingtag" = "ukmda" - "Route53FQDN" = "calcengine.ukmeteors.co.uk" - "DNSRecordType" = "A" } } ################################################ @@ -42,14 +46,14 @@ resource "aws_network_interface" "calcserver_if" { ################################################ resource "aws_instance" "ubuntu_calc_server" { - ami = "ami-0bdf149a42243bde8" - instance_type = "c6g.4xlarge" - iam_instance_profile = aws_iam_instance_profile.calcserverrole.name - key_name = aws_key_pair.marks_key.key_name + ami = "ami-0bdf149a42243bde8" + instance_type = "c6g.4xlarge" + iam_instance_profile = aws_iam_instance_profile.calcserverrole.name + key_name = aws_key_pair.marks_key.key_name tags = { - "Name" = "Calcengine2" - "billingtag" = "ukmda" - "Route53FQDN" = "calcengine2.ukmeteors.co.uk" + "Name" = "calcengine_ub" + "billingtag" = "ukmda" + "Route53FQDN" = "calcengine_ub.ukmeteors.co.uk" "DNSRecordType" = "A" } root_block_device { @@ -59,9 +63,11 @@ resource "aws_instance" "ubuntu_calc_server" { } volume_size = 100 } - network_interface { + primary_network_interface { network_interface_id = aws_network_interface.ubuntu_calcserver_if.id - device_index = 0 + } + metadata_options { + http_tokens = "required" } } @@ -85,14 +91,14 @@ resource "aws_network_interface" "ubuntu_calcserver_if" { resource "aws_instance" "admin_server" { - ami = "ami-0c1ce90bf42d9802b" - instance_type = "t2.micro" - iam_instance_profile = aws_iam_instance_profile.S3FullAccess.name - key_name = aws_key_pair.marks_key.key_name + ami = "ami-0c1ce90bf42d9802b" + instance_type = "t2.micro" + iam_instance_profile = aws_iam_instance_profile.S3FullAccess.name + key_name = aws_key_pair.marks_key.key_name tags = { - "Name" = "AdminServer" - "billingtag" = "ukmda" - "Route53FQDN" = "adminserver.ukmeteors.co.uk" + "Name" = "AdminServer" + "billingtag" = "ukmda" + "Route53FQDN" = "adminserver.ukmeteors.co.uk" "DNSRecordType" = "A" } root_block_device { @@ -102,14 +108,8 @@ resource "aws_instance" "admin_server" { } volume_size = 8 } -} - -# elastic IP attached to the calc_server -resource "aws_eip" "adminserver_eip" { - instance = aws_instance.calc_server.id - domain = "vpc" - tags = { - billingtag = "ukmda" + metadata_options { + http_tokens = "required" } -} +} From c7076f7d984a1ab8026b290c8b8406afb7f9f289 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 30 Aug 2025 20:30:35 +0100 Subject: [PATCH 34/83] tidy up and align with latest aws and terraform standard --- archive/terraform/ukmda/loggingbucket.tf | 8 ++--- archive/terraform/ukmda/ukmdaadmin_bucket.tf | 3 +- archive/terraform/ukmda/ukmdalive_bucket.tf | 3 +- archive/terraform/ukmda/ukmdashared_bucket.tf | 31 +++++++++---------- .../terraform/ukmda/ukmdawebsite_bucket.tf | 3 ++ 5 files changed, 25 insertions(+), 23 deletions(-) diff --git a/archive/terraform/ukmda/loggingbucket.tf b/archive/terraform/ukmda/loggingbucket.tf index 7938ec4a..adbffcf7 100644 --- a/archive/terraform/ukmda/loggingbucket.tf +++ b/archive/terraform/ukmda/loggingbucket.tf @@ -10,12 +10,14 @@ resource "aws_s3_bucket" "logbucket" { resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" { bucket = aws_s3_bucket.logbucket.id + transition_default_minimum_object_size = "all_storage_classes_128K" rule { status = "Enabled" id = "purge old versions" noncurrent_version_expiration { noncurrent_days = 30 } + filter {} } rule { id = "purge old ukmda-shared logs" @@ -23,7 +25,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" { expiration { days = 10 - expired_object_delete_marker = false } filter { @@ -40,7 +41,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" { expiration { days = 10 - expired_object_delete_marker = false } filter { @@ -57,7 +57,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" { expiration { days = 10 - expired_object_delete_marker = false } filter { @@ -106,12 +105,14 @@ resource "aws_s3_bucket" "logbucket_w1" { resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp_w1" { bucket = aws_s3_bucket.logbucket_w1.id provider = aws.eu-west-1-prov + transition_default_minimum_object_size = "varies_by_storage_class" rule { status = "Enabled" id = "purge old versions" noncurrent_version_expiration { noncurrent_days = 30 } + filter {} } rule { id = "purge old ukmda-live logs" @@ -119,7 +120,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp_w1" { expiration { days = 10 - expired_object_delete_marker = false } filter { diff --git a/archive/terraform/ukmda/ukmdaadmin_bucket.tf b/archive/terraform/ukmda/ukmdaadmin_bucket.tf index c588da60..a36f9a43 100644 --- a/archive/terraform/ukmda/ukmdaadmin_bucket.tf +++ b/archive/terraform/ukmda/ukmdaadmin_bucket.tf @@ -3,17 +3,18 @@ resource "aws_s3_bucket" "adminbucket" { force_destroy = false tags = { "billingtag" = "ukmda" + "ukmdatype" = "admin" } } resource "aws_s3_bucket_lifecycle_configuration" "adminbucketlcp" { bucket = aws_s3_bucket.adminbucket.id + transition_default_minimum_object_size = "varies_by_storage_class" rule { id = "purge old emails" status = "Enabled" expiration { days = 30 - expired_object_delete_marker = false } filter { prefix = "email/" diff --git a/archive/terraform/ukmda/ukmdalive_bucket.tf b/archive/terraform/ukmda/ukmdalive_bucket.tf index 2889ce61..7e1c405b 100644 --- a/archive/terraform/ukmda/ukmdalive_bucket.tf +++ b/archive/terraform/ukmda/ukmdalive_bucket.tf @@ -65,6 +65,7 @@ resource "aws_s3_bucket_policy" "ukmdalivebp" { resource "aws_s3_bucket_lifecycle_configuration" "ukmdalivelcp" { bucket = aws_s3_bucket.ukmdalive.id provider = aws.eu-west-1-prov + transition_default_minimum_object_size = "varies_by_storage_class" rule { status = "Enabled" id = "1 months delete" @@ -76,8 +77,8 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalivelcp" { } expiration { days = 30 - expired_object_delete_marker = false } + filter {} } } diff --git a/archive/terraform/ukmda/ukmdashared_bucket.tf b/archive/terraform/ukmda/ukmdashared_bucket.tf index 26a0c013..bad938fe 100644 --- a/archive/terraform/ukmda/ukmdashared_bucket.tf +++ b/archive/terraform/ukmda/ukmdashared_bucket.tf @@ -79,12 +79,14 @@ resource "aws_s3_bucket_acl" "ukmdasharedacl" { resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" { bucket = aws_s3_bucket.ukmdashared.id + transition_default_minimum_object_size = "all_storage_classes_128K" rule { status = "Enabled" id = "purge old versions" noncurrent_version_expiration { noncurrent_days = 30 } + filter {} } rule { status = "Enabled" @@ -98,25 +100,12 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" { storage_class = "STANDARD_IA" } } - rule { - status = "Enabled" - id = "Transition to Glacier" - filter { - prefix = "archive/" - } - - transition { - days = 180 - storage_class = "GLACIER_IR" - } - } rule { id = "purge athena queries" status = "Enabled" expiration { days = 2 - expired_object_delete_marker = false } filter { @@ -133,7 +122,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" { expiration { days = 30 - expired_object_delete_marker = false } filter { @@ -150,7 +138,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" { expiration { days = 60 - expired_object_delete_marker = false } filter { @@ -167,7 +154,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" { expiration { days = 30 - expired_object_delete_marker = false } filter { @@ -184,7 +170,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" { expiration { days = 30 - expired_object_delete_marker = false } filter { @@ -195,6 +180,18 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" { noncurrent_days = 10 } } + rule { + status = "Enabled" + id = "Transition to Glacier" + filter { + prefix = "archive/" + } + + transition { + days = 180 + storage_class = "GLACIER_IR" + } + } } diff --git a/archive/terraform/ukmda/ukmdawebsite_bucket.tf b/archive/terraform/ukmda/ukmdawebsite_bucket.tf index 17e9083b..e1d2de67 100644 --- a/archive/terraform/ukmda/ukmdawebsite_bucket.tf +++ b/archive/terraform/ukmda/ukmdawebsite_bucket.tf @@ -4,6 +4,7 @@ resource "aws_s3_bucket" "archsite" { force_destroy = false tags = { "billingtag" = "ukmda" + "ukmdatype" = "website" } } @@ -86,12 +87,14 @@ resource "aws_s3_bucket_logging" "ukmdawebsitelogs" { resource "aws_s3_bucket_lifecycle_configuration" "archsitelcp" { bucket = aws_s3_bucket.archsite.id + transition_default_minimum_object_size = "varies_by_storage_class" rule { status = "Enabled" id = "purge old versions" noncurrent_version_expiration { noncurrent_days = 30 } + filter {} } rule { status = "Enabled" From a89d60236cf23fc89964849a91798fec3ad2a72f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 30 Aug 2025 20:31:00 +0100 Subject: [PATCH 35/83] tidy up route tables --- archive/terraform/ukmda/vpc.tf | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/archive/terraform/ukmda/vpc.tf b/archive/terraform/ukmda/vpc.tf index 79ba622d..455e6c3a 100644 --- a/archive/terraform/ukmda/vpc.tf +++ b/archive/terraform/ukmda/vpc.tf @@ -73,11 +73,16 @@ resource "aws_default_route_table" "ec2_rtbl" { vpc_peering_connection_id = aws_vpc_peering_connection.ukmdatommpeering.id } tags = { - "Name" = "ec2_rtbl" + "Name" = "public_rtbl" billingtag = "Management" } } +resource "aws_route_table_association" "ec2_rtbl_assoc" { + subnet_id = aws_subnet.ec2_subnet.id + route_table_id = aws_default_route_table.ec2_rtbl.id +} + # internet gateway for this account resource "aws_internet_gateway" "main_igw" { vpc_id = aws_vpc.ec2_vpc.id From a9ed55bbf2bec40a342c725369dcbfde6eb327c3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 1 Sep 2025 13:15:24 +0100 Subject: [PATCH 36/83] function to check runtime of calcserver for cost analysis purposes --- archive/utils/checkCalcServerRunTime.sh | 66 +++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 archive/utils/checkCalcServerRunTime.sh diff --git a/archive/utils/checkCalcServerRunTime.sh b/archive/utils/checkCalcServerRunTime.sh new file mode 100644 index 00000000..8380d69f --- /dev/null +++ b/archive/utils/checkCalcServerRunTime.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# script to extract the start/stop times of the Calculation Engine server from the logs +# I have a spreadsheet that consumes this and works out the cost for various models of server (c8g.2xlarge, c8g.4xlarge etc) + +cd ~/prod/logs + +csl=/tmp/calcserver_timings.txt +csl2=/tmp/csl.txt +[ -f $csl ] && rm $csl +touch $csl + +ls -1tr nightly*.gz | while read i +do + mfz=matchJob${i:10:50} + gunzip $i + gunzip $mfz + nf=$(basename $i .gz) + mf=$(basename $mfz .gz) + egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl + gzip $nf + gzip $mf +done +ls -1tr nightlyJob* | grep -v gz | while read nf +do + mf=matchJob${nf:10:50} + egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl +done +[ -f $csl2 ] && rm $csl2 +touch $csl2 +cat $csl | while read i +do + parta=$(echo $i | awk -F ">" '{print $2}') + partb=$(echo $parta | awk -F " " '{printf("%s, %s, %s\n",$1,$2,$3)}') + echo $partb >> $csl2 +done +[ -f $csl ] && rm $csl +python << EOD +lis = open('$csl2','r').readlines() +data = [x.strip().split(',') for x in lis] +currdt = None +results = [] +res = [] +col = 0 +with open('/tmp/computeservertimes.csv','w') as outf: + for i in range(len(data)): + mth = data[i][0] + dy = data[i][1] + tim = data[i][2] + dtstr = f'{mth} {dy}' + if currdt is None: + currdt = dtstr + if currdt != dtstr: + outf.write(','.join(res) + '\n') + res = [] + col = 0 + if col == 0: + res.append(dtstr) + res.append(tim) + else: + res.append(tim) + col += 1 + currdt = dtstr + outf.write(','.join(res) + '\n') +EOD +[ -f $csl2 ] && rm $csl2 From bb61a0b3911d37147bbcc2885b2dc306263f834b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 2 Sep 2025 11:08:04 +0100 Subject: [PATCH 37/83] avoid generating unneeded files during manual upload of soln plus some syntax fixes --- archive/containers/trajsolver/trajsolver.py | 1 + .../maintenance/compressOldArchiveData.py | 8 ++++--- .../maintenance/recreateOrbitPages.py | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py index ff1ce8f2..2d42835f 100644 --- a/archive/containers/trajsolver/trajsolver.py +++ b/archive/containers/trajsolver/trajsolver.py @@ -170,6 +170,7 @@ def checkIfFileNeeded(filename): # update this if there's an additional file created by WMPL that we want # to use on the website + # NB THIS ALSO HAS TO BE CHANGED in maintenance.recreateOrbitPagesif more files needed if 'orbit_top.png' in filename or 'orbit_side.png' in filename or 'ground_track.png' in filename: return True if 'velocities.png' in filename or 'lengths.png' in filename or 'lags_all.png' in filename: diff --git a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py index bbed0b0f..ed04fced 100644 --- a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py +++ b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py @@ -79,6 +79,7 @@ def getAllS3Objects(s3, **base_kwargs): continuation_token = response.get('NextContinuationToken') """ + def deleteFiles(flist, srcbucket='ukmda-shared'): chunk_size = 900 chunked_list = [flist[i:i + chunk_size] for i in range(0, len(flist), chunk_size)] @@ -176,7 +177,7 @@ def moveJpgsAndMp4s(source_bucket, yr, ym): idx = f'{fldr}/index.html' rootdir = os.path.split(idx)[0] try: - s3.download_file(source_bucket, idx, f'/tmp/oldindex.html') + s3.download_file(source_bucket, idx, '/tmp/oldindex.html') except Exception: continue lis = open('/tmp/oldindex.html', 'r').readlines() @@ -202,13 +203,14 @@ def moveJpgsAndMp4s(source_bucket, yr, ym): s3.upload_file('/tmp/newindex.html', source_bucket, idx, ExtraArgs=extraargs) print('done') + def fixBrokenIndexes(source_bucket, yr, ym): bucket = s3res.Bucket(source_bucket) files = [os.key for os in bucket.objects.filter(Prefix=f'reports/{yr}/orbits/{ym}/')] idxs = [x for x in files if 'index.html' in x] for idx in idxs: print(idx) - s3.download_file(source_bucket, idx, f'/tmp/oldindex.html') + s3.download_file(source_bucket, idx, '/tmp/oldindex.html') lis = open('/tmp/oldindex.html', 'r').readlines() rootdir = os.path.split(idx)[0] # skip files that dont have the line in them @@ -252,7 +254,7 @@ def updateIndexes(idxs, source_bucket): for idx in idxs: if len(idx.split('/')) < 7: continue - s3.download_file(source_bucket, idx, f'/tmp/oldindex.html') + s3.download_file(source_bucket, idx, '/tmp/oldindex.html') lis = open('/tmp/oldindex.html', 'r').readlines() # skip files that dont have the line in them mtch = [li for li in lis if 'download a zip of the' in li] diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py index 4b2f8fdf..46b450f6 100644 --- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py +++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py @@ -136,6 +136,24 @@ def fixupTrajComments(traj, availableimages, outdir, picklename): return +def checkIfFileNeeded(filename): + + # update this if there's an additional file created by WMPL that we want + # to use on the website + # NB THIS ALSO HAS TO BE CHANGED in TRAJSOLVER + if 'orbit_top.png' in filename or 'orbit_side.png' in filename or 'ground_track.png' in filename: + return True + if 'velocities.png' in filename or 'lengths.png' in filename or 'lags_all.png' in filename: + return True + if 'abs_mag.png' in filename or 'abs_mag_ht.png' in filename or 'report.txt' in filename: + return True + if 'all_angular_residuals.png' in filename or 'all_spatial_total_residuals_height.png' in filename: + return True + if 'trajectory.pickle' in filename: + return True + return False + + def recreateOrbitFiles(outdir, pickname, doupload=False): traj = loadPickle(outdir, pickname) traj.save_results = True @@ -181,6 +199,9 @@ def recreateOrbitFiles(outdir, pickname, doupload=False): else: webfldr = f'reports/{yr}/orbits/{ym}' for fil in files: + if not checkIfFileNeeded(fil): + print(f'skipping {fil}') + continue locfname = f'{outdir}/{fil}' if 'summary' in fil or 'extrajpgs.txt' or 'html' in fil: keyname = f"{webfldr}/{orbfldr}/{fil}" From a311c02341751ccc22bbc05fee269706eefde8a3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 18 Sep 2025 11:15:33 +0100 Subject: [PATCH 38/83] add script to compress older data on S3 to save cost --- archive/deployment/pylib.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml index 162f4ab6..46a4afc7 100644 --- a/archive/deployment/pylib.yml +++ b/archive/deployment/pylib.yml @@ -37,6 +37,7 @@ - {src: '{{srcdir}}/ukmon_pylib/converters/__init__.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/converters/README.md', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no } + - {src: '{{srcdir}}/ukmon_pylib/maintenance/compressOldArchiveData.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/maintenance/cleanDdbTables.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/maintenance/dataMaintenance.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } - {src: '{{srcdir}}/ukmon_pylib/maintenance/deDuplicate.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no } From 5b0a5475dd2acd295cc97169e9f8d0ff2d3a77af Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 18 Sep 2025 11:15:45 +0100 Subject: [PATCH 39/83] remove zip file as no longer used --- archive/ukmon_pylib/maintenance/rerunFailedLambdas.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py index 186be127..feb2a507 100644 --- a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py +++ b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py @@ -83,17 +83,17 @@ def checkFails(fails): s3bucket=os.getenv('WEBSITEBUCKET', default='s3://ukmda-website')[5:] realfails = [] for f in fails: - # example f : matches/RMSCorrelate/trajectories/2022/202202/20220227/20220227_011240.522_UK + # example f : matches/RMSCorrelate/trajectories/2022/202202/20220227/20220227_011240.522_UK/20220227_011240_trajectory.pickle orb = f[f.find('/20')+1:] + orb = 'reports/' + orb[:5] + 'orbits' + orb[4:] pth, _ = os.path.split(orb) - _, zipname = os.path.split(pth) - keyf = pth + '/' + zipname + '.zip' + keyf = f'{pth}/index.html' try: _ = s3.head_object(Bucket=s3bucket, Key=keyf) #print(f'{pth} already done') except: - print(f'adding {keyf}') + print(f'adding {f}') realfails.append(f) return realfails From 848ed64f764812fcd82d2bee7cf795c72ed94cdd Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 18 Sep 2025 11:16:34 +0100 Subject: [PATCH 40/83] work around issue removing files from onedrive cloud --- fbCollector/fireballCollector.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 658dad4c..07aafd63 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -890,7 +890,8 @@ def getEventData(self): return def getGMNData(self): - camlist = [line for line in os.listdir(self.dir_path) if self.correct_datafile_name(line)] + print(self.dir_path) + camlist = [line for line in os.listdir(os.path.join(self.dir_path,'jpgs')) if self.correct_datafile_name(line)] dts=[] camids=[] for cam in camlist: @@ -939,7 +940,10 @@ def getGMNData(self): os.makedirs(targ, exist_ok=True) for f in os.listdir(srcdir): shutil.copy(os.path.join(srcdir, f), targ) - shutil.rmtree(os.path.join(self.dir_path, dtstr)) + try: + shutil.rmtree(os.path.join(self.dir_path, dtstr)) + except Exception: + pass tkMessageBox.showinfo("Data Collected", 'data collected from GMN') self.update_listbox(self.get_bin_list()) return From 7c4ad4598a9e2d8b581d54cf4eaf4b20b604306d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 18 Sep 2025 11:16:53 +0100 Subject: [PATCH 41/83] remove ubuntu version of calcserver for now, will revisit later --- archive/terraform/ukmda/ec2.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/ec2.tf index 97b2a567..8c1ed70c 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/ec2.tf @@ -44,7 +44,7 @@ resource "aws_network_interface" "calcserver_if" { ################################################ # Ubuntu calc server ################################################ - +/* resource "aws_instance" "ubuntu_calc_server" { ami = "ami-0bdf149a42243bde8" instance_type = "c6g.4xlarge" @@ -84,7 +84,7 @@ resource "aws_network_interface" "ubuntu_calcserver_if" { "billingtag" = "ukmda" } } - +*/ ################################################ # admin server ################################################ From 10117c7c489bc065cc1a63c4f230e916182d2f04 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 18 Sep 2025 11:17:28 +0100 Subject: [PATCH 42/83] comment to help future maintainers --- archive/ukmon_pylib/maintenance/recreateOrbitPages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py index 46b450f6..61b5c5bd 100644 --- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py +++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py @@ -140,7 +140,7 @@ def checkIfFileNeeded(filename): # update this if there's an additional file created by WMPL that we want # to use on the website - # NB THIS ALSO HAS TO BE CHANGED in TRAJSOLVER + # NB THIS ALSO HAS TO BE CHANGED in TRAJSOLVER if more files needed if 'orbit_top.png' in filename or 'orbit_side.png' in filename or 'ground_track.png' in filename: return True if 'velocities.png' in filename or 'lengths.png' in filename or 'lags_all.png' in filename: From b2fb0da270c1bd6751a8089d2620f0e918fe8522 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 24 Sep 2025 22:45:50 +0100 Subject: [PATCH 43/83] make more generic --- archive/ukmon_pylib/converters/gmnTxtToPandas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/converters/gmnTxtToPandas.py b/archive/ukmon_pylib/converters/gmnTxtToPandas.py index 958d64b8..ad98886f 100644 --- a/archive/ukmon_pylib/converters/gmnTxtToPandas.py +++ b/archive/ukmon_pylib/converters/gmnTxtToPandas.py @@ -17,7 +17,8 @@ 'Lat1','Lat1sd','Lon1','Lon1sd','H1','H1sd','Lat2','Lat2sd','Lon2','Lon2sd','H2','H2sd', 'Dur','Amag','PkHt','F1','mass','Qc','MedianFitErr','BegIn','EndIn','NumStat','stats'] -dirpath='F:/videos/MeteorCam/gmndata' +datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data') +dirpath = os.path.join(datadir, 'gmndata') def loadOneFile(fname): From 3e4fb750ff4f83fcfb03ad52aa1a23d1a99c48d3 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 24 Sep 2025 22:46:06 +0100 Subject: [PATCH 44/83] bugfix for when there are not many stale keys --- archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py index 34115d7a..f1b47728 100644 --- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py +++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py @@ -347,7 +347,10 @@ def rollKeysForUsers(nusers=3): _, stalekeys, _, _ = getKeyStatuses(excludeadm=True) if len(stalekeys) == 0: return - sampledf = stalekeys.sample(n=nusers) + if len(stalekeys) > nusers: + sampledf = stalekeys.sample(n=nusers) + else: + sampledf = stalekeys for _, rw in sampledf.iterrows(): uid = rw.user if 'testtraj' in uid: From 72e32a2c76c0abee5c3dc3dbbee3ef536a34b6fc Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 24 Sep 2025 22:46:19 +0100 Subject: [PATCH 45/83] add function to pull GMN data --- archive/utils/getGmnData.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 archive/utils/getGmnData.sh diff --git a/archive/utils/getGmnData.sh b/archive/utils/getGmnData.sh new file mode 100644 index 00000000..bfbaf939 --- /dev/null +++ b/archive/utils/getGmnData.sh @@ -0,0 +1,19 @@ +#!/bin/bash +here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $here/../config.ini >/dev/null 2>&1 +conda activate $HOME/miniconda3/envs/${WMPL_ENV} + +cd $DATADIR/gmndata + +y1=$(date +%Y) +y2=$(date +%Y --date 'yesterday') +rsync -vtz gmn.uwo.ca:/srv/meteor-ro/rms/gmn/extracted_data/auto_output/traj_summary_data/*$y1.txt . --exclude traj_summary_all.txt +if [ "$ym1" != "$ym2" ] ; then + rsync -vtz gmn.uwo.ca:/srv/meteor-ro/rms/gmn/extracted_data/auto_output/traj_summary_data/*$y2.txt . --exclude traj_summary_all.txt +fi +ym1=$(date +%Y%m) +ym2=$(date +%Y%m --date 'yesterday') +rsync -vtz gmn.uwo.ca:/srv/meteor-ro/rms/gmn/extracted_data/auto_output/traj_summary_data/monthly/*${ym1}.txt ./monthly --exclude traj_summary_all.txt +if [ "$ym1" != "$ym2" ] ; then + rsync -vtz gmn.uwo.ca:/srv/meteor-ro/rms/gmn/extracted_data/auto_output/traj_summary_data/monthly/*${ym2}.txt ./monthly --exclude traj_summary_all.txt +fi \ No newline at end of file From 9eebf419589b499715d6220976bc130678be5ca7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Wed, 24 Sep 2025 22:46:35 +0100 Subject: [PATCH 46/83] change threshold for diskspace alerting on helper --- archive/utils/statsToMqtt.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh index 50b58a69..3235281a 100644 --- a/archive/utils/statsToMqtt.sh +++ b/archive/utils/statsToMqtt.sh @@ -5,7 +5,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then source /home/bitnami/venvs/openhabstuff/bin/activate python $here/statsToMqtt.py diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') - if [ $diskpct -gt 90 ] ; then + if [ $diskpct -gt 9 ] ; then /home/bitnami/src/maintenance/mysql_checks.sh msg="$(hostname) diskspace was ${diskpct}%" subj="Diskspace alert for $(hostname)" @@ -27,7 +27,7 @@ else export PYTHONPATH=$SRC/ukmon_pylib:$PYTHONPATH python -m metrics.statsToMqtt diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g') - if [ $diskpct -gt 90 ] ; then + if [ $diskpct -gt 95 ] ; then msg="$(hostname) diskspace is ${diskpct}%" subj="Diskspace alert for $(hostname)" hn="$(hostname)@aws" From e9ae955c2e77850d538fbb5a61674b79b4697060 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 29 Sep 2025 13:53:27 +0100 Subject: [PATCH 47/83] fix a couple of bugs in geExtraOrbitFiles --- .../getExtraFilesV2/pythoncode/createOrbitPageIndex.py | 2 +- .../getExtraFilesV2/pythoncode/getExtraFiles.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py b/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py index 8d0cd2cb..ab4e2418 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py +++ b/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py @@ -34,7 +34,7 @@ def createOrbitPageIndex(fldr, websitebucket, s3): #zipf = orbitname + '.zip' #if os.path.isfile(os.path.join(fldr, zipf)): # idxf.write(f"Click here to download a zip of the raw and processed data.\n") - #idxf.write("\n") + idxf.write("\n") idxf.write("

Detailed report below graphs

\n") idxf.write("

Click on an image to see a larger view

\n") diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py index 24174ccc..6f08afed 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py +++ b/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py @@ -85,7 +85,7 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): if 'FF_' not in ffname and 'FR_' not in ffname: ffname = 'FF_' + ffname ffname = ffname.replace('FR_', 'FF_').replace('.bin', '.fits') - if '.bin' not in ffname and '.fits' not in ffname: + if '.bin' not in ffname and '.fits' not in ffname and '.jpg' not in ffname: ffname = ffname + '.fits' gotff = True except Exception: @@ -101,9 +101,10 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): id = spls[1] dtstr = spls[2] tmstr = spls[3] - jpgname=f'img/single/{dtstr[:4]}/{dtstr[:6]}/{ffname}'.replace('fits','jpg') - mp4name=f'img/mp4/{dtstr[:4]}/{dtstr[:6]}/{ffname}'.replace('fits','mp4') + jpgname=f'img/single/{dtstr[:4]}/{dtstr[:6]}/{ffname}'.replace('.fits','.jpg') + mp4name=f'img/mp4/{dtstr[:4]}/{dtstr[:6]}/{ffname}'.replace('.fits','.mp4').replace('.jpg','.mp4') print('about to get a list of available jpgs') + print(jpgname) res = s3.meta.client.list_objects_v2(Bucket=websitebucket,Prefix=jpgname) if res['KeyCount'] > 0: jpgf.write(f'{jpgname}\n') From 421904cb3415627e5af91ae05c9ddcfef39e7f45 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 3 Oct 2025 12:08:18 +0100 Subject: [PATCH 48/83] update location of GMN status report --- archive/website/templates/frontpage.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/website/templates/frontpage.html b/archive/website/templates/frontpage.html index 4ff72d8d..c1af5067 100644 --- a/archive/website/templates/frontpage.html +++ b/archive/website/templates/frontpage.html @@ -63,7 +63,7 @@

Coverage

We currently have #NUMCAMS# cameras in the network, the locations of which are shown at the foot of this page. Camera Coverage can be seen here. Camera status can be quickly checked here or - in more detail on the GMN site here.

+ in more detail on the GMN site here.

Summary Data

The graph below shows activity year to date and the table summarises our entire dataset. You can also see an excellent map of all detections here, From 8b0be667e6a8b6c7a47194593a203f48461da05e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 3 Oct 2025 12:09:11 +0100 Subject: [PATCH 49/83] Add more documentation --- archive/website/createSummaryTable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archive/website/createSummaryTable.sh b/archive/website/createSummaryTable.sh index 33577ded..fab46348 100644 --- a/archive/website/createSummaryTable.sh +++ b/archive/website/createSummaryTable.sh @@ -1,6 +1,6 @@ #!/bin/bash # Copyright (C) 2018-2023 Mark McIntyre -# Creates the table and bar chart that appear on the website homepage. +# Create the archive website homepage, and the table and bar chart that appear on the page. # Also creates the coverage map, and copies the logfile to the site. # # Parameters From a8e7a514cb74fac77674e918e77820302cb8da9f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 4 Oct 2025 13:00:59 +0100 Subject: [PATCH 50/83] bugfix to correct target path for downloads --- fbCollector/download_events.sh | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/fbCollector/download_events.sh b/fbCollector/download_events.sh index 7c47e3f4..d75b9a43 100644 --- a/fbCollector/download_events.sh +++ b/fbCollector/download_events.sh @@ -4,25 +4,25 @@ here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" dt=$1 basedir=$(grep basedir $here/config.ini | awk -F= '{print $2}'|tr -d '\r' | sed 's|f:|/mnt/f|g' | sed 's|c:|/mnt/c|g') -echo $basedir +echo $basedir $dt sleep 30 dtns=$dt mkdir -p $basedir/$dtns -cd $basedir/$dtns +pushd $basedir/$dtns -rsync -avz gmn.uwo.ca:/home/uk*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns -rsync -avz gmn.uwo.ca:/home/be*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns -rsync -avz gmn.uwo.ca:/home/ie*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns -rsync -avz gmn.uwo.ca:/home/nl*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns +rsync -avz gmn.uwo.ca:/home/uk*/files/event_monitor/*${dt}*.bz2 . +rsync -avz gmn.uwo.ca:/home/be*/files/event_monitor/*${dt}*.bz2 . +rsync -avz gmn.uwo.ca:/home/ie*/files/event_monitor/*${dt}*.bz2 . +rsync -avz gmn.uwo.ca:/home/nl*/files/event_monitor/*${dt}*.bz2 . if [ "$2" == "all" ] ; then - rsync -avz gmn.uwo.ca:/home/fr*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns - rsync -avz gmn.uwo.ca:/home/de*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns - rsync -avz gmn.uwo.ca:/home/es*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns - rsync -avz gmn.uwo.ca:/home/ch*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns - rsync -avz gmn.uwo.ca:/home/it*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns - rsync -avz gmn.uwo.ca:/home/cz*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns - rsync -avz gmn.uwo.ca:/home/hr*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns - rsync -avz gmn.uwo.ca:/home/sk*/files/event_monitor/*${dt}*.bz2 $basedir/$dtns + rsync -avz gmn.uwo.ca:/home/fr*/files/event_monitor/*${dt}*.bz2 . + rsync -avz gmn.uwo.ca:/home/de*/files/event_monitor/*${dt}*.bz2 . + rsync -avz gmn.uwo.ca:/home/es*/files/event_monitor/*${dt}*.bz2 . + rsync -avz gmn.uwo.ca:/home/ch*/files/event_monitor/*${dt}*.bz2 . + rsync -avz gmn.uwo.ca:/home/it*/files/event_monitor/*${dt}*.bz2 . + rsync -avz gmn.uwo.ca:/home/cz*/files/event_monitor/*${dt}*.bz2 . + rsync -avz gmn.uwo.ca:/home/hr*/files/event_monitor/*${dt}*.bz2 . + rsync -avz gmn.uwo.ca:/home/sk*/files/event_monitor/*${dt}*.bz2 . fi for f in *.bz2 ; do tar -xvf $f ; done @@ -65,3 +65,4 @@ mkdir -p ./mp4s cp -f */*.jpg ./jpgs mv -f ./jpgs/*captured_stack* ./stacks cp -f */*.mp4 ./mp4s +popd \ No newline at end of file From 50afdfea3e7e1137a9671d2460aa86dee1da992a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 4 Oct 2025 13:06:06 +0100 Subject: [PATCH 51/83] remove config.ini and replace with sample --- fbCollector/config.ini | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 fbCollector/config.ini diff --git a/fbCollector/config.ini b/fbCollector/config.ini deleted file mode 100644 index 35305ab7..00000000 --- a/fbCollector/config.ini +++ /dev/null @@ -1,30 +0,0 @@ -# tailor this config file to your needs -# $HOME will be translated into your homedir (~ on linux, $env:userprofile on windows) -[Fireballs] -basedir=$HOME/OneDrive/pictures/meteors/fireballs - -livebucket=ukmda-live -uploadbucket=ukmda-shared -uploadfolder=fireballs/uploads -ukmonprofile=ukmonshared - -[gmnconnection] -# the GMN key needs to be configured with Denis Vida -# if you have permission you can then collect data from the GMN server -gmnkey=~/.ssh/gmnanalysis -gmnserver=gmn.uwo.ca -gmnuser=analysis - -[reduction] -# To use the reduction options RMS must be installed and an RMS python conda environment created -rms_loc=$HOME/OneDrive/dev/RMS -rms_env=RMS - -[solver] -# To use the solver options WMPL must be installed and a WMPL python conda environment created -wmpl_loc=$HOME/OneDrive/dev/WesternMeteorPyLib -wmpl_env=ukmon-shared - -[sharing] -# create this folder if you want to share raw data with other UKMON members -shrfldr=$HOME/Dropbox/Raw_Data \ No newline at end of file From 4db79c6370830b6b0500d5cab454fd85d98761ad Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sat, 4 Oct 2025 13:07:01 +0100 Subject: [PATCH 52/83] add sample ini file --- .gitignore | 1 + fbCollector/config.ini.sample | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 fbCollector/config.ini.sample diff --git a/.gitignore b/.gitignore index e46843f5..3c3e770e 100644 --- a/.gitignore +++ b/.gitignore @@ -614,3 +614,4 @@ tests/testing/RMS/ tests/testing/WesternMeteorPyLib/ archive/ukmon_pylib/tests/testdata.tar.gz servercopybkp/* +fbcollector/config.ini \ No newline at end of file diff --git a/fbCollector/config.ini.sample b/fbCollector/config.ini.sample new file mode 100644 index 00000000..700a730c --- /dev/null +++ b/fbCollector/config.ini.sample @@ -0,0 +1,35 @@ +# tailor this config file to your needs +# Note that $HOME will be translated into your homedir (~ on linux, $env:userprofile on windows) +[Fireballs] +# the location to which fireball data will be downloaded eg c:\fireballs. +basedir= + +# You must have a UKMON AWS key/secret pair with write permission to the upload folder +# Add these to your ~/.aws/credentials file and put the name of the profile here +ukmonprofile= + +# don't change these +livebucket=ukmda-live +uploadbucket=ukmda-shared +uploadfolder=fireballs/uploads + +[gmnconnection] +# the GMN key needs to be configured with Denis Vida +# if you have permission you can then collect data from the GMN server +gmnkey=~/.ssh/gmnanalysis +gmnserver=gmn.uwo.ca +gmnuser=analysis + +[reduction] +# To use the reduction options RMS must be installed and an RMS python conda environment created +rms_loc= +rms_env=RMS + +[solver] +# To use the solver options WMPL must be installed and a WMPL python conda environment created +wmpl_loc= +wmpl_env=wmpl_env + +[sharing] +# create this folder if you want to share raw data with other UKMON members +shrfldr= \ No newline at end of file From 6d276c48864559c960019a6ed6b180e21355c3e9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Oct 2025 16:31:42 +0100 Subject: [PATCH 53/83] A lot of tidying up Add documentation Improve config reading --- fbCollector/README.md | 47 +++++++++ fbCollector/config.ini.sample | 35 +++---- fbCollector/fbCollector.ps1 | 13 +-- fbCollector/fbcollector.yml | 14 +-- fbCollector/fireballCollector.py | 174 +++++++++++++++++-------------- fbCollector/helperfunctions.ps1 | 107 ------------------- fbCollector/requirements.txt | 2 + fbCollector/vars.yml | 4 +- 8 files changed, 179 insertions(+), 217 deletions(-) create mode 100644 fbCollector/README.md delete mode 100644 fbCollector/helperfunctions.ps1 diff --git a/fbCollector/README.md b/fbCollector/README.md new file mode 100644 index 00000000..66d21a5f --- /dev/null +++ b/fbCollector/README.md @@ -0,0 +1,47 @@ +# A Fireball Collector and Analyser for UKMON and GMN + +This tool allows authorised users to collect fireball data from UKMON and GMN and then to reduce and solve it. + +## Prerequisites + +* [WMPL](https://github.com/wmpg/WesternMeteorPyLib/), used to solve trajectories. +* [RMS](https://github.com/CroatianMeteorNetwork/RMS), used to reduce raw data. +* A local folder where you will store fireball data. This is called `basedir` in this documentation. + +The above are sufficient to collect data from UKMON and to analyse data either collected from UKMON or by other means. There are additional configuration options that can be used to collect data from GMN and upload solutions to UKMON. These are described inline below. + +## Installation +* Clone this repository to a location of your choice +* Install WMPL and RMS, and activate the WMPL python virtual environment. +* Change directory into the location of this code then install the additional requiremnts with `pip install -r requirements.txt` +* Copy `config.ini.sample` to `config.ini` and update the values of `basedir`, `rms_loc`, `rms_env`, `wmpl_loc`, and `wmpl_env` as appropriate. + +## Using the App +* Launch the app by running `fbCollector.ps1` in a Powershell window. After a few seconds the GUI window should appear. +* In the box labelled `Image Selection` enter a date and time in the format `YYYYMMDD_HHMMSS`, then click "Get Images". +* After a few seconds, the listbox below should be populated with images from around the time you selected, provided the UKMON live feed captured something. + * If nothing appears, check the UKMON [livestream](https://archive.ukmeteors.co.uk/live/index.html) to make sure you chose the correct time. + +* Go through the image list and click `Remove` to delete any that are not of the event you are interested in. +* Once you've whittled the list down to just the interesting events, you can select `Get ECSVs` from the `Raw` menu. This will attempt to get raw data for each image. You can also click `Get Videos` to collect any video data thats available. +* Now you can click `Solve` from the `Solve` menu. This will invoke WMPL and will attempt to find a trajectory that matches the raw data. It may take some time. +* If the solver is successful, you can view the solution by selecting `View Solution` from the `Solve` menu. The left-hand list will now show the output of the solver so you can examine it. You can switch back to viewing the raw images by selecting `Review Images` from the `Review` menu. +* If the solve process fails or if the solution seems very poor then try excluding some detections. To do this, select `Excl/Incl ECSV` from the `Raw` menu then rerun the Solver. +* If the solution was bad its also worth deleting it before attempting a rerun via `Delete Solution` from the `Solve` menu. + +## Manually Reducing an Image +If you've installed RMS then if you have a FITS or FR file from RMS along with the camera's config and platepar files, you can run RMS's `SkyFit2` tool to analyse the image - select and image then select `Reduce Selected Image` from the `Raw` menu. + +## Sending Solutions to UKMON +Once you have a solution, select `Upload orbit` from the `Solve` menu. This will create a Zip file that bundles the created trajectory pickle file with any images and videos. You can then upload the file to Dropbox, Google Drive or another file-sharing site and email a link to fireballdata@ukmeteornetwork.org where one of our team will validate and upload it to our Archive. + +#### API Key +Members of the UKMON team who frequently create solutions can request an API key to upload solutions less than 10 MB in size. We'll contact you if we think this is applicable. + +### Collecting data from GMN +Members of the GMN coordinators group who have permission from Denis Vida can use this tool to collect data directly from GMN or using the GMN Watchlist. If you fall into this category you can fill in the `[gmnanalysis]` section of the config file with the name of your SSH private key and other details. This will activate additional menu options to `Get GMN Raw Data` and to use the `Watchlist`. + +## Logs +The programme creates logs in your system's `TEMP` folder. + + diff --git a/fbCollector/config.ini.sample b/fbCollector/config.ini.sample index 700a730c..4b26b0d3 100644 --- a/fbCollector/config.ini.sample +++ b/fbCollector/config.ini.sample @@ -1,25 +1,10 @@ # tailor this config file to your needs # Note that $HOME will be translated into your homedir (~ on linux, $env:userprofile on windows) + [Fireballs] # the location to which fireball data will be downloaded eg c:\fireballs. basedir= -# You must have a UKMON AWS key/secret pair with write permission to the upload folder -# Add these to your ~/.aws/credentials file and put the name of the profile here -ukmonprofile= - -# don't change these -livebucket=ukmda-live -uploadbucket=ukmda-shared -uploadfolder=fireballs/uploads - -[gmnconnection] -# the GMN key needs to be configured with Denis Vida -# if you have permission you can then collect data from the GMN server -gmnkey=~/.ssh/gmnanalysis -gmnserver=gmn.uwo.ca -gmnuser=analysis - [reduction] # To use the reduction options RMS must be installed and an RMS python conda environment created rms_loc= @@ -28,8 +13,20 @@ rms_env=RMS [solver] # To use the solver options WMPL must be installed and a WMPL python conda environment created wmpl_loc= -wmpl_env=wmpl_env +wmpl_env=wmpl [sharing] -# create this folder if you want to share raw data with other UKMON members -shrfldr= \ No newline at end of file +# create a folder in Dropbox or similar if you want to share raw data with others and then put +# its *local* path here eg c:\users\yourname\dropbox\meteors +shrfldr= + +[gmn] +# the GMN key needs to be configured with Denis Vida +# if you have permission you can then collect data from the GMN server +gmnkey=~/.ssh/gmnanalysis +gmnserver=gmn.uwo.ca +gmnuser=analysis + +[ukmon] +apikey= + diff --git a/fbCollector/fbCollector.ps1 b/fbCollector/fbCollector.ps1 index 9854e1e8..29c0a786 100644 --- a/fbCollector/fbCollector.ps1 +++ b/fbCollector/fbCollector.ps1 @@ -4,14 +4,15 @@ # push-location $PSScriptRoot -. .\helperfunctions.ps1 -$ini=get-inicontent .\config.ini +$wmpl_env=((select-string .\config.ini -pattern "wmpl_env" -list).line).split('=')[1] +$wmpl_loc=((select-string .\config.ini -pattern "wmpl_loc" -list).line).split('=')[1] -conda activate $ini['solver']['wmpl_env'] -$wmplloc = $ini['solver']['wmpl_loc'] -$wmplloc = $wmplloc.replace('$HOME',$env:userprofile) +conda activate $wmpl_env -$env:pythonpath="$wmplloc" +$wmpl_loc = $wmpl_loc.replace('$HOME',$env:userprofile) +$wmpl_loc = $wmpl_loc.replace('\','/') + +$env:pythonpath="$wmpl_loc" if ($args.count -lt 1) { python fireballCollector.py diff --git a/fbCollector/fbcollector.yml b/fbCollector/fbcollector.yml index 9d18f218..83463e68 100644 --- a/fbCollector/fbcollector.yml +++ b/fbCollector/fbcollector.yml @@ -14,9 +14,11 @@ tags: [dev,prod] delegate_to: localhost with_items: - - {src: '{{srcdir}}/fbCollector.ps1', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/fireballCollector.py', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/download_events.sh', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/config.ini', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/noimage.jpg', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } - - {src: '{{srcdir}}/ukmda.ico', dest: '{{destdir}}/fbCollector/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/fbCollector.ps1', dest: '{{destdir}}/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/fireballCollector.py', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } + - {src: '{{srcdir}}/download_events.sh', dest: '{{destdir}}/', mode: '755', backup: no, directory_mode: no } + - {src: '{{srcdir}}/config.ini.sample', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } + - {src: '{{srcdir}}/noimage.jpg', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } + - {src: '{{srcdir}}/ukmda.ico', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } + - {src: '{{srcdir}}/requirements.txt', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no } + diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 07aafd63..ed9ad988 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -17,11 +17,10 @@ from PIL import Image import requests -import boto3 import paramiko from scp import SCPClient -from meteortools.ukmondb import getECSVs as getecsv +from meteortools.ukmondb import getECSVs, getLiveJpgs from wmpl.Formats.ECSV import loadECSVs from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric @@ -35,8 +34,6 @@ from PIL import Image as img from PIL import ImageTk -from meteortools.ukmondb import getLiveJpgs, createTxtFile - config_file = '' noimg_file = '' @@ -125,9 +122,6 @@ def __init__(self, parent, patt=None): self.parent = parent self.fb_dir = '' - self.upload_bucket = '' - self.upload_folder = '' - self.live_bucket = '' self.gmn_key = '' self.gmn_user = '' self.gmn_server = '' @@ -135,6 +129,8 @@ def __init__(self, parent, patt=None): self.wmpl_env = '' self.rms_loc = '' self.rms_env = '' + self.api_key = None + self.share_loc = '' self.selected = {} self.evtMonTriggered = None self.review_stack = False @@ -167,24 +163,44 @@ def readConfig(self): localcfg = configparser.ConfigParser() localcfg.read(config_file) self.fb_dir = os.path.expanduser(localcfg['Fireballs']['basedir'].replace('$HOME','~')).replace('\\','/') - self.upload_bucket = localcfg['Fireballs']['uploadbucket'] - self.upload_folder = localcfg['Fireballs']['uploadfolder'] - self.live_bucket = localcfg['Fireballs']['livebucket'] os.makedirs(self.fb_dir, exist_ok=True) try: - self.gmn_key = localcfg['gmnconnection']['gmnkey'] - self.gmn_user = localcfg['gmnconnection']['gmnuser'] - self.gmn_server = localcfg['gmnconnection']['gmnserver'] + self.gmn_key = localcfg['gmn']['gmnkey'] + self.gmn_user = localcfg['gmn']['gmnuser'] + self.gmn_server = localcfg['gmn']['gmnserver'] except: - pass + self.gmn_key = None - self.wmpl_loc = os.path.expanduser(localcfg['solver']['wmpl_loc'].replace('$HOME','~')).replace('\\','/') - self.wmpl_env= localcfg['solver']['wmpl_env'] - self.rms_loc = os.path.expanduser(localcfg['reduction']['rms_loc'].replace('$HOME','~')).replace('\\','/') - self.rms_env = localcfg['reduction']['rms_env'] + try: + self.api_key = localcfg['ukmon']['apikey'] + if os.path.isfile(os.path.expanduser(self.api_key)): + key = open(os.path.expanduser(self.api_key), 'r').readlines()[0].strip() + else: + key = self.api_key + if len(key) < 40: + key = None + self.api_key = key + except: + self.api_key = None + # log.info(f'apikey "{self.api_key}"') - self.shareloc = os.path.expanduser(localcfg['sharing']['shrfldr'].replace('$HOME','~')).replace('\\','/') + try: + self.wmpl_loc = os.path.expanduser(localcfg['solver']['wmpl_loc'].replace('$HOME','~')).replace('\\','/') + self.wmpl_env= localcfg['solver']['wmpl_env'] + except: + log.error('WMPL location and environment name must be configured') + quitApp() + try: + self.rms_loc = os.path.expanduser(localcfg['reduction']['rms_loc'].replace('$HOME','~')).replace('\\','/') + self.rms_env = localcfg['reduction']['rms_env'] + except: + self.rms_loc = None + + try: + self.share_loc = os.path.expanduser(localcfg['sharing']['shrfldr'].replace('$HOME','~')).replace('\\','/') + except: + self.share_loc = None return def quitApplication(self): @@ -260,40 +276,48 @@ def initUI(self): rawMenu = Menu(self.menuBar, tearoff=0) rawMenu.add_command(label="Get Live Images", command=self.get_data) rawMenu.add_command(label="Get Videos", command=self.get_vids) - rawMenu.add_command(label="Get Traj Pickle", command=self.get_trajpickle) rawMenu.add_separator() - rawMenu.add_command(label="Get GMN Raw Data", command=self.getGMNData) - rawMenu.add_separator() - rawMenu.add_command(label="Get ECSVs", command=self.getECSVs) - self.menuBar.add_cascade(label="Raw", underline=0, menu=rawMenu) + rawMenu.add_command(label="Get ECSVs", command=self.getRequestedECSVs) + rawMenu.add_command(label="Excl/Incl ECSV", command=self.ignoreCamera) + if self.rms_loc: + rawMenu.add_command(label="Reduce Selected Image", command=self.reduceCamera) + rawMenu.add_separator() + if self.share_loc: + rawMenu.add_command(label="Share Raw Data", command=self.uploadRaw) + if self.gmn_key: + rawMenu.add_separator() + watchMenu = Menu(self.menuBar, tearoff=0) + watchMenu.add_command(label="Get GMN Raw Data", command=self.getGMNData) + watchMenu.add_separator() + watchMenu.add_command(label="Get Watchlist", command=self.getWatchlist) + watchMenu.add_command(label="View Watchlist", command=self.viewWatchlist) + watchMenu.add_command(label="Upload Watchlist", command=self.putWatchlist) + watchMenu.add_separator() + watchMenu.add_command(label="Fetch Watchlist Event", command=self.getEventData) + rawMenu.add_cascade(label='GMN', menu=watchMenu) + # self.menuBar.add_cascade(label="Watchlist", underline=0, menu=watchMenu) - watchMenu = Menu(self.menuBar, tearoff=0) - watchMenu.add_command(label="Get Watchlist", command=self.getWatchlist) - watchMenu.add_command(label="View Watchlist", command=self.viewWatchlist) - watchMenu.add_command(label="Upload Watchlist", command=self.putWatchlist) - watchMenu.add_separator() - watchMenu.add_command(label="Fetch Event Data", command=self.getEventData) - self.menuBar.add_cascade(label="Watchlist", underline=0, menu=watchMenu) + self.menuBar.add_cascade(label="Raw", underline=0, menu=rawMenu) revMenu = Menu(self.menuBar, tearoff=0) revMenu.add_command(label="Review Stacks", command=self.checkStacks) + revMenu.add_command(label="Review Images", command=self.viewData) revMenu.add_command(label="Clean Folder", command=self.clean_folder) self.menuBar.add_cascade(label="Review", underline=0, menu=revMenu) solveMenu = Menu(self.menuBar, tearoff=0) - solveMenu.add_command(label="Reduce Data", command=self.reduceCamera) - solveMenu.add_command(label="Toggle Ignore", command=self.ignoreCamera) - solveMenu.add_separator() - solveMenu.add_command(label="View Raw Data", command=self.viewData) - solveMenu.add_command(label="Upload Raw Data", command=self.uploadRaw) - solveMenu.add_separator() solveMenu.add_command(label="Solve", command=self.solveOrbit) solveMenu.add_separator() solveMenu.add_command(label="View Solution", command=self.viewSolution) solveMenu.add_command(label="Delete Solution", command=self.removeSolution) solveMenu.add_separator() - solveMenu.add_command(label="Upload Orbit", command=self.uploadOrbit) + solveMenu.add_command(label="Upload Solution", command=self.uploadOrbit) self.menuBar.add_cascade(label="Solve", underline=0, menu=solveMenu) + + otherMenu = Menu(self.menuBar, tearoff=0) + otherMenu.add_command(label="Get Traj Pickle", command=self.get_trajpickle) + self.menuBar.add_cascade(label="Other", underline=0, menu=otherMenu) + # buttons self.save_panel = LabelFrame(self, text=' Image Selection ') self.save_panel.grid(row = 1, columnspan = 2, sticky='WE') @@ -375,14 +399,18 @@ def ignoreCamera(self): if len(ecsvs) == 0 and len(rejs) == 0: log.info('no files to reject or include') return - elif len(ecsvs) == 1 and len(rejs) == 0: - os.rename(ecsvs[0], ecsvs[0].replace('.ecsv','_REJECT.ecsv')) - jpgname = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) - os.rename(jpgname[0], jpgname[0].replace('.jpg','_REJECT.jpg')) - elif len(ecsvs) == 0 and len(rejs) == 1: - os.rename(rejs[0], rejs[0].replace('_REJECT.ecsv','.ecsv')) - jpgname = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) - os.rename(jpgname[0], jpgname[0].replace('_REJECT.jpg','.jpg')) + elif len(ecsvs) > 0 and len(rejs) == 0: + for ecsv in ecsvs: + os.rename(ecsv, ecsv.replace('.ecsv','_REJECT.ecsv')) + jpgnames = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) + for jpgname in jpgnames: + os.rename(jpgname, jpgname.replace('.jpg','_REJECT.jpg')) + elif len(ecsvs) == 0 and len(rejs) > 0: + for rej in rejs: + os.rename(rej, rej.replace('_REJECT.ecsv','.ecsv')) + jpgnames = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image)) + for jpgname in jpgnames: + os.rename(jpgname, jpgname.replace('_REJECT.jpg','.jpg')) else: # more than one ECSV or REJ file to handle, urk log.info('urk') @@ -490,38 +518,45 @@ def uploadOrbit(self): shutil.rmtree(tmpdir) except Exception: pass - apikey = open(os.path.expanduser('~/.ssh/fbuploadkey.txt')).readlines()[0].strip() - headers = {'Content-type': 'application/zip', 'Slug': orbname[:15], 'apikey': apikey} - url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbname[:15]}.zip' - r = requests.put(url, data=open(zfname+'.zip', 'rb'), headers=headers) #, auth=('username', 'pass')) - #print(r.text) - if r.status_code != 200: - tkMessageBox.showinfo('Warning', 'Problem with upload') + + if self.api_key: + headers = {'Content-type': 'application/zip', 'Slug': orbname[:15], 'apikey': self.api_key} + url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbname[:15]}.zip' + r = requests.put(url, data=open(zfname+'.zip', 'rb'), headers=headers) #, auth=('username', 'pass')) + #print(r.text) + if r.status_code != 200: + tkMessageBox.showinfo('Warning', f'Problem with upload, {r.status_code}') + else: + tkMessageBox.showinfo('Info', 'Orbit Uploaded') + return else: - tkMessageBox.showinfo('Info', 'Orbit Uploaded') - return + tkMessageBox.showinfo('Info', 'Zip File created') def uploadRaw(self): zfname = os.path.join(os.getenv('TMP'), os.path.basename(self.dir_path)) log.info(f'zfname is {zfname}') shutil.make_archive(zfname,'zip',self.dir_path) try: - targname = os.path.join(self.shareloc, os.path.basename(zfname)+'.zip') + targname = os.path.join(self.share_loc, os.path.basename(zfname)+'.zip') log.info(f'targname is {targname}') shutil.copyfile(zfname+'.zip', targname) tkMessageBox.showinfo('Info', 'Raw Data Uploaded to Dropbox') - subprocess.Popen(f'explorer "{self.shareloc}"') + subprocess.Popen(f'explorer "{self.share_loc}"') except Exception: tkMessageBox.showinfo('Warning', 'Problem with upload') return def viewData(self): - dirpath = self.dir_path.replace('/', '\\') - log.info(f'self-dir-path {dirpath}') - subprocess.Popen(f'explorer "{dirpath}"') + self.review_stack = False + self.soln_outputdir = None + print(self.dir_path) + bin_list = self.get_bin_list() + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) return - def getECSVs(self): + def getRequestedECSVs(self): notgotlist=[] img_list = self.get_bin_list() for current_image in img_list: @@ -543,7 +578,7 @@ def getOneEcsv(self, current_image): #dtval = datetime.datetime.strptime(datestr, '%Y%m%d_%H%M%S') #datestr = dtval.strftime('%Y-%m-%dT%H:%M:%S') try: - lis = getecsv(statid, datestr, savefiles=True, outdir=os.path.join(self.dir_path, statid)) + lis = getECSVs(statid, datestr, savefiles=True, outdir=os.path.join(self.dir_path, statid)) for li in lis: if 'issue getting data' in li: return False @@ -718,21 +753,6 @@ def update_image(self, thing): self.timestamp_label.configure(text = os.path.split(self.current_image)[1]) return - def save_image(self): - """ Marks the image as of interest - """ - current_image = self.listbox.get(ACTIVE) - if current_image == '': - return - log.info(f'marking {current_image}') - srcfile = createTxtFile(current_image, self.dir_path) - _, targfile = os.path.split(srcfile) - s3 = boto3.client('s3') - s3.upload_file(srcfile, self.upload_bucket, f'{self.upload_folder}/{targfile}') - cur_index = int(self.listbox.curselection()[0]) - self.listbox.itemconfig(cur_index, fg = 'green') - self.selected[current_image] = (1, srcfile) - def clean_folder(self): stacklist = os.listdir(os.path.join(self.dir_path, 'stacks')) camlist = [x[:6] for x in stacklist if 'stack.jpg' in x] diff --git a/fbCollector/helperfunctions.ps1 b/fbCollector/helperfunctions.ps1 deleted file mode 100644 index 08b14d92..00000000 --- a/fbCollector/helperfunctions.ps1 +++ /dev/null @@ -1,107 +0,0 @@ -# helper functions - -Function Get-IniContent { - <# - .Synopsis - Gets the content of an INI file - - .Description - Gets the content of an INI file and returns it as a hashtable - - .Notes - Author : Oliver Lipkau - Blog : http://oliver.lipkau.net/blog/ - Source : https://github.com/lipkau/PsIni - http://gallery.technet.microsoft.com/scriptcenter/ea40c1ef-c856-434b-b8fb-ebd7a76e8d91 - Version : 1.0 - 2010/03/12 - Initial release - 1.1 - 2014/12/11 - Typo (Thx SLDR) - Typo (Thx Dave Stiff) - - #Requires -Version 2.0 - - .Inputs - System.String - - .Outputs - System.Collections.Hashtable - - .Parameter FilePath - Specifies the path to the input file. - - .Example - $FileContent = Get-IniContent "C:\myinifile.ini" - ----------- - Description - Saves the content of the c:\myinifile.ini in a hashtable called $FileContent - - .Example - $inifilepath | $FileContent = Get-IniContent - ----------- - Description - Gets the content of the ini file passed through the pipe into a hashtable called $FileContent - - .Example - C:\PS>$FileContent = Get-IniContent "c:\settings.ini" - C:\PS>$FileContent["Section"]["Key"] - ----------- - Description - Returns the key "Key" of the section "Section" from the C:\settings.ini file - - .Link - Out-IniFile - #> - - [CmdletBinding()] - Param( - [ValidateNotNullOrEmpty()] - [ValidateScript({(Test-Path $_) -and ((Get-Item $_).Extension -eq ".ini")})] - [Parameter(ValueFromPipeline=$True,Mandatory=$True)] - [string]$FilePath - ) - - Begin - {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function started"} - - Process - { - Write-Verbose "$($MyInvocation.MyCommand.Name):: Processing file: $Filepath" - - $ini = @{} - switch -regex -file $FilePath - { - "^\[(.+)\]$" # Section - { - $section = $matches[1] - $ini[$section] = @{} - $CommentCount = 0 - } - "^(;.*)$" # Comment - { - if (!($section)) - { - $section = "No-Section" - $ini[$section] = @{} - } - $value = $matches[1] - $CommentCount = $CommentCount + 1 - $name = "Comment" + $CommentCount - $ini[$section][$name] = $value - } - "(.+?)\s*=\s*(.*)" # Key - { - if (!($section)) - { - $section = "No-Section" - $ini[$section] = @{} - } - $name,$value = $matches[1..2] - $ini[$section][$name] = $value - } - } - Write-Verbose "$($MyInvocation.MyCommand.Name):: Finished Processing file: $FilePath" - Return $ini - } - - End - {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function ended"} -} \ No newline at end of file diff --git a/fbCollector/requirements.txt b/fbCollector/requirements.txt index e3cff877..60f47e8c 100644 --- a/fbCollector/requirements.txt +++ b/fbCollector/requirements.txt @@ -6,3 +6,5 @@ paramiko meteortools scp pyqtgraph +xmltodict +requests \ No newline at end of file diff --git a/fbCollector/vars.yml b/fbCollector/vars.yml index 186e973a..e7e28879 100644 --- a/fbCollector/vars.yml +++ b/fbCollector/vars.yml @@ -1,2 +1,2 @@ - srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/ukmda-dataprocessing/fbCollector" - destdir: "/mnt/c/Users/{{ lookup('env','USER' )}}//meteors/fbCollector" + srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/fbCollector" + destdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/meteors/fbCollector" From ca3088ee6b9bea569c9bd7f42f9b15becedd4b31 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Tue, 7 Oct 2025 16:49:29 +0100 Subject: [PATCH 54/83] remove dependency on meteortools and cleanup requirements file --- fbCollector/fireballCollector.py | 78 +++++++++++++++++++++++++++++++- fbCollector/requirements.txt | 5 +- 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index ed9ad988..fdc2a58f 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -16,11 +16,11 @@ import xmltodict from PIL import Image import requests +import pandas as pd import paramiko from scp import SCPClient -from meteortools.ukmondb import getECSVs, getLiveJpgs from wmpl.Formats.ECSV import loadECSVs from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric @@ -110,6 +110,82 @@ def on_validate(self, new_value): return True +def getECSVs(stationID, dateStr, savefiles=False, outdir='.'): + """ + Retrieve a detection in ECSV format for the specified date + """ + apiurl='https://api.ukmeteors.co.uk/getecsv?stat={}&dt={}' + res = requests.get(apiurl.format(stationID, dateStr)) + ecsvlines='' + if res.status_code == 200: + rawdata=res.text.strip() + if len(rawdata) > 10: + ecsvlines=rawdata.split('\n') # convert the raw data into a python list + if savefiles is True: + os.makedirs(outdir, exist_ok=True) + fnamebase = dateStr.replace(':','_').replace('.','_') # create an output filename + j=0 + outf = False + for li in ecsvlines: + if 'issue getting data' in li: + print(li) + return li + if '# %ECSV' in li: + if outf is not False: + outf.close() + j=j+1 + fname = fnamebase + f'_ukmda_{stationID}_M{j:03d}.ecsv' + outf = open(os.path.join(outdir, fname), 'w') + print('saving to ', os.path.join(outdir,fname)) + if outf: + outf.write(f'{li}\n') + else: + print('no ECSV marker found in data') + else: + print('no error, but no data returned') + else: + print(res.status_code) + return ecsvlines + + +def _download(url, outdir, fname=None): + get_response = requests.get(url, stream=True) + if fname is None: + fname = url.split('?')[0].split("/")[-1] + with open(os.path.join(outdir, fname), 'wb') as f: + for chunk in get_response.iter_content(chunk_size=4096): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + return fname + + +def getLiveJpgs(dtstr, outdir=None): + """ + Retrieve live images from the ukmon website that match a pattern + """ + if outdir is None: + outdir = dtstr + os.makedirs(outdir, exist_ok=True) + + apiurl = 'https://api.ukmeteors.co.uk/liveimages/getlive' + + while len(dtstr) < 15: + dtstr = dtstr + '0' + isodt1 = datetime.datetime.strptime(dtstr[:15],'%Y%m%d_%H%M%S') + fromdstr = isodt1.isoformat()[:19]+'.000Z' + isodt2 = isodt1 + datetime.timedelta(minutes=1) + todstr = isodt2.isoformat()[:19]+'.000Z' + liveimgs = pd.read_json(f'{apiurl}?dtstr={fromdstr}&enddtstr={todstr}&fmt=withxml') + + for _, thisimg in liveimgs.iterrows(): + try: + jpgurl = thisimg .urls['url'] + fname = _download(jpgurl, outdir) + log.info(f'retrieved {fname}') + except: + log.warning(f'{img.image_name} unavailable') + + class fbCollector(Frame): def __init__(self, parent, patt=None): diff --git a/fbCollector/requirements.txt b/fbCollector/requirements.txt index 60f47e8c..f8283c73 100644 --- a/fbCollector/requirements.txt +++ b/fbCollector/requirements.txt @@ -1,10 +1,9 @@ # requirements for the fbCollector tool # Copyright (C) 2018-2023 Mark McIntyre -boto3 Pillow paramiko -meteortools scp pyqtgraph xmltodict -requests \ No newline at end of file +requests +pandas \ No newline at end of file From 95096d5705652dfe5af69ee998f5231bedb0316e Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 9 Oct 2025 15:37:08 +0100 Subject: [PATCH 55/83] bugfix to getExtrafiles when pickle comment is malformed --- .../getExtraFilesV2/pythoncode/getExtraFiles.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py index 6f08afed..8c71bd7b 100644 --- a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py +++ b/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py @@ -80,7 +80,10 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): try: js = json.loads(obs.comment) ffname = js['ff_name'] - print(ffname) + if ffname[0] == '[': + ffname = ffname[2:-2].split(',') + if isinstance(ffname, list): + ffname = ffname[0] # case when filename is nonstandard if 'FF_' not in ffname and 'FR_' not in ffname: ffname = 'FF_' + ffname @@ -97,6 +100,7 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3): else: pass if gotff is True: + print(ffname) spls = ffname.split('_') id = spls[1] dtstr = spls[2] From 25be5b96087cfa81d62baf496d2fccd792022a66 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 9 Oct 2025 15:37:36 +0100 Subject: [PATCH 56/83] include extragpgs file in uploads --- archive/ukmon_pylib/maintenance/recreateOrbitPages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py index 61b5c5bd..4f5f050c 100644 --- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py +++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py @@ -151,6 +151,8 @@ def checkIfFileNeeded(filename): return True if 'trajectory.pickle' in filename: return True + if 'extra' in filename: + return True return False From 7715ac724a2454bc6fe773c96fe9d433ae07cf90 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 9 Oct 2025 15:38:01 +0100 Subject: [PATCH 57/83] attempt to avoid delete problems with uploads --- fbCollector/fireballCollector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index fdc2a58f..09d48112 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -579,9 +579,9 @@ def uploadOrbit(self): os.makedirs(tmpdir, exist_ok=True) shutil.copyfile(pickfile, os.path.join(tmpdir, orbname)) if os.path.isdir(os.path.join(self.dir_path, 'jpgs')): - shutil.copytree(os.path.join(self.dir_path, 'jpgs'), os.path.join(tmpdir, 'jpgs')) + shutil.copytree(os.path.join(self.dir_path, 'jpgs'), os.path.join(tmpdir, 'jpgs'), dirs_exist_ok=True) if os.path.isdir(os.path.join(self.dir_path, 'mp4s')): - shutil.copytree(os.path.join(self.dir_path, 'mp4s'), os.path.join(tmpdir, 'mp4s')) + shutil.copytree(os.path.join(self.dir_path, 'mp4s'), os.path.join(tmpdir, 'mp4s'), dirs_exist_ok=True) for path, _, files in os.walk(self.dir_path): for name in files: if '_dyn_mass_fit' in name: From 6c81139af97043ff5cd415ec1d39e5c2b2efdf73 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 9 Oct 2025 15:40:31 +0100 Subject: [PATCH 58/83] update readme for fbCollector --- fbCollector/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fbCollector/README.md b/fbCollector/README.md index 66d21a5f..fc59cfe7 100644 --- a/fbCollector/README.md +++ b/fbCollector/README.md @@ -4,8 +4,8 @@ This tool allows authorised users to collect fireball data from UKMON and GMN an ## Prerequisites -* [WMPL](https://github.com/wmpg/WesternMeteorPyLib/), used to solve trajectories. -* [RMS](https://github.com/CroatianMeteorNetwork/RMS), used to reduce raw data. +* You have installed [WMPL](https://github.com/wmpg/WesternMeteorPyLib/), used to solve trajectories. +* You have installed [RMS](https://github.com/CroatianMeteorNetwork/RMS), used to reduce raw data. * A local folder where you will store fireball data. This is called `basedir` in this documentation. The above are sufficient to collect data from UKMON and to analyse data either collected from UKMON or by other means. There are additional configuration options that can be used to collect data from GMN and upload solutions to UKMON. These are described inline below. From 084460dd512ab21e7fd1472948eacd67e426ca2b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Thu, 9 Oct 2025 21:07:01 +0100 Subject: [PATCH 59/83] small bugfixes to find the gmn script properly --- fbCollector/fbCollector.ps1 | 4 ++-- fbCollector/fireballCollector.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/fbCollector/fbCollector.ps1 b/fbCollector/fbCollector.ps1 index 29c0a786..52d6e2c1 100644 --- a/fbCollector/fbCollector.ps1 +++ b/fbCollector/fbCollector.ps1 @@ -15,9 +15,9 @@ $wmpl_loc = $wmpl_loc.replace('\','/') $env:pythonpath="$wmpl_loc" if ($args.count -lt 1) { - python fireballCollector.py + python $PSScriptRoot/fireballCollector.py }else { - python fireballCollector.py -d $args[0] + python $PSScriptRoot/fireballCollector.py -d $args[0] } Pop-Location diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 09d48112..46dc216e 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -212,6 +212,7 @@ def __init__(self, parent, patt=None): self.review_stack = False self.soln_outputdir = None self.log_files_to_keep = 30 + self.script_loc = os.path.dirname(__file__) self.readConfig() @@ -220,7 +221,8 @@ def __init__(self, parent, patt=None): self.dir_path = self.fb_dir.strip() else: self.dir_path = os.path.join(self.fb_dir, patt) - log.info(f"Fireball folder is {self.fb_dir}") + log.info(f'Fireball folder is {self.fb_dir}') + log.info(f'Scripts folder is {self.script_loc}') self.initUI() if self.dir_path != self.fb_dir: @@ -958,7 +960,7 @@ def putWatchlist(self): log.info('Uploading watchlist') evtfile = os.path.join(self.fb_dir,'event_watchlist.txt') scpcli.put(evtfile, 'event_watchlist.txt') - self.evtMonTriggered = datetime.datetime.now() + datetime.timedelta(minutes=10) + self.evtMonTriggered = datetime.datetime.now() + datetime.timedelta(minutes=5) return def getEventData(self): @@ -966,18 +968,20 @@ def getEventData(self): if len(evtdate) < 15: tkMessageBox.showinfo("Warning", f'Need seconds in the event date field {evtdate}') return - cmd = os.path.join(os.path.dirname(__file__), 'download_events.sh') + f' {evtdate} 1' + cmd = os.path.join(self.script_loc, 'download_events.sh') + f' {evtdate} 1' if ':' in cmd: drv = cmd[0].lower() cmd = '/mnt/' + drv + cmd[2:] cmd = cmd.replace('\\','/') - log.warning(f'executing {cmd}') + log.info(f'executing {cmd}') if self.evtMonTriggered is None: - tkMessageBox.showinfo("Warning", 'Event Monitor has not been triggered') - return - if datetime.datetime.now() < self.evtMonTriggered: - tkMessageBox.showinfo("Warning", f'Wait till at least {self.evtMonTriggered.strftime("%H:%M:%S")}') - return + ret = tkMessageBox.askyesno("Warning", 'Event Monitor has not been triggered, continue?') + if ret is False: + return + elif datetime.datetime.now() < self.evtMonTriggered: + ret = tkMessageBox.askyesno("Wait", f'Should wait till {self.evtMonTriggered.strftime("%H:%M:%S")} - continue anyway?') + if ret is False: + return os.chdir(self.fb_dir) log.info(f'getting data for {evtdate}') procid = subprocess.Popen(('bash','-c', cmd)) From 09154061c80ab1d5ca1f1f9901297eef0457f0bc Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 10 Oct 2025 14:07:49 +0100 Subject: [PATCH 60/83] switch to new match data api handler that uses the mysql database --- .../samfunctions/matchDataApi/localTest.ps1 | 3 ++ .../matchDataApi/requirements.txt | 6 +++ .../samfunctions/matchDataApi/samconfig.toml | 12 ++++++ .../samfunctions/matchDataApi/template.yml | 38 +++++++++++++++++++ .../samfunctions/matchDataApi/testDetail.json | 7 ++++ .../matchDataApi/testMatches.json | 7 ++++ .../samfunctions/matchDataApi/testPoints.json | 8 ++++ .../matchDataApi/testStation.json | 8 ++++ .../matchDataApi/testSummary.json | 7 ++++ archive/terraform/ukmda/apigw-domain.tf | 13 +++++++ .../files/matchDataApi/matchDataApiHandler.py | 4 +- archive/terraform/ukmda/matchApiLambda.tf | 2 +- 12 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 archive/samfunctions/matchDataApi/localTest.ps1 create mode 100644 archive/samfunctions/matchDataApi/requirements.txt create mode 100644 archive/samfunctions/matchDataApi/samconfig.toml create mode 100644 archive/samfunctions/matchDataApi/template.yml create mode 100644 archive/samfunctions/matchDataApi/testDetail.json create mode 100644 archive/samfunctions/matchDataApi/testMatches.json create mode 100644 archive/samfunctions/matchDataApi/testPoints.json create mode 100644 archive/samfunctions/matchDataApi/testStation.json create mode 100644 archive/samfunctions/matchDataApi/testSummary.json diff --git a/archive/samfunctions/matchDataApi/localTest.ps1 b/archive/samfunctions/matchDataApi/localTest.ps1 new file mode 100644 index 00000000..c71174de --- /dev/null +++ b/archive/samfunctions/matchDataApi/localTest.ps1 @@ -0,0 +1,3 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# sam build --profile ukmonshared --region eu-west-1 +sam local invoke --profile ukmonshared -e $args[0] --region eu-west-1 diff --git a/archive/samfunctions/matchDataApi/requirements.txt b/archive/samfunctions/matchDataApi/requirements.txt new file mode 100644 index 00000000..917ec643 --- /dev/null +++ b/archive/samfunctions/matchDataApi/requirements.txt @@ -0,0 +1,6 @@ +# +# requirements for matchDataApi +# Copyright (C) 2025 Mark McIntyre +# +# mysql-connector-python +pymysql diff --git a/archive/samfunctions/matchDataApi/samconfig.toml b/archive/samfunctions/matchDataApi/samconfig.toml new file mode 100644 index 00000000..074b92a0 --- /dev/null +++ b/archive/samfunctions/matchDataApi/samconfig.toml @@ -0,0 +1,12 @@ +# Copyright (C) 2018-2023 Mark McIntyre +version = 0.1 +[default] +[default.deploy] +[default.deploy.parameters] +stack_name = "matchDataApi" +s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-1cy2k1achj9nv" +s3_prefix = "matchDataApi" +region = "eu-west-1" +capabilities = "CAPABILITY_IAM" +image_repositories = [] +profile = "ukmda_admin" diff --git a/archive/samfunctions/matchDataApi/template.yml b/archive/samfunctions/matchDataApi/template.yml new file mode 100644 index 00000000..2d2deb74 --- /dev/null +++ b/archive/samfunctions/matchDataApi/template.yml @@ -0,0 +1,38 @@ +# SAM build file for searchArchive +# Copyright (C) 2018-2023 Mark McIntyre +AWSTemplateFormatVersion: '2010-09-09' +Transform: 'AWS::Serverless-2016-10-31' +Resources: + matchDataApi: + Type: AWS::Serverless::Function + Properties: + Handler: matchDataApi.lambda_handler + Runtime: python3.11 + FunctionName: matchDataApi + Description: API backend that fetches match data + Timeout: 120 + Policies: [AmazonS3ReadOnlyAccess] + Environment: + Variables: + ARCHBUCKET: ukmda-shared + MPLCONFIGDIR: /tmp/mpl + Events: + HttpGet: + Type: Api + Properties: + Path: '/' + Method: get + RequestParameters: + - method.request.querystring.reqtyp: + Required: true + - method.request.querystring.reqval: + Required: true + - method.request.querystring.points: + Required: false + - method.request.querystring.statid: + Required: false + Tags: + billingtag: "ukmda" + typetag: "api" + Metadata: + BuildMethod: python3.11 diff --git a/archive/samfunctions/matchDataApi/testDetail.json b/archive/samfunctions/matchDataApi/testDetail.json new file mode 100644 index 00000000..ee4ab413 --- /dev/null +++ b/archive/samfunctions/matchDataApi/testDetail.json @@ -0,0 +1,7 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "detail", + "reqval": "20251008_235928.502_UK" + } +} \ No newline at end of file diff --git a/archive/samfunctions/matchDataApi/testMatches.json b/archive/samfunctions/matchDataApi/testMatches.json new file mode 100644 index 00000000..67a03d17 --- /dev/null +++ b/archive/samfunctions/matchDataApi/testMatches.json @@ -0,0 +1,7 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "matches", + "reqval": "20251008" + } +} \ No newline at end of file diff --git a/archive/samfunctions/matchDataApi/testPoints.json b/archive/samfunctions/matchDataApi/testPoints.json new file mode 100644 index 00000000..afcac236 --- /dev/null +++ b/archive/samfunctions/matchDataApi/testPoints.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "detail", + "reqval": "20251008_235928.502_UK", + "points": 1 + } +} \ No newline at end of file diff --git a/archive/samfunctions/matchDataApi/testStation.json b/archive/samfunctions/matchDataApi/testStation.json new file mode 100644 index 00000000..22e338fa --- /dev/null +++ b/archive/samfunctions/matchDataApi/testStation.json @@ -0,0 +1,8 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "station", + "statid": "UK0006", + "reqval": "20251008" + } +} \ No newline at end of file diff --git a/archive/samfunctions/matchDataApi/testSummary.json b/archive/samfunctions/matchDataApi/testSummary.json new file mode 100644 index 00000000..785013bc --- /dev/null +++ b/archive/samfunctions/matchDataApi/testSummary.json @@ -0,0 +1,7 @@ +{ + "httpMethod": "GET", + "queryStringParameters": { + "reqtyp": "summary", + "reqval": "20251008" + } +} \ No newline at end of file diff --git a/archive/terraform/ukmda/apigw-domain.tf b/archive/terraform/ukmda/apigw-domain.tf index 3cb7995e..1baacdf3 100644 --- a/archive/terraform/ukmda/apigw-domain.tf +++ b/archive/terraform/ukmda/apigw-domain.tf @@ -40,6 +40,11 @@ data "aws_api_gateway_rest_api" "matchpickleapi" { provider = aws.eu-west-1-prov } +data "aws_api_gateway_rest_api" "matchdataapi" { + name = "matchDataApi" + provider = aws.eu-west-1-prov +} + data "aws_api_gateway_rest_api" "liveimagesapi" { name = "getLiveImages" provider = aws.eu-west-1-prov @@ -71,6 +76,14 @@ resource "aws_api_gateway_base_path_mapping" "camdetapi" { provider = aws.eu-west-1-prov } +resource "aws_api_gateway_base_path_mapping" "matchdataapi" { + api_id = data.aws_api_gateway_rest_api.matchdataapi.id + stage_name = "Prod" + domain_name = aws_api_gateway_domain_name.apigwdomain.domain_name + base_path = "matches" + provider = aws.eu-west-1-prov +} + resource "aws_api_gateway_base_path_mapping" "pickleapi" { api_id = data.aws_api_gateway_rest_api.matchpickleapi.id stage_name = "Prod" diff --git a/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py b/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py index 6b1b2e96..57ffa44c 100644 --- a/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py +++ b/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py @@ -24,7 +24,7 @@ def lambda_handler(event, context): if reqtyp not in ['matches','detail','station','summary']: res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}' else: - print(f'{reqtyp} {reqval} {points}') + print(f'reqtype {reqtyp} reqval {reqval} points {points}') if reqtyp == 'summary': d1 = datetime.datetime.strptime(reqval, '%Y%m%d') idxfile = 'matches/matched/matches-full-{:04d}.csv'.format(d1.year) @@ -47,6 +47,7 @@ def lambda_handler(event, context): fhi = {"FileHeaderInfo": "Use"} elif reqtyp == 'station': statid = qs['statid'] + print(f'statid {statid}') d1 = datetime.datetime.strptime(reqval, '%Y%m%d') idxfile = 'matches/matched/matches-full-{:04d}.csv'.format(d1.year) res = '{"no matches"}' @@ -78,6 +79,7 @@ def lambda_handler(event, context): print('report file unavailable') res = '{"points": "unavailable"}' + print(res) return { 'statusCode': 200, 'body': res diff --git a/archive/terraform/ukmda/matchApiLambda.tf b/archive/terraform/ukmda/matchApiLambda.tf index 30348402..68be3913 100644 --- a/archive/terraform/ukmda/matchApiLambda.tf +++ b/archive/terraform/ukmda/matchApiLambda.tf @@ -57,7 +57,7 @@ resource "aws_api_gateway_base_path_mapping" "matchapi" { api_id = aws_api_gateway_rest_api.matchapi_apigateway.id stage_name = "prod" domain_name = aws_api_gateway_domain_name.apigwdomain.domain_name - base_path = "matches" + base_path = "matchesv1" provider = aws.eu-west-1-prov depends_on = [aws_api_gateway_stage.matchapistage] } From 5f90efc2ecb9769f6c3b18b28408a5ed4b5a80cd Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 10 Oct 2025 14:08:03 +0100 Subject: [PATCH 61/83] remove unused metric alarm --- archive/terraform/ukmda/cloudwatch.tf | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/archive/terraform/ukmda/cloudwatch.tf b/archive/terraform/ukmda/cloudwatch.tf index 10e03875..61ab690a 100644 --- a/archive/terraform/ukmda/cloudwatch.tf +++ b/archive/terraform/ukmda/cloudwatch.tf @@ -3,33 +3,6 @@ # # Copyright (C) 2018-2023 Mark McIntyre -resource "aws_cloudwatch_metric_alarm" "calcServerDiskSpace" { - alarm_name = "CalcServer diskspace" - comparison_operator = "GreaterThanThreshold" - evaluation_periods = "1" - metric_name = "disk_used_percent" - namespace = "CWAgent" - period = "300" - statistic = "Average" - threshold = "90" - alarm_description = "CalcServer diskspace has gone over 90%" - insufficient_data_actions = [] - datapoints_to_alarm = 1 - alarm_actions = [aws_sns_topic.ukmdaalerts.arn, ] - dimensions = { - "ImageId" = aws_instance.calc_server.ami - "InstanceId" = aws_instance.calc_server.id - "InstanceType" = aws_instance.calc_server.instance_type - "device" = "nvme0n1p1" - "fstype" = "xfs" - "path" = "/" - } - tags = { - "billingtag" = "ukmda" - } - actions_enabled = true -} - resource "aws_cloudwatch_metric_alarm" "calcServerIdle" { alarm_name = "CalcServer idle shutdown" comparison_operator = "LessThanOrEqualToThreshold" From edbe01419d41447ca93291e7074976b8e5e64325 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 10 Oct 2025 14:08:29 +0100 Subject: [PATCH 62/83] update calcserver type and keep ubuntu version for now --- archive/terraform/ukmda/ec2.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/ec2.tf index 8c1ed70c..29c5fd71 100644 --- a/archive/terraform/ukmda/ec2.tf +++ b/archive/terraform/ukmda/ec2.tf @@ -2,7 +2,7 @@ resource "aws_instance" "calc_server" { ami = "ami-0df2d8f6def0bd716" - instance_type = "c8g.4xlarge" + instance_type = "c8g.2xlarge" iam_instance_profile = aws_iam_instance_profile.calcserverrole.name key_name = aws_key_pair.marks_key.key_name force_destroy = false @@ -44,7 +44,7 @@ resource "aws_network_interface" "calcserver_if" { ################################################ # Ubuntu calc server ################################################ -/* + resource "aws_instance" "ubuntu_calc_server" { ami = "ami-0bdf149a42243bde8" instance_type = "c6g.4xlarge" @@ -84,7 +84,7 @@ resource "aws_network_interface" "ubuntu_calcserver_if" { "billingtag" = "ukmda" } } -*/ + ################################################ # admin server ################################################ From 9840d30c9b1a678c77a98de5f5538e50703881e7 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 10 Oct 2025 15:38:07 +0100 Subject: [PATCH 63/83] adding SSM variables for db access --- archive/terraform/ukmda/ssm_variables.tf | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 archive/terraform/ukmda/ssm_variables.tf diff --git a/archive/terraform/ukmda/ssm_variables.tf b/archive/terraform/ukmda/ssm_variables.tf new file mode 100644 index 00000000..29fba5ac --- /dev/null +++ b/archive/terraform/ukmda/ssm_variables.tf @@ -0,0 +1,43 @@ +# Copyright (C) 2018-2023 Mark McIntyre + +# SSM parameters for use in Lambdas + +resource "aws_ssm_parameter" "prod_dbhost" { + provider = aws.eu-west-1-prov + name = "prod_dbhost" + type = "String" + value = "3.11.55.160" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "prod_dbname" { + provider = aws.eu-west-1-prov + name = "prod_dbname" + type = "String" + value = "ukmon" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "prod_dbpw" { + provider = aws.eu-west-1-prov + name = "prod_dbpw" + type = "SecureString" + value = "Batch33mdl" + tags = { + "billingtag" = "ukmon" + } +} + +resource "aws_ssm_parameter" "prod_dbuser" { + provider = aws.eu-west-1-prov + name = "prod_dbuser" + type = "String" + value = "batch" + tags = { + "billingtag" = "ukmon" + } +} From 09ad729212aaa4075f97545a44828d951aa2ac8a Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Fri, 10 Oct 2025 15:57:51 +0100 Subject: [PATCH 64/83] new match api written with AWS SAM --- .../samfunctions/matchDataApi/matchDataApi.py | 169 ++++++++++++++++++ .../samfunctions/matchDataApi/template.yml | 2 +- 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 archive/samfunctions/matchDataApi/matchDataApi.py diff --git a/archive/samfunctions/matchDataApi/matchDataApi.py b/archive/samfunctions/matchDataApi/matchDataApi.py new file mode 100644 index 00000000..b7c2023e --- /dev/null +++ b/archive/samfunctions/matchDataApi/matchDataApi.py @@ -0,0 +1,169 @@ +# Copyright (C) 2018-2023 Mark McIntyre +# +# python code behind the MatchData API +# + +import os +import boto3 +import json +import pymysql.cursors + + +def getSqlLoginDetails(): + # retrieve password and host from SSM. This allows me to manage them from Terraform + ssm = boto3.client('ssm', region_name='eu-west-1') + res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True) + password = res['Parameter']['Value'] + res = ssm.get_parameter(Name='prod_dbhost') + host = res['Parameter']['Value'] + # should really do these too but they won't change often if at all + user = 'batch' + db = 'ukmon' + return host, user, password, db + + +def getStationData(statid, dtstr): + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + try: + with connection.cursor() as cursor: + if statid is None: + sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%'" + else: + sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' and s.stations like '%{statid}%'" + cursor.execute(sql) + result = cursor.fetchall() + finally: + connection.close() + res='' + for event in result: + res = res + json.dumps(event) + '\n' + return res + + +def getSummaryData(dtstr): + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\ + '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\ + 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations' + expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%'" + result=[] + try: + with connection.cursor() as cursor: + cursor.execute(expr) + result = cursor.fetchall() + finally: + connection.close() + if len(result) > 0: + res = json.dumps(result) + else: + res=json.dumps({'result': 'no data'}) + res = res.replace('}, {','},\n{') + return res + + +def getDetailData(orbname): + host, user, passwd, db = getSqlLoginDetails() + connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor) + expr = f"SELECT * from matches s where s.orbname like '{orbname}%'" + try: + with connection.cursor() as cursor: + cursor.execute(expr) + result = cursor.fetchall() + finally: + connection.close() + if len(result) > 0: + res = json.dumps(result[0]) + else: + res=json.dumps({'result': 'no data'}) + res = res.replace('}, {','},\n{') + return res + + +def lambda_handler(event, context): + webbuck = os.getenv('WEBBUCKET', default='ukmda-website') + #print('received event', json.dumps(event)) + qs = event['queryStringParameters'] + if qs is None: + return { + 'statusCode': 200, + 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1]' + } + reqtyp = qs['reqtyp'] + points = False + if 'points' in qs: + points = True + + if reqtyp == 'station': + statid = qs['statid'] + dtstr = qs['reqval'] + print(f'station data requested for {statid} on {dtstr}') + res = getStationData(statid, dtstr) + elif reqtyp == 'summary': + dtstr = qs['reqval'] + print(f'summary data requested for {dtstr}') + res = getSummaryData(dtstr) + elif reqtyp == 'matches': + dtstr = qs['reqval'] + print(f'match data requested for {dtstr}') + res = getStationData(None, dtstr) + elif reqtyp == 'detail': + orbname = qs['reqval'] + print(f'detail requested for {orbname}') + res = getDetailData(orbname) + if points: + print(f'points requested for {orbname}') + if 'imgstr' in res: + url = json.loads(res)['imgstr'] + url = url.replace('ground_track.png','report.txt') + reppth = url.split('//', 1)[1].split('/',1)[1] + _, repname = os.path.split(reppth) + try: + tmpfname = f'/tmp/{repname}' + s3 = boto3.client('s3') + s3.download_file(webbuck, reppth, tmpfname) + flis = open(tmpfname, 'r').readlines() + os.remove(tmpfname) + res = fileToJsonString(flis) + except Exception: + print('report file unavailable') + res = '{"points": "unavailable"}' + else: + res = '{"points": "unavailable"}' + + else: + res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}' + + print(res) + return { + 'statusCode': 200, + 'body': res + } + + +def fileToJsonString(flis): + hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao', + 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd', + 'lag','vel','pvel', 'hres','vres','ares','vmag','amag'] + ptsarray='[' + gotpts = False + for fli in flis: + if 'Points' in fli: + gotpts = True + continue + elif '------' in fli or ' No' in fli: + continue + elif gotpts is True and (len(fli) < 2 or 'Notes' in fli): + gotpts=False + break + elif gotpts is True: + spls = fli.split(',') + thisrow = '{' + for h, s in zip(hdr, spls): + thisrow = thisrow + f'"{h}": "{s.strip()}",' + thisrow = thisrow[:-1] + '},' + ptsarray = ptsarray + thisrow + ptsarray = ptsarray[:-1] + ']' + ptsarray = '{' + f'"points": {ptsarray}' + '}' + return ptsarray diff --git a/archive/samfunctions/matchDataApi/template.yml b/archive/samfunctions/matchDataApi/template.yml index 2d2deb74..d07529b3 100644 --- a/archive/samfunctions/matchDataApi/template.yml +++ b/archive/samfunctions/matchDataApi/template.yml @@ -11,7 +11,7 @@ Resources: FunctionName: matchDataApi Description: API backend that fetches match data Timeout: 120 - Policies: [AmazonS3ReadOnlyAccess] + Policies: [AmazonS3ReadOnlyAccess, AmazonSSMReadOnlyAccess ] Environment: Variables: ARCHBUCKET: ukmda-shared From 6885f3166dde690ca8fa2ef8d85516442529d87f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 17:46:31 +0100 Subject: [PATCH 65/83] add cleanup phase --- .github/workflows/build_usermgmt.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build_usermgmt.yml b/.github/workflows/build_usermgmt.yml index 5d078517..ac0dc268 100644 --- a/.github/workflows/build_usermgmt.yml +++ b/.github/workflows/build_usermgmt.yml @@ -69,3 +69,13 @@ jobs: tag_name: 2024.04.02 default_release_name: User Management Tool default_release_body_path: usermgmt\windows\release_notes.md + cleanup: + name: clean up package + runs-on: win10 + needs: + - build + - release + steps: + - name: delete package + run: | + c:\temp\ukmon_usermgmt.zip From 7dade291a3f2fb80de76db137cb1089c5191c7c6 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 17:49:32 +0100 Subject: [PATCH 66/83] update usernamagement release notes --- usermgmt/windows/release_notes.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usermgmt/windows/release_notes.md b/usermgmt/windows/release_notes.md index de9a86dd..979fd1c8 100644 --- a/usermgmt/windows/release_notes.md +++ b/usermgmt/windows/release_notes.md @@ -3,7 +3,7 @@ This application is used to add, modify and disable RMS stations linked to UKMON. The app is intended for use only by the UKMON committee members responsible for managing users. ## Before Installation -First you should contact me, Mark McIntyre. You will need to send me your ssh public key, and i will then create a userid for you on our server and send you the details. +First you should contact me, Mark McIntyre so we can set up a userid for you. ## Installation Download the zip file and expand it to a location of your choice, for example `c:\programdata\ukmon` @@ -15,4 +15,5 @@ Save the config file and run the app. The menus should be pretty self-explanatory. ## Release Notes -2024.04.2 - first release. \ No newline at end of file +2024.04.2 - first release. +2025.10.1 - overhaul to fix a plethora of bugs and add new features. \ No newline at end of file From aae01bb8eca3482540784454b6dbd0e8dcb20e7b Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 17:51:46 +0100 Subject: [PATCH 67/83] update version to 2025.10.1 --- .github/workflows/build_usermgmt.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_usermgmt.yml b/.github/workflows/build_usermgmt.yml index ac0dc268..14a6b1ba 100644 --- a/.github/workflows/build_usermgmt.yml +++ b/.github/workflows/build_usermgmt.yml @@ -66,7 +66,7 @@ jobs: #tags: true draft: true overwrite: true - tag_name: 2024.04.02 + tag_name: 2025.10.1 default_release_name: User Management Tool default_release_body_path: usermgmt\windows\release_notes.md cleanup: From b24796e03496055c0d95e92a746d4ef88bdb76e4 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 17:52:21 +0100 Subject: [PATCH 68/83] update readme and remove entries from sample config file --- fbCollector/README.md | 1 + fbCollector/config.ini.sample | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fbCollector/README.md b/fbCollector/README.md index fc59cfe7..4887015b 100644 --- a/fbCollector/README.md +++ b/fbCollector/README.md @@ -40,6 +40,7 @@ Members of the UKMON team who frequently create solutions can request an API key ### Collecting data from GMN Members of the GMN coordinators group who have permission from Denis Vida can use this tool to collect data directly from GMN or using the GMN Watchlist. If you fall into this category you can fill in the `[gmnanalysis]` section of the config file with the name of your SSH private key and other details. This will activate additional menu options to `Get GMN Raw Data` and to use the `Watchlist`. +If you're running on Windows, you will need WSL2 enabled and the rsync tool installed in WSL2. ## Logs The programme creates logs in your system's `TEMP` folder. diff --git a/fbCollector/config.ini.sample b/fbCollector/config.ini.sample index 4b26b0d3..889cf554 100644 --- a/fbCollector/config.ini.sample +++ b/fbCollector/config.ini.sample @@ -23,9 +23,9 @@ shrfldr= [gmn] # the GMN key needs to be configured with Denis Vida # if you have permission you can then collect data from the GMN server -gmnkey=~/.ssh/gmnanalysis +gmnkey= +gmnuser= gmnserver=gmn.uwo.ca -gmnuser=analysis [ukmon] apikey= From 22dd2527c63350a746f72099e66d817f05f8604f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 17:53:14 +0100 Subject: [PATCH 69/83] minor tweak to config file --- fbCollector/config.ini.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fbCollector/config.ini.sample b/fbCollector/config.ini.sample index 889cf554..2128f65d 100644 --- a/fbCollector/config.ini.sample +++ b/fbCollector/config.ini.sample @@ -21,7 +21,7 @@ wmpl_env=wmpl shrfldr= [gmn] -# the GMN key needs to be configured with Denis Vida +# the GMN SSH key needs to be configured with Denis Vida # if you have permission you can then collect data from the GMN server gmnkey= gmnuser= From c0759f97d38a8f6176c5fe92a5cb98dd2345609c Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 18:42:43 +0100 Subject: [PATCH 70/83] Bugfixes and improvements to support building as a windows exe --- fbCollector/fireballCollector.py | 143 +++++++++++++++++++++---------- 1 file changed, 97 insertions(+), 46 deletions(-) diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py index 46dc216e..078f07ad 100644 --- a/fbCollector/fireballCollector.py +++ b/fbCollector/fireballCollector.py @@ -21,8 +21,12 @@ import paramiko from scp import SCPClient -from wmpl.Formats.ECSV import loadECSVs -from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric +try: + from wmpl.Formats.ECSV import loadECSVs + from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric + solveravailable = 'enabled' +except Exception: + solveravailable = 'disabled' import tkinter as tk import tkinter.filedialog as tkFileDialog @@ -53,6 +57,16 @@ def log_timestamp(): return datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') +def showConfig(): + if platform.system() == 'Darwin': # macOS + procid = subprocess.Popen(('open', config_file)) + elif platform.system() == 'Windows': # Windows + procid = subprocess.Popen(('cmd','/c',config_file)) + else: # linux variants + procid = subprocess.Popen(('xdg-open', config_file)) + procid.wait() + + class StyledButton(Button): """ Button with style. """ @@ -212,7 +226,7 @@ def __init__(self, parent, patt=None): self.review_stack = False self.soln_outputdir = None self.log_files_to_keep = 30 - self.script_loc = os.path.dirname(__file__) + self.script_loc = os.path.split(config_file)[0] self.readConfig() @@ -345,35 +359,44 @@ def initUI(self): fileMenu.add_separator() fileMenu.add_command(label="Delete This Folder", command=self.delFolder) fileMenu.add_separator() - fileMenu.add_command(label="Configuration", command=self.showConfig) + fileMenu.add_command(label="Configuration", command=self.reviewConfig) fileMenu.add_command(label="View Logs", command=self.viewLogs) fileMenu.add_separator() fileMenu.add_command(label="Exit", command=self.quitApplication) self.menuBar.add_cascade(label="File", underline=0, menu=fileMenu) + if self.rms_loc: + rmsavailable = 'active' + else: + rmsavailable = 'disabled' + if self.share_loc: + shareavailable = 'active' + else: + shareavailable = 'disabled' + if self.gmn_key: + gmnavailable ='active' + else: + gmnavailable ='disabled' + rawMenu = Menu(self.menuBar, tearoff=0) rawMenu.add_command(label="Get Live Images", command=self.get_data) rawMenu.add_command(label="Get Videos", command=self.get_vids) rawMenu.add_separator() rawMenu.add_command(label="Get ECSVs", command=self.getRequestedECSVs) rawMenu.add_command(label="Excl/Incl ECSV", command=self.ignoreCamera) - if self.rms_loc: - rawMenu.add_command(label="Reduce Selected Image", command=self.reduceCamera) - rawMenu.add_separator() - if self.share_loc: - rawMenu.add_command(label="Share Raw Data", command=self.uploadRaw) - if self.gmn_key: - rawMenu.add_separator() - watchMenu = Menu(self.menuBar, tearoff=0) - watchMenu.add_command(label="Get GMN Raw Data", command=self.getGMNData) - watchMenu.add_separator() - watchMenu.add_command(label="Get Watchlist", command=self.getWatchlist) - watchMenu.add_command(label="View Watchlist", command=self.viewWatchlist) - watchMenu.add_command(label="Upload Watchlist", command=self.putWatchlist) - watchMenu.add_separator() - watchMenu.add_command(label="Fetch Watchlist Event", command=self.getEventData) - rawMenu.add_cascade(label='GMN', menu=watchMenu) - # self.menuBar.add_cascade(label="Watchlist", underline=0, menu=watchMenu) + rawMenu.add_command(label="Reduce Selected Image", command=self.reduceCamera, state=rmsavailable) + rawMenu.add_separator() + rawMenu.add_command(label="Share Raw Data", command=self.uploadRaw, state=shareavailable) + rawMenu.add_separator() + watchMenu = Menu(self.menuBar, tearoff=0) + watchMenu.add_command(label="Get GMN Raw Data", command=self.getGMNData, state=gmnavailable) + watchMenu.add_separator() + watchMenu.add_command(label="Get Watchlist", command=self.getWatchlist, state=gmnavailable) + watchMenu.add_command(label="View Watchlist", command=self.viewWatchlist, state=gmnavailable) + watchMenu.add_command(label="Upload Watchlist", command=self.putWatchlist, state=gmnavailable) + watchMenu.add_separator() + watchMenu.add_command(label="Fetch Watchlist Event", command=self.getEventData, state=gmnavailable) + rawMenu.add_cascade(label='GMN', menu=watchMenu, state=gmnavailable) self.menuBar.add_cascade(label="Raw", underline=0, menu=rawMenu) @@ -384,7 +407,7 @@ def initUI(self): self.menuBar.add_cascade(label="Review", underline=0, menu=revMenu) solveMenu = Menu(self.menuBar, tearoff=0) - solveMenu.add_command(label="Solve", command=self.solveOrbit) + solveMenu.add_command(label="Solve", command=self.solveOrbit, state=solveravailable) solveMenu.add_separator() solveMenu.add_command(label="View Solution", command=self.viewSolution) solveMenu.add_command(label="Delete Solution", command=self.removeSolution) @@ -439,6 +462,11 @@ def initUI(self): self.timestamp_label = Label(self, text = "CCNNNN YYYY-MM-DD HH:MM.SS.mms", font=("Courier", 12)) self.timestamp_label.grid(row = 7, column = 3, sticky = "E") + def reviewConfig(self): + showConfig() + self.readConfig() + self.initUI() + def reduceCamera(self): current_image = self.listbox.get(ACTIVE) if current_image == '': @@ -452,16 +480,20 @@ def reduceCamera(self): _ = subprocess.run(['powershell.exe', tmpscr]) frs = glob.glob(os.path.join(dirname, 'FR*.bin')) if len(frs) > 0: - if not tkMessageBox.askyesno("Rerun", f'{len(frs)} FR files detected - rerun?'): - return - for fr in frs: - with open(tmpscr, 'w') as outf: - outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {fr} -c {dirname}/.config\n') - _ = subprocess.run(['powershell.exe', tmpscr]) + if tkMessageBox.askyesno("Rerun", f'{len(frs)} FR files detected - rerun?'): + for fr in frs: + with open(tmpscr, 'w') as outf: + outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {fr} -c {dirname}/.config\n') + _ = subprocess.run(['powershell.exe', tmpscr]) try: os.remove(tmpscr) except: pass + os.makedirs(os.path.join(self.dir_path,'ecsvs'), exist_ok=True) + ecsvfs = glob.glob1(dirname, '*.ecsv') + for ecsv in ecsvfs: + shutil.copyfile(os.path.join(dirname, ecsv), os.path.join(self.dir_path, 'ecsvs', ecsv)) + return def ignoreCamera(self): @@ -504,11 +536,15 @@ def solveOrbit(self): log.info('Using ECSV files:') ecsv_names = [] ecsv_paths = [] + os.makedirs(os.path.join(self.dir_path,'ecsvs'), exist_ok=True) for entry in sorted(os.walk(self.dir_path), key=lambda x: x[0]): dir_name, _, file_names = entry + if 'ecsvs' in dir_name: + continue for fn in file_names: + if fn.lower().endswith(".ecsv"): + shutil.copyfile(os.path.join(dir_name, fn), os.path.join(self.dir_path, 'ecsvs', fn)) if fn.lower().endswith(".ecsv") and 'REJECT' not in dir_name.upper() and 'REJECT' not in fn.upper(): - # Add ECSV file, but skip duplicates if fn not in ecsv_names: ecsv_paths.append(os.path.join(dir_name, fn)) @@ -540,6 +576,7 @@ def viewSolution(self): if not self.soln_outputdir: solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') if len(solndir) == 0: + tkMessageBox.showinfo('Warning', 'No solution to review') return solndir = os.path.join(self.dir_path, solndir[0]) self.soln_outputdir = solndir @@ -553,6 +590,7 @@ def removeSolution(self): if not self.soln_outputdir: solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*') if len(solndir) == 0: + tkMessageBox.showinfo('Warning', 'No solution to remove') return solndir = os.path.join(self.dir_path, solndir[0]) self.soln_outputdir = solndir @@ -565,17 +603,24 @@ def uploadOrbit(self): pickles=[] for path, _, files in os.walk(self.dir_path): for name in files: - if '.pickle' in name and '_mc_' not in name: + if '.pickle' in name and '_mc_' not in name and 'tmpzip' not in path: + log.info(f'adding pickle {name}') pickles.append(os.path.join(path, name)) + if name.lower().endswith(".ecsv") and 'ecsvs' not in path: + log.info(f'copying {name}') + shutil.copyfile(os.path.join(path, name), os.path.join(self.dir_path, 'ecsvs', name)) if len(pickles) == 0: return - elif len(pickles) == 1: + pickles = list(set(pickles)) + if len(pickles) == 1: pickfile = pickles[0] else: pickfile = tkFileDialog.askopenfilename(title='Select Orbit Pickle', defaultextension='*.pickle', initialdir=self.dir_path, initialfile='*.pickle', filetypes=[('pickles','*.pickle')]) + if not pickfile: + return orbname = os.path.split(pickfile)[1] tmpdir = os.path.join(self.dir_path, 'tmpzip') os.makedirs(tmpdir, exist_ok=True) @@ -584,12 +629,13 @@ def uploadOrbit(self): shutil.copytree(os.path.join(self.dir_path, 'jpgs'), os.path.join(tmpdir, 'jpgs'), dirs_exist_ok=True) if os.path.isdir(os.path.join(self.dir_path, 'mp4s')): shutil.copytree(os.path.join(self.dir_path, 'mp4s'), os.path.join(tmpdir, 'mp4s'), dirs_exist_ok=True) + if os.path.isdir(os.path.join(self.dir_path, 'ecsvs')): + shutil.copytree(os.path.join(self.dir_path, 'ecsvs'), os.path.join(tmpdir, 'ecsvs'), dirs_exist_ok=True) for path, _, files in os.walk(self.dir_path): for name in files: if '_dyn_mass_fit' in name: im = Image.open(os.path.join(path, name)).convert("RGB") im.save(os.path.join(tmpdir,'jpgs', name[:-4] + '.jpg')) - break zfname = os.path.join(self.dir_path, orbname[:15]) shutil.make_archive(zfname,'zip',tmpdir) try: @@ -660,6 +706,13 @@ def getOneEcsv(self, current_image): for li in lis: if 'issue getting data' in li: return False + os.makedirs(os.path.join(self.dir_path,'ecsvs'), exist_ok=True) + ecsvfs = glob.glob1(os.path.join(self.dir_path, statid), '*.ecsv') + for ecsv in ecsvfs: + shutil.copyfile(os.path.join(self.dir_path, statid, ecsv), os.path.join(self.dir_path, 'ecsvs', ecsv)) + + shutil.copyfile() + ## finish here return True except Exception: return False @@ -700,10 +753,12 @@ def update_listbox(self, bin_list): def checkStacks(self): self.review_stack = True bin_list = self.get_bin_list() - for b in bin_list: - self.selected[b] = (0, '') - self.update_listbox(bin_list) - + if len(bin_list) > 0: + for b in bin_list: + self.selected[b] = (0, '') + self.update_listbox(bin_list) + else: + tkMessageBox.showinfo('Warning', 'No stacks to review') def loadFolder(self): self.review_stack = False @@ -926,15 +981,6 @@ def viewWatchlist(self): else: self.putWatchlist() - def showConfig(self): - if platform.system() == 'Darwin': # macOS - procid = subprocess.Popen(('open', config_file)) - elif platform.system() == 'Windows': # Windows - procid = subprocess.Popen(('cmd','/c',config_file)) - else: # linux variants - procid = subprocess.Popen(('xdg-open', config_file)) - procid.wait() - def getWatchlist(self): k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key)) c = paramiko.SSHClient() @@ -1054,8 +1100,13 @@ def getGMNData(self): parser.add_argument("-d", "--datepatt", type=str, help="date pattern to retrieve") args = parser.parse_args() - dir_ = os.path.dirname(os.path.realpath(__file__)) + dir_ = os.getcwd() config_file = os.path.join(dir_, 'config.ini') + if not os.path.isfile(config_file): + shutil.copyfile(os.path.join(dir_, 'config.ini.sample'), config_file) + tkMessageBox.showinfo("Config Missing", 'Please configure before using') + showConfig() + noimg_file = os.path.join(dir_, 'noimage.jpg') log = logging.getLogger(__name__) From 19dbad5a39e0af43b3e164698ca456fc44cde1cb Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 18:42:55 +0100 Subject: [PATCH 71/83] new build script --- .github/workflows/build_fbanalyser.yml | 83 ++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 .github/workflows/build_fbanalyser.yml diff --git a/.github/workflows/build_fbanalyser.yml b/.github/workflows/build_fbanalyser.yml new file mode 100644 index 00000000..6c312f09 --- /dev/null +++ b/.github/workflows/build_fbanalyser.yml @@ -0,0 +1,83 @@ +name: build_usermgmt +env: + AWS_REGION: eu-west-2 +permissions: + id-token: write + contents: read + +on: + push: + branches: [ dev, markmac99, master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] +jobs: + local_tests: + name: local test stage + runs-on: win10 + steps: + - name: "☁️ checkout repository" + uses: actions/checkout@v4 + - name: testing + run: | + echo "hello from test" + conda activate fbcollector + dir fbCollector + build: + name: build stage + runs-on: win10 + needs: local_tests + steps: + - name: building + run: | + echo "running build step" + conda activate fbcollector + cd fbCollector + pyinstaller ./fireballCollector.py --onefile --windowed --icon .\ukmda.ico + package: + name: package app + runs-on: win10 + needs: + - local_tests + - build + steps: + - name: packaging + run: | + echo "packaging the app now" + cd usermgmt\windows\dist + copy ../ukmda.ico . + copy ../config.ini.sample . + copy ../download_events.sh . + copy ../noimage.jpg . + copy ../README.md . + compress-archive -path . -update -destinationpath c:\temp\fbcollector.zip + release: + name: release package + runs-on: win10 + needs: + - local_tests + - build + - package + steps: + - name: release + uses: xresloader/upload-to-github-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.MM_PAT }} + with: + file: c:\temp\fbcollector.zip + #tags: true + draft: true + overwrite: true + tag_name: 2025.10.1 + default_release_name: Fireball Data Gathering and Analysis + default_release_body_path: fbCollector\README.md + cleanup: + name: clean up package + runs-on: win10 + needs: + - build + - release + steps: + - name: delete package + run: | + c:\temp\fbcollector.zip From 7a592a83877e63c9debef6d476160ce7e86a2c75 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 18:43:49 +0100 Subject: [PATCH 72/83] rename build script --- .github/workflows/build_fbanalyser.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_fbanalyser.yml b/.github/workflows/build_fbanalyser.yml index 6c312f09..4022c0d9 100644 --- a/.github/workflows/build_fbanalyser.yml +++ b/.github/workflows/build_fbanalyser.yml @@ -1,4 +1,4 @@ -name: build_usermgmt +name: build_fbmanagement env: AWS_REGION: eu-west-2 permissions: From 8e7cc8a2448521d799b543d2b7cb582db0c52c23 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 18:57:20 +0100 Subject: [PATCH 73/83] bugs in build scripts --- .github/workflows/build_fbanalyser.yml | 6 +++--- .github/workflows/build_usermgmt.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_fbanalyser.yml b/.github/workflows/build_fbanalyser.yml index 4022c0d9..85e602bb 100644 --- a/.github/workflows/build_fbanalyser.yml +++ b/.github/workflows/build_fbanalyser.yml @@ -44,7 +44,7 @@ jobs: - name: packaging run: | echo "packaging the app now" - cd usermgmt\windows\dist + cd fbCollector\dist copy ../ukmda.ico . copy ../config.ini.sample . copy ../download_events.sh . @@ -65,7 +65,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.MM_PAT }} with: file: c:\temp\fbcollector.zip - #tags: true + tags: true draft: true overwrite: true tag_name: 2025.10.1 @@ -80,4 +80,4 @@ jobs: steps: - name: delete package run: | - c:\temp\fbcollector.zip + del c:\temp\fbcollector.zip diff --git a/.github/workflows/build_usermgmt.yml b/.github/workflows/build_usermgmt.yml index 14a6b1ba..33781eaa 100644 --- a/.github/workflows/build_usermgmt.yml +++ b/.github/workflows/build_usermgmt.yml @@ -78,4 +78,4 @@ jobs: steps: - name: delete package run: | - c:\temp\ukmon_usermgmt.zip + del c:\temp\ukmon_usermgmt.zip From a27ef95fad99555042dd6eaa229ba485516268f9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 19:17:05 +0100 Subject: [PATCH 74/83] Tidy up actions scripts --- .github/workflows/automated-testing.yml | 2 +- .github/workflows/build_fbanalyser.yml | 83 -------------- .github/workflows/build_tools.yml | 139 ++++++++++++++++++++++++ .github/workflows/build_usermgmt.yml | 81 -------------- 4 files changed, 140 insertions(+), 165 deletions(-) delete mode 100644 .github/workflows/build_fbanalyser.yml create mode 100644 .github/workflows/build_tools.yml delete mode 100644 .github/workflows/build_usermgmt.yml diff --git a/.github/workflows/automated-testing.yml b/.github/workflows/automated-testing.yml index a193bdca..73a1dd96 100644 --- a/.github/workflows/automated-testing.yml +++ b/.github/workflows/automated-testing.yml @@ -1,4 +1,4 @@ -name: automated_testing +name: Automated Testing env: AWS_REGION: eu-west-2 permissions: diff --git a/.github/workflows/build_fbanalyser.yml b/.github/workflows/build_fbanalyser.yml deleted file mode 100644 index 85e602bb..00000000 --- a/.github/workflows/build_fbanalyser.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: build_fbmanagement -env: - AWS_REGION: eu-west-2 -permissions: - id-token: write - contents: read - -on: - push: - branches: [ dev, markmac99, master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] -jobs: - local_tests: - name: local test stage - runs-on: win10 - steps: - - name: "☁️ checkout repository" - uses: actions/checkout@v4 - - name: testing - run: | - echo "hello from test" - conda activate fbcollector - dir fbCollector - build: - name: build stage - runs-on: win10 - needs: local_tests - steps: - - name: building - run: | - echo "running build step" - conda activate fbcollector - cd fbCollector - pyinstaller ./fireballCollector.py --onefile --windowed --icon .\ukmda.ico - package: - name: package app - runs-on: win10 - needs: - - local_tests - - build - steps: - - name: packaging - run: | - echo "packaging the app now" - cd fbCollector\dist - copy ../ukmda.ico . - copy ../config.ini.sample . - copy ../download_events.sh . - copy ../noimage.jpg . - copy ../README.md . - compress-archive -path . -update -destinationpath c:\temp\fbcollector.zip - release: - name: release package - runs-on: win10 - needs: - - local_tests - - build - - package - steps: - - name: release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.MM_PAT }} - with: - file: c:\temp\fbcollector.zip - tags: true - draft: true - overwrite: true - tag_name: 2025.10.1 - default_release_name: Fireball Data Gathering and Analysis - default_release_body_path: fbCollector\README.md - cleanup: - name: clean up package - runs-on: win10 - needs: - - build - - release - steps: - - name: delete package - run: | - del c:\temp\fbcollector.zip diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml new file mode 100644 index 00000000..817aa540 --- /dev/null +++ b/.github/workflows/build_tools.yml @@ -0,0 +1,139 @@ +name: Build Fireball and Management Tools +env: + AWS_REGION: eu-west-2 +permissions: + id-token: write + contents: read + +on: + push: + branches: [ dev, markmac99, master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] +jobs: + local_tests: + name: local test stage + runs-on: win10 + steps: + - name: "☁️ checkout repository" + uses: actions/checkout@v4 + - name: testing + run: | + echo "hello from test" + conda activate ukmon-admin + dir + build_usermgmt: + name: build stage + runs-on: win10 + needs: local_tests + steps: + - name: building + run: | + echo "running build step" + conda activate ukmon-admin + cd usermgmt\windows + pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico + dir dist + package_usermgmt: + name: package app + runs-on: win10 + needs: + - local_tests + - build_usermgmt + steps: + - name: packaging + run: | + echo "packaging the app now" + cd usermgmt\windows\dist + copy ../camera.ico . + ((get-content ..\stationmaint.cfg) -replace "adm_mark","") -replace "~/.ssh/ukmonhelper","" | set-content -path ./stationmaint.cfg + compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip + release_usermgmt: + name: release package + runs-on: win10 + needs: + - local_tests + - build_usermgmt + - package_usermgmt + steps: + - name: release + uses: xresloader/upload-to-github-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.MM_PAT }} + with: + file: c:\temp\ukmon_usermgmt.zip + tags: true + draft: true + overwrite: true + tag_name: 2025.10.1 + default_release_name: User Management Tool + default_release_body_path: usermgmt\windows\release_notes.md + cleanup_usermgmt: + name: clean up package + runs-on: win10 + needs: + - build_usermgmt + - release_usermgmt + steps: + - name: delete temp usermgmt package + run: | + del c:\temp\ukmon_usermgmt.zip + build_fbtool: + name: build stage + runs-on: win10 + needs: local_tests + steps: + - name: building + run: | + echo "running build step" + conda activate fbcollector + cd fbCollector + pyinstaller ./fireballCollector.py --onefile --windowed --icon .\ukmda.ico + package_fbtool: + name: package app + runs-on: win10 + needs: + - local_tests + - build_fbtool + steps: + - name: packaging + run: | + echo "packaging the app now" + cd fbCollector\dist + copy ../ukmda.ico . + copy ../config.ini.sample . + copy ../download_events.sh . + copy ../noimage.jpg . + copy ../README.md . + compress-archive -path . -update -destinationpath c:\temp\fbcollector.zip + release_fbtool: + name: release package + runs-on: win10 + needs: + - local_tests + - build_fbtool + - package_fbtool + steps: + - name: release + uses: xresloader/upload-to-github-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.MM_PAT }} + with: + file: c:\temp\fbcollector.zip + tags: true + draft: true + overwrite: true + tag_name: 2025.10.1 + default_release_name: Fireball Data Gathering and Analysis + default_release_body_path: fbCollector\README.md + cleanup_fbtool: + name: clean up temp package + runs-on: win10 + needs: + - build_fbtool + - release_fbtool + steps: + - name: delete package + run: | + del c:\temp\fbcollector.zip diff --git a/.github/workflows/build_usermgmt.yml b/.github/workflows/build_usermgmt.yml deleted file mode 100644 index 33781eaa..00000000 --- a/.github/workflows/build_usermgmt.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: build_usermgmt -env: - AWS_REGION: eu-west-2 -permissions: - id-token: write - contents: read - -on: - push: - branches: [ dev, markmac99, master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] -jobs: - local_tests: - name: local test stage - runs-on: win10 - steps: - - name: "☁️ checkout repository" - uses: actions/checkout@v4 - - name: testing - run: | - echo "hello from test" - conda activate ukmon-admin - dir usermgmt\windows - build: - name: build stage - runs-on: win10 - needs: local_tests - steps: - - name: building - run: | - echo "running build step" - conda activate ukmon-admin - cd usermgmt\windows - pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico - dir dist - package: - name: package app - runs-on: win10 - needs: - - local_tests - - build - steps: - - name: packaging - run: | - echo "packaging the app now" - cd usermgmt\windows\dist - copy ../camera.ico . - ((get-content ..\stationmaint.cfg) -replace "adm_mark","") -replace "~/.ssh/ukmonhelper","" | set-content -path ./stationmaint.cfg - compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip - release: - name: release package - runs-on: win10 - needs: - - local_tests - - build - - package - steps: - - name: release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.MM_PAT }} - with: - file: c:\temp\ukmon_usermgmt.zip - #tags: true - draft: true - overwrite: true - tag_name: 2025.10.1 - default_release_name: User Management Tool - default_release_body_path: usermgmt\windows\release_notes.md - cleanup: - name: clean up package - runs-on: win10 - needs: - - build - - release - steps: - - name: delete package - run: | - del c:\temp\ukmon_usermgmt.zip From eab89e3ddf48e085fc023380397415e78c02abef Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 19:32:35 +0100 Subject: [PATCH 75/83] minor renaming of jobs in the build script --- .github/workflows/build_tools.yml | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml index 817aa540..9c270366 100644 --- a/.github/workflows/build_tools.yml +++ b/.github/workflows/build_tools.yml @@ -24,7 +24,7 @@ jobs: conda activate ukmon-admin dir build_usermgmt: - name: build stage + name: build usermanagement runs-on: win10 needs: local_tests steps: @@ -36,10 +36,9 @@ jobs: pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico dir dist package_usermgmt: - name: package app + name: package usermanagement runs-on: win10 needs: - - local_tests - build_usermgmt steps: - name: packaging @@ -50,11 +49,9 @@ jobs: ((get-content ..\stationmaint.cfg) -replace "adm_mark","") -replace "~/.ssh/ukmonhelper","" | set-content -path ./stationmaint.cfg compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip release_usermgmt: - name: release package + name: release usermanagement runs-on: win10 needs: - - local_tests - - build_usermgmt - package_usermgmt steps: - name: release @@ -70,17 +67,16 @@ jobs: default_release_name: User Management Tool default_release_body_path: usermgmt\windows\release_notes.md cleanup_usermgmt: - name: clean up package + name: clean up temp usermanagement files runs-on: win10 needs: - - build_usermgmt - release_usermgmt steps: - name: delete temp usermgmt package run: | del c:\temp\ukmon_usermgmt.zip build_fbtool: - name: build stage + name: build fbcollector runs-on: win10 needs: local_tests steps: @@ -91,10 +87,9 @@ jobs: cd fbCollector pyinstaller ./fireballCollector.py --onefile --windowed --icon .\ukmda.ico package_fbtool: - name: package app + name: package fbcollector runs-on: win10 needs: - - local_tests - build_fbtool steps: - name: packaging @@ -108,11 +103,9 @@ jobs: copy ../README.md . compress-archive -path . -update -destinationpath c:\temp\fbcollector.zip release_fbtool: - name: release package + name: release fbcollector runs-on: win10 needs: - - local_tests - - build_fbtool - package_fbtool steps: - name: release @@ -125,13 +118,12 @@ jobs: draft: true overwrite: true tag_name: 2025.10.1 - default_release_name: Fireball Data Gathering and Analysis + default_release_name: Tools to Gather Fireball data and Manage UKMON Cameras default_release_body_path: fbCollector\README.md cleanup_fbtool: - name: clean up temp package + name: clean up temp fbcollector files runs-on: win10 needs: - - build_fbtool - release_fbtool steps: - name: delete package From 1648948f42726f4175bcaec2fefdd1bcdddbd98f Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 22:33:48 +0100 Subject: [PATCH 76/83] manage config and local sync better --- .github/workflows/build_tools.yml | 2 +- .gitignore | 3 ++- usermgmt/windows/stationMaint2.py | 27 ++++++++++++++----- ...ationmaint.cfg => stationmaint.ini.sample} | 4 +-- 4 files changed, 26 insertions(+), 10 deletions(-) rename usermgmt/windows/{stationmaint.cfg => stationmaint.ini.sample} (75%) diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml index 9c270366..2877b277 100644 --- a/.github/workflows/build_tools.yml +++ b/.github/workflows/build_tools.yml @@ -46,7 +46,7 @@ jobs: echo "packaging the app now" cd usermgmt\windows\dist copy ../camera.ico . - ((get-content ..\stationmaint.cfg) -replace "adm_mark","") -replace "~/.ssh/ukmonhelper","" | set-content -path ./stationmaint.cfg + copy ../stationmaint.cfg . compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip release_usermgmt: name: release usermanagement diff --git a/.gitignore b/.gitignore index 3c3e770e..c1ab2368 100644 --- a/.gitignore +++ b/.gitignore @@ -614,4 +614,5 @@ tests/testing/RMS/ tests/testing/WesternMeteorPyLib/ archive/ukmon_pylib/tests/testdata.tar.gz servercopybkp/* -fbcollector/config.ini \ No newline at end of file +fbcollector/config.ini +usermgmt/windows/stationmaint.ini diff --git a/usermgmt/windows/stationMaint2.py b/usermgmt/windows/stationMaint2.py index cf6a81d3..7ddae0c7 100644 --- a/usermgmt/windows/stationMaint2.py +++ b/usermgmt/windows/stationMaint2.py @@ -21,18 +21,30 @@ from configparser import ConfigParser import logging import logging.handlers +import shutil +import platform +import subprocess log = logging.getLogger("logger") +config_file = '' + def loadConfig(cfgdir): - cfgfile = os.path.join(cfgdir, 'stationmaint.cfg') + config_file = os.path.join(cfgdir, 'stationmaint.ini') + if not os.path.isfile(config_file): + tkMessageBox.showinfo("Config Missing", 'Please configure before using') + shutil.copyfile(f'{config_file}.sample', config_file) + if platform.system() == 'Darwin': # macOS + procid = subprocess.Popen(('open', config_file)) + elif platform.system() == 'Windows': # Windows + procid = subprocess.Popen(('cmd','/c',config_file)) + else: # linux variants + procid = subprocess.Popen(('xdg-open', config_file)) + procid.wait() cfg = ConfigParser() - if not os.path.isfile(cfgfile): - tkMessageBox.showinfo('Warning', f'config file {cfgfile} not found') - exit(0) - cfg.read(cfgfile) + cfg.read(config_file) server = cfg['helper']['helperip'] user = cfg['helper']['user'] keyfile = cfg['helper']['sshkey'] @@ -306,7 +318,6 @@ def __init__(self, parent, cfgdir): os.makedirs('users', exist_ok=True) os.makedirs('inifs', exist_ok=True) os.makedirs('sshkeys', exist_ok=True) - self.resyncLocalFiles() self.ddb = self.conn.resource('dynamodb', region_name='eu-west-2') try: @@ -404,6 +415,10 @@ def __init__(self, parent, cfgdir): ("end_edit_cell", self.end_edit_cell)]) self.sheet.popup_menu_add_command('Sort', self.columns_sort, table_menu = False, index_menu = False) + self.parent.title('Please wait, syncing data...') + self.resyncLocalFiles() + self.parent.title('Station Maintenance') + return def hide_columns_right_click(self, event = None): currently_displayed = self.sheet.display_columns() diff --git a/usermgmt/windows/stationmaint.cfg b/usermgmt/windows/stationmaint.ini.sample similarity index 75% rename from usermgmt/windows/stationmaint.cfg rename to usermgmt/windows/stationmaint.ini.sample index 210eb98d..0a6b9934 100644 --- a/usermgmt/windows/stationmaint.cfg +++ b/usermgmt/windows/stationmaint.ini.sample @@ -1,6 +1,6 @@ [helper] -USER=adm_mark -SSHKEY=~/.ssh/ukmonhelper +USER= +SSHKEY= HELPERIP=3.11.55.160 REMOTEDIR=/home/ec2-user/prod/data PLATEPARDIR= From dce0fde36ec52a9774e1b573e16b78933197ff08 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 22:51:12 +0100 Subject: [PATCH 77/83] typo in the build script --- .github/workflows/build_tools.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml index 2877b277..cc0eba8f 100644 --- a/.github/workflows/build_tools.yml +++ b/.github/workflows/build_tools.yml @@ -46,7 +46,7 @@ jobs: echo "packaging the app now" cd usermgmt\windows\dist copy ../camera.ico . - copy ../stationmaint.cfg . + copy ../stationmaint.ini.sample . compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip release_usermgmt: name: release usermanagement From a214ca2c33685cf4280ac00c65cc8f0aa749ea24 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 23:21:33 +0100 Subject: [PATCH 78/83] remove unused line ni cpp config --- .vscode/c_cpp_properties.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index c0662c55..1e99cf94 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -5,8 +5,7 @@ "includePath": [ "${workspaceFolder}/**", "${vcpkgRoot}/x64-windows/include", - "${vcpkgRoot}/x86-windows/include", - "e:/dev/opencv3/opencv-3.4.11/build/install/include" + "${vcpkgRoot}/x86-windows/include" ], "defines": [ "_DEBUG", From 10eaeab6b971229118f2d20f3a33e07c020a3bf9 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Sun, 12 Oct 2025 23:22:44 +0100 Subject: [PATCH 79/83] tidy up yaml files --- .github/workflows/automated-testing.yml | 8 +- .github/workflows/build_tools.yml | 250 ++++++++++++------------ .github/workflows/codeql-analysis.yml | 58 +++--- 3 files changed, 158 insertions(+), 158 deletions(-) diff --git a/.github/workflows/automated-testing.yml b/.github/workflows/automated-testing.yml index 73a1dd96..5b3f625a 100644 --- a/.github/workflows/automated-testing.yml +++ b/.github/workflows/automated-testing.yml @@ -7,10 +7,10 @@ permissions: on: push: - branches: [ dev, markmac99 ] + branches: [dev, markmac99] pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: [master] jobs: conttests: runs-on: thelinux @@ -20,7 +20,7 @@ jobs: username: ${{ secrets.DH_LOGIN_MM }} password: ${{ secrets.DH_PATT_MM }} env: - BRANCH: ${{ github.head_ref || github.ref_name }} + BRANCH: ${{ github.head_ref || github.ref_name }} volumes: - testdata:/data steps: @@ -46,7 +46,7 @@ jobs: apitests: runs-on: ubuntu-latest steps: - - name: checkout code + - name: checkout code uses: actions/checkout@v3 - name: run api tests run: | diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml index cc0eba8f..70a3d6a5 100644 --- a/.github/workflows/build_tools.yml +++ b/.github/workflows/build_tools.yml @@ -1,131 +1,131 @@ name: Build Fireball and Management Tools env: - AWS_REGION: eu-west-2 + AWS_REGION: eu-west-2 permissions: - id-token: write - contents: read + id-token: write + contents: read on: - push: - branches: [ dev, markmac99, master ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ master ] + push: + branches: [dev, markmac99, master] + pull_request: + # The branches below must be a subset of the branches above + branches: [master] jobs: - local_tests: - name: local test stage - runs-on: win10 - steps: - - name: "☁️ checkout repository" - uses: actions/checkout@v4 - - name: testing - run: | - echo "hello from test" - conda activate ukmon-admin - dir - build_usermgmt: - name: build usermanagement - runs-on: win10 - needs: local_tests - steps: - - name: building - run: | - echo "running build step" - conda activate ukmon-admin - cd usermgmt\windows - pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico - dir dist - package_usermgmt: - name: package usermanagement - runs-on: win10 - needs: - - build_usermgmt - steps: - - name: packaging - run: | - echo "packaging the app now" - cd usermgmt\windows\dist - copy ../camera.ico . - copy ../stationmaint.ini.sample . - compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip - release_usermgmt: - name: release usermanagement - runs-on: win10 - needs: - - package_usermgmt - steps: - - name: release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.MM_PAT }} - with: - file: c:\temp\ukmon_usermgmt.zip - tags: true - draft: true - overwrite: true - tag_name: 2025.10.1 - default_release_name: User Management Tool - default_release_body_path: usermgmt\windows\release_notes.md - cleanup_usermgmt: - name: clean up temp usermanagement files - runs-on: win10 - needs: - - release_usermgmt - steps: - - name: delete temp usermgmt package - run: | - del c:\temp\ukmon_usermgmt.zip - build_fbtool: - name: build fbcollector - runs-on: win10 - needs: local_tests - steps: - - name: building - run: | - echo "running build step" - conda activate fbcollector - cd fbCollector - pyinstaller ./fireballCollector.py --onefile --windowed --icon .\ukmda.ico - package_fbtool: - name: package fbcollector - runs-on: win10 - needs: - - build_fbtool - steps: - - name: packaging - run: | - echo "packaging the app now" - cd fbCollector\dist - copy ../ukmda.ico . - copy ../config.ini.sample . - copy ../download_events.sh . - copy ../noimage.jpg . - copy ../README.md . - compress-archive -path . -update -destinationpath c:\temp\fbcollector.zip - release_fbtool: - name: release fbcollector - runs-on: win10 - needs: - - package_fbtool - steps: - - name: release - uses: xresloader/upload-to-github-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.MM_PAT }} - with: - file: c:\temp\fbcollector.zip - tags: true - draft: true - overwrite: true - tag_name: 2025.10.1 - default_release_name: Tools to Gather Fireball data and Manage UKMON Cameras - default_release_body_path: fbCollector\README.md - cleanup_fbtool: - name: clean up temp fbcollector files - runs-on: win10 - needs: - - release_fbtool - steps: - - name: delete package - run: | - del c:\temp\fbcollector.zip + local_tests: + name: local test stage + runs-on: win10 + steps: + - name: "☁️ checkout repository" + uses: actions/checkout@v4 + - name: testing + run: | + echo "hello from test" + conda activate ukmon-admin + dir + build_usermgmt: + name: build usermanagement + runs-on: win10 + needs: local_tests + steps: + - name: building + run: | + echo "running build step" + conda activate ukmon-admin + cd usermgmt\windows + pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico + dir dist + package_usermgmt: + name: package usermanagement + runs-on: win10 + needs: + - build_usermgmt + steps: + - name: packaging + run: | + echo "packaging the app now" + cd usermgmt\windows\dist + copy ../camera.ico . + copy ../stationmaint.ini.sample . + compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip + release_usermgmt: + name: release usermanagement + runs-on: win10 + needs: + - package_usermgmt + steps: + - name: release + uses: xresloader/upload-to-github-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.MM_PAT }} + with: + file: c:\temp\ukmon_usermgmt.zip + tags: true + draft: true + overwrite: true + tag_name: 2025.10.1 + default_release_name: User Management Tool + default_release_body_path: usermgmt\windows\release_notes.md + cleanup_usermgmt: + name: clean up temp usermanagement files + runs-on: win10 + needs: + - release_usermgmt + steps: + - name: delete temp usermgmt package + run: | + del c:\temp\ukmon_usermgmt.zip + build_fbtool: + name: build fbcollector + runs-on: win10 + needs: local_tests + steps: + - name: building + run: | + echo "running build step" + conda activate fbcollector + cd fbCollector + pyinstaller ./fireballCollector.py --onefile --windowed --icon .\ukmda.ico + package_fbtool: + name: package fbcollector + runs-on: win10 + needs: + - build_fbtool + steps: + - name: packaging + run: | + echo "packaging the app now" + cd fbCollector\dist + copy ../ukmda.ico . + copy ../config.ini.sample . + copy ../download_events.sh . + copy ../noimage.jpg . + copy ../README.md . + compress-archive -path . -update -destinationpath c:\temp\fbcollector.zip + release_fbtool: + name: release fbcollector + runs-on: win10 + needs: + - package_fbtool + steps: + - name: release + uses: xresloader/upload-to-github-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.MM_PAT }} + with: + file: c:\temp\fbcollector.zip + tags: true + draft: true + overwrite: true + tag_name: 2025.10.1 + default_release_name: Tools to Gather Fireball data and Manage UKMON Cameras + default_release_body_path: fbCollector\README.md + cleanup_fbtool: + name: clean up temp fbcollector files + runs-on: win10 + needs: + - release_fbtool + steps: + - name: delete package + run: | + del c:\temp\fbcollector.zip diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 86dbe2e6..77e99e8a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,12 +13,12 @@ name: "CodeQL" on: push: - branches: [ master ] + branches: [master] pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: [master] schedule: - - cron: '26 14 * * 6' + - cron: "26 14 * * 6" jobs: analyze: @@ -32,40 +32,40 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'javascript', 'python' ] + language: ["javascript", "python"] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] # Learn more: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout repository + uses: actions/checkout@v3 - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language - #- run: | - # make bootstrap - # make release + #- run: | + # make bootstrap + # make release - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 From f353ece4164d60247eb91cc87f47cd323d42b836 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Oct 2025 11:02:07 +0100 Subject: [PATCH 80/83] fix stationmaint window min width --- usermgmt/windows/stationMaint2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/usermgmt/windows/stationMaint2.py b/usermgmt/windows/stationMaint2.py index 7ddae0c7..88e12f5d 100644 --- a/usermgmt/windows/stationMaint2.py +++ b/usermgmt/windows/stationMaint2.py @@ -955,6 +955,7 @@ def log_timestamp(): log.info("Program start") root = tk.Tk() + root.minsize(400, 100) app = CamMaintenance(root, dir_) root.iconbitmap(os.path.join(dir_,'camera.ico')) root.protocol("WM_DELETE_WINDOW", app.on_closing) From 68a9abaa960c5e2f6c3be91fce0570c3cf5d4652 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Oct 2025 13:15:44 +0100 Subject: [PATCH 81/83] bugfix to find correct config file --- usermgmt/server/updateAwsKey.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/usermgmt/server/updateAwsKey.sh b/usermgmt/server/updateAwsKey.sh index ab53776a..ff508560 100644 --- a/usermgmt/server/updateAwsKey.sh +++ b/usermgmt/server/updateAwsKey.sh @@ -8,7 +8,14 @@ targ=$1 force=$2 here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -source $here/../config.ini >/dev/null 2>&1 +if [ -f ../config.ini ] ; then + source ~/dev/config.ini + keydir=/home/ec2-user/dev/keymgmt +else + source ~/prod/config.ini + keydir=/home/ec2-user/keymgmt + +fi conda activate $HOME/miniconda3/envs/${WMPL_ENV} export AWS_PROFILE=ukmonshared From dc64100ab260957571f58373f2be21eb6fdc386d Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Oct 2025 13:17:00 +0100 Subject: [PATCH 82/83] add functionality to disable a camera properly, and remove redundant code --- usermgmt/windows/stationMaint2.py | 104 +++++++++++++----------------- 1 file changed, 45 insertions(+), 59 deletions(-) diff --git a/usermgmt/windows/stationMaint2.py b/usermgmt/windows/stationMaint2.py index 88e12f5d..576d84fc 100644 --- a/usermgmt/windows/stationMaint2.py +++ b/usermgmt/windows/stationMaint2.py @@ -343,17 +343,14 @@ def __init__(self, parent, cfgdir): camMenu = Menu(self.menuBar, tearoff=0) camMenu.add_command(label = "Add Camera", command = self.addCamera) camMenu.add_command(label = "Relocate Camera", command = self.moveCamera) - camMenu.add_command(label = "Deactivate Camera", command = self.delCamera) camMenu.add_separator() - camMenu.add_command(label = "Remove Location", command = self.delOperator) + camMenu.add_command(label = "Check Camera", command = self.checkLastUpdate) camMenu.add_separator() camMenu.add_command(label = "Download Platepar", command = self.getPlate) - camMenu.add_command(label = "Update platepar", command = self.newPlate) + camMenu.add_command(label = "Update Platepar", command = self.newPlate) camMenu.add_separator() camMenu.add_command(label = "Update SSH Key", command = self.newSSHKey) camMenu.add_command(label = "Update AWS Key", command = self.newAWSKey) - camMenu.add_separator() - camMenu.add_command(label = "Check Camera", command = self.checkLastUpdate) ownMenu = Menu(self.menuBar, tearoff=0) ownMenu.add_command(label = "View Owner Data", command = self.viewOwnerData) @@ -442,6 +439,10 @@ def end_edit_cell(self, event): 'direction': data[2], 'camtype': str(data[3]), 'active': int(data[4]), 'oldcode': data[1], 'created': data[8]} addRow(newdata, ddb=self.ddb) + if int(data[4]) > 1: + self.disableEnableUnixUser(data[0], data[2], False) + else: + self.disableEnableUnixUser(data[0], data[2], True) return event[3] def end_delete_rows(self, event): @@ -504,14 +505,6 @@ def on_closing(self): self.parent.quit() self.parent.destroy() - def delCamera(self): - tk.messagebox.showinfo(title="Information", message='To remove a camera, set Active=current date yyyymmdd') - return - - def delOperator(self): - tk.messagebox.showinfo(title="Information", message='Not implemented yet') - return - def viewOwnerData(self): statOwnerDialog(self) return @@ -708,23 +701,23 @@ def uploadPlatepar(self, camdets, plateparfile): log.info(f'running {command}') _, stdout, stderr = c.exec_command(command, timeout=10) for line in iter(stdout.readline, ""): - log.info(line, end="") + log.info(line) for line in iter(stderr.readline, ""): - log.info(line, end="") + log.info(line) command = f'sudo mv {uplfile} /var/sftp/{camname}/platepar/platepar_cmn2010.cal' log.info(f'running {command}') _, stdout, stderr = c.exec_command(command, timeout=10) for line in iter(stdout.readline, ""): - log.info(line, end="") + log.info(line) for line in iter(stderr.readline, ""): - log.info(line, end="") + log.info(line) command = f'sudo chown {camname}:{camname} /var/sftp/{camname}/platepar/platepar_cmn2010.cal' log.info(f'running {command}') _, stdout, stderr = c.exec_command(command, timeout=10) for line in iter(stdout.readline, ""): - log.info(line, end="") + log.info(line) for line in iter(stderr.readline, ""): - log.info(line, end="") + log.info(line) scpcli.close() c.close() return @@ -777,51 +770,17 @@ def createNewAwsKey(self, location, caminfo): c.connect(hostname = server, username = user, pkey = k) command = f'/home/{user}/keymgmt/updateAwsKey.sh {location} force' log.info(f'running {command}') - _, stdout, stderr = c.exec_command(command, timeout=10) - for line in iter(stdout.readline, ""): - log.info(line, end="") + _, stdout, stderr = c.exec_command(command, timeout=60) for line in iter(stderr.readline, ""): - log.info(line, end="") + log.info(line) + for line in iter(stdout.readline, ""): + log.info(line) + tkMessageBox.showinfo('Updated', line.split(' to')[0]) log.info('done') c.close() return - def updateKeyfile(self, caminfo, location): - server = self.cfg['helper']['platepardir'] - user='ec2-user' - keyf = os.path.join('jsonkeys', location + '.key') - currkey = json.load(open(keyf, 'r')) - archcsvf = os.path.join('csvkeys', location.lower() + '_arch.csv') - with open(archcsvf,'w') as outf: - outf.write('Access key ID,Secret access key\n') - outf.write('{},{}\n'.format(currkey['AccessKey']['AccessKeyId'], currkey['AccessKey']['SecretAccessKey'])) - - affectedcamlist = caminfo[caminfo.site==location] - keyfile = self.cfg['helper']['sshkey'] - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = user, pkey = k) - scpcli = SCPClient(c.get_transport()) - # push the raw keyfile - scpcli.put(keyf, 'keymgmt/rawkeys/live/') - scpcli.put(archcsvf, 'keymgmt/rawkeys/csvkeys/') - scpcli.close() - c.close() - for _, cam in affectedcamlist.iterrows(): - cameraname = cam.site.lower() + '_' + cam.site.lower() - keyfile = os.path.join('sshkeys', cameraname + '.pub') - k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) - c = paramiko.SSHClient() - c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - c.connect(hostname = server, username = cameraname, pkey = k) - scpcli = SCPClient(c.get_transport()) - scpcli.put(archcsvf, '.') - scpcli.close() - c.close() - - def getSSHkey(self, loc, dir): server= self.cfg['helper']['helperip'] user='ec2-user' @@ -889,7 +848,34 @@ def addNewUnixUser(self, location, cameraname, oldcamname='', updatemode=0): scpcli.close() c.close() return - + + def disableEnableUnixUser(self, loc, dir, enable): + cameraname = loc.lower() + '_' + dir.lower() + server = self.cfg['helper']['helperip'] + user='ec2-user' + log.info(f'updating Unix user {cameraname}') + keyfile = self.cfg['helper']['sshkey'] + k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile)) + c = paramiko.SSHClient() + c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + c.connect(hostname = server, username = user, pkey = k) + except Exception: + c.connect(hostname = server+'.', username = user, pkey = k) + if enable is False: + command = f'/usr/bin/sudo /usr/sbin/usermod --expiredate 1 {cameraname}' + else: + command = f'/usr/bin/sudo /usr/sbin/usermod --expiredate "" {cameraname}' + log.info(f'running {command}') + line = '' + _, stdout, stderr = c.exec_command(command, timeout=60) + for line in iter(stderr.readline, ""): + log.info(line) + for line in iter(stdout.readline, ""): + log.info(line) + if enable is False: + tkMessageBox.showinfo('Disabled', f'{cameraname} {line}') + return def createKeyFile(self, location): archbucket = self.cfg['store']['srcbucket'] From 590c2920b4f33ea392a87d1dc87fe4906836fb91 Mon Sep 17 00:00:00 2001 From: Mark McIntyre Date: Mon, 13 Oct 2025 13:37:31 +0100 Subject: [PATCH 83/83] update the Readme file and add documentation links --- .github/workflows/build_tools.yml | 1 + .gitignore | 1 + usermgmt/README.md | 50 +++++++++++++++++-------------- usermgmt/windows/stationMaint2.py | 22 +++++++++++++- 4 files changed, 50 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml index 70a3d6a5..7b2865d3 100644 --- a/.github/workflows/build_tools.yml +++ b/.github/workflows/build_tools.yml @@ -47,6 +47,7 @@ jobs: cd usermgmt\windows\dist copy ../camera.ico . copy ../stationmaint.ini.sample . + copy ../../README.md . compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip release_usermgmt: name: release usermanagement diff --git a/.gitignore b/.gitignore index c1ab2368..6e4b5f8b 100644 --- a/.gitignore +++ b/.gitignore @@ -616,3 +616,4 @@ archive/ukmon_pylib/tests/testdata.tar.gz servercopybkp/* fbcollector/config.ini usermgmt/windows/stationmaint.ini +usermgmt/windows/README.md diff --git a/usermgmt/README.md b/usermgmt/README.md index 4bb751c0..d966a353 100644 --- a/usermgmt/README.md +++ b/usermgmt/README.md @@ -1,6 +1,7 @@ # UKMDA User Management -This folder contains the scripts used to add/maintain new cameras to the network. +This Windows application is used to add/maintain new cameras to the network. Its use is restricted to members of the UKMON +admin team. ## Basic Principles A camera consists of an RMS ID, location and pointing direction. @@ -13,15 +14,18 @@ Each _camera_ is allocated a unique *Unix* ID. The public ssh key provided by th # User Management tool -Unix and AWS User creation and configuration are managed by *stationMaint.ps1* which calls a python programme that uses native AWS and Unix libraries to execute the required commands. The tool requires AWS and SSH credentials which are restricted to administrators (see note below). +Unix and AWS User creation and configuration are managed by *stationMaint.exe* which is a compiled python programme that uses native AWS and Unix libraries to execute the required commands. ## Setup -Prerequisites: -* anaconda or miniconda -* An AWS profile with the usermaintenance IAM permissions. The default profile name is *ukmda_maint*. -* An SSH key thats permissioned to connect to the server. The default key name is *ukmda_admin*. - -To install the app, copy the python files, requirements.txt and powershell script to a folder of your choosing, and update the environment variables in the powershell script as needed. Now run the powershell script which will create a suitable Conda environment, install the requirements and launch the tool. +Reminder: To use the app you will have to be permissioned on our servers by one of the Admin team. + +* Download the latest zip file from [here](https://github.com/ukmda/ukmda-dataprocessing/releases) and expand it to a location of your choosing on your computer, for example `%userprofile%\ukmon\usermgmt` +* Create a desktop shortcut pointing to `dist\stationmaint2.exe` in this folder. + +When you first run the app you will be prompted to provide the credentials we gave you to connect to our server. + +### Linux or MacOS +In principle the code should work on Linux-like OSes. You'd need to copy the python file, icon file, sample ini file, requirements file and this README to a folder of your choosing then create a python virtual environment, activate it and install the requirements. ## Adding a new camera To add a new camera the camera operator must supply the following: @@ -37,28 +41,28 @@ by other cameras at the same location. Once the information has been gathered, select Camera/Add and fill in the boxes. Note that the boxes are part-prepopulated with values from whatever row your cursor was on, so that if you're adding another camera at an existing location, you can save a bit of typing. ## Amending a camera -The location and pointing direction cannot be changed once set. To change these values you'll need to follow the process to move a camera. +To change location or pointing direction follow the process to move a camera. All other values can be amended by selecting the line and updating the values. ## Moving a camera -To move a camera to a new location, select the row containing the camera then select Camera/Move. After you fill in any new information, a new unix and AWS user will be created and the configuration files for the old user will be updated. The old camera must then be marked disabled as explained next. +IMPORTANT NOTE: the camera owner should NOT make any changes - the changes will automatically flow down to their station. -IMPORTANT NOTE: the camera owner should NOT make any changes - the system will automatically update their configuration. - -## Disabling a camera -To disable a camera, change the Active column from 1 to today's date in YYYYMMDD format eg 20220715. This removes the camera from current reporting, but retains the details for any historical reporting. - -# Disabling a location -To disable a location, we revoke the corresponding AWS user's keys, and if necessary, delete the AWS user and Unix ID. No other user will be affected. This is not currently managed by the UI as it is a drastic action that we do not want to carry out accidentally! +To move a camera to a new location, select the row containing the camera then select Camera/Move. After you fill in any new information, a new unix and AWS user will be created and the configuration files for the old user will be updated. The old camera must then be marked disabled as explained next. -# Permissions Needed -This tool requires two sets of permissions: -* an AWS profile with permissions to add/amend/delete users in IAM. This is required to manage the AWS roles. -* an SSH key with permissions to login to the Batch server. This is required to manage the Unix accounts. +## Disabling and Re-enabling a camera +To disable a camera, change the Active column from 1 to the last active date in YYYYMMDD format eg 20220715. This removes the camera from current reporting and disables the unix user, but retains the details for any historical reporting. A camera can be reactivated by setting the Active column back to 1. -# Parameters -The powershell script lists the parameters that need to be set, such as AWS profile to use, the buckets and so on. The ansible YAML script sets these values. +## Disabling a location +Disable all cameras at that location, and then update the AWS key. We never remove a location as there would be a risk of losing historic data. +## Other Functionality +The tool also provides some other functionality: +* Owners menu - search for station details and owner details. +* Camera menu + * Check camera - this does a quick sanity test to see when the camera last connected to our server. + * Download the current plateper for a camera and upload a new one. The new plate will be automaticlly installed on the camera the following morning. + * Update the SSH key - if a user sends a new SSH key, we need to add it to the server. + * Update the AWS key - force an update of the location's AWS key. Note that keys are autorolled every 60 days so this would only be needed if the key had been compromised. # Copyright All code Copyright (C) 2018-2023 Mark McIntyre \ No newline at end of file diff --git a/usermgmt/windows/stationMaint2.py b/usermgmt/windows/stationMaint2.py index 576d84fc..90db6005 100644 --- a/usermgmt/windows/stationMaint2.py +++ b/usermgmt/windows/stationMaint2.py @@ -309,6 +309,7 @@ def __init__(self, parent, cfgdir): self.parent = parent Frame.__init__(self, parent) + self.config_dir = cfgdir self.cfg, awskeys = loadConfig(cfgdir) self.conn = boto3.Session(aws_access_key_id=awskeys['key'], aws_secret_access_key=awskeys['secret']) self.bucket_name = self.cfg['store']['srcbucket'] @@ -356,9 +357,14 @@ def __init__(self, parent, cfgdir): ownMenu.add_command(label = "View Owner Data", command = self.viewOwnerData) ownMenu.add_command(label = "Search Data", command = self.searchOwnerData) + helpMenu = Menu(self.menuBar, tearoff=0) + helpMenu.add_command(label = "Documentation", command = self.viewReadme) + helpMenu.add_command(label = "About", command = self.aboutBox) + self.menuBar.add_cascade(label="File", underline=0, menu=fileMenu) self.menuBar.add_cascade(label="Camera", underline=0, menu=camMenu) self.menuBar.add_cascade(label="Owners", underline=0, menu=ownMenu) + self.menuBar.add_cascade(label="Help", underline=0, menu=helpMenu) parent.grid_columnconfigure(0, weight = 1) parent.grid_rowconfigure(0, weight = 1) @@ -908,6 +914,20 @@ def createIniFile(self, cameraname): outf.write('export UKMONKEY=~/.ssh/ukmon\n') outf.write('export RMSCFG=~/source/RMS/.config\n') return + + def viewReadme(self): + docfile = os.path.join(self.config_dir, 'README.md') + if platform.system() == 'Darwin': # macOS + procid = subprocess.Popen(('open', docfile)) + elif platform.system() == 'Windows': # Windows + procid = subprocess.Popen(('cmd','/c',f'notepad {docfile}')) + else: # linux variants + procid = subprocess.Popen(('xdg-open', docfile)) + procid.wait() + + def aboutBox(self): + tkMessageBox.showinfo('About', + 'UKMON user / camera maintenance tool.\nAll rights reserved, Mark McIntyre, 2024') def log_timestamp(): @@ -918,7 +938,7 @@ def log_timestamp(): if __name__ == '__main__': # Initialize main window - dir_ = os.getcwd() #os.path.dirname(os.path.realpath(__file__)) + dir_ = os.getcwd() log.setLevel(logging.INFO)