diff --git a/.github/workflows/automated-testing.yml b/.github/workflows/automated-testing.yml
index a193bdca9..5b3f625ad 100644
--- a/.github/workflows/automated-testing.yml
+++ b/.github/workflows/automated-testing.yml
@@ -1,4 +1,4 @@
-name: automated_testing
+name: Automated Testing
env:
AWS_REGION: eu-west-2
permissions:
@@ -7,10 +7,10 @@ permissions:
on:
push:
- branches: [ dev, markmac99 ]
+ branches: [dev, markmac99]
pull_request:
# The branches below must be a subset of the branches above
- branches: [ master ]
+ branches: [master]
jobs:
conttests:
runs-on: thelinux
@@ -20,7 +20,7 @@ jobs:
username: ${{ secrets.DH_LOGIN_MM }}
password: ${{ secrets.DH_PATT_MM }}
env:
- BRANCH: ${{ github.head_ref || github.ref_name }}
+ BRANCH: ${{ github.head_ref || github.ref_name }}
volumes:
- testdata:/data
steps:
@@ -46,7 +46,7 @@ jobs:
apitests:
runs-on: ubuntu-latest
steps:
- - name: checkout code
+ - name: checkout code
uses: actions/checkout@v3
- name: run api tests
run: |
diff --git a/.github/workflows/build_tools.yml b/.github/workflows/build_tools.yml
new file mode 100644
index 000000000..7b2865d36
--- /dev/null
+++ b/.github/workflows/build_tools.yml
@@ -0,0 +1,132 @@
+name: Build Fireball and Management Tools
+env:
+ AWS_REGION: eu-west-2
+permissions:
+ id-token: write
+ contents: read
+
+on:
+ push:
+ branches: [dev, markmac99, master]
+ pull_request:
+ # The branches below must be a subset of the branches above
+ branches: [master]
+jobs:
+ local_tests:
+ name: local test stage
+ runs-on: win10
+ steps:
+ - name: "☁️ checkout repository"
+ uses: actions/checkout@v4
+ - name: testing
+ run: |
+ echo "hello from test"
+ conda activate ukmon-admin
+ dir
+ build_usermgmt:
+ name: build usermanagement
+ runs-on: win10
+ needs: local_tests
+ steps:
+ - name: building
+ run: |
+ echo "running build step"
+ conda activate ukmon-admin
+ cd usermgmt\windows
+ pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico
+ dir dist
+ package_usermgmt:
+ name: package usermanagement
+ runs-on: win10
+ needs:
+ - build_usermgmt
+ steps:
+ - name: packaging
+ run: |
+ echo "packaging the app now"
+ cd usermgmt\windows\dist
+ copy ../camera.ico .
+ copy ../stationmaint.ini.sample .
+ copy ../../README.md .
+ compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip
+ release_usermgmt:
+ name: release usermanagement
+ runs-on: win10
+ needs:
+ - package_usermgmt
+ steps:
+ - name: release
+ uses: xresloader/upload-to-github-release@v1
+ env:
+ GITHUB_TOKEN: ${{ secrets.MM_PAT }}
+ with:
+ file: c:\temp\ukmon_usermgmt.zip
+ tags: true
+ draft: true
+ overwrite: true
+ tag_name: 2025.10.1
+ default_release_name: User Management Tool
+ default_release_body_path: usermgmt\windows\release_notes.md
+ cleanup_usermgmt:
+ name: clean up temp usermanagement files
+ runs-on: win10
+ needs:
+ - release_usermgmt
+ steps:
+ - name: delete temp usermgmt package
+ run: |
+ del c:\temp\ukmon_usermgmt.zip
+ build_fbtool:
+ name: build fbcollector
+ runs-on: win10
+ needs: local_tests
+ steps:
+ - name: building
+ run: |
+ echo "running build step"
+ conda activate fbcollector
+ cd fbCollector
+ pyinstaller ./fireballCollector.py --onefile --windowed --icon .\ukmda.ico
+ package_fbtool:
+ name: package fbcollector
+ runs-on: win10
+ needs:
+ - build_fbtool
+ steps:
+ - name: packaging
+ run: |
+ echo "packaging the app now"
+ cd fbCollector\dist
+ copy ../ukmda.ico .
+ copy ../config.ini.sample .
+ copy ../download_events.sh .
+ copy ../noimage.jpg .
+ copy ../README.md .
+ compress-archive -path . -update -destinationpath c:\temp\fbcollector.zip
+ release_fbtool:
+ name: release fbcollector
+ runs-on: win10
+ needs:
+ - package_fbtool
+ steps:
+ - name: release
+ uses: xresloader/upload-to-github-release@v1
+ env:
+ GITHUB_TOKEN: ${{ secrets.MM_PAT }}
+ with:
+ file: c:\temp\fbcollector.zip
+ tags: true
+ draft: true
+ overwrite: true
+ tag_name: 2025.10.1
+ default_release_name: Tools to Gather Fireball data and Manage UKMON Cameras
+ default_release_body_path: fbCollector\README.md
+ cleanup_fbtool:
+ name: clean up temp fbcollector files
+ runs-on: win10
+ needs:
+ - release_fbtool
+ steps:
+ - name: delete package
+ run: |
+ del c:\temp\fbcollector.zip
diff --git a/.github/workflows/build_usermgmt.yml b/.github/workflows/build_usermgmt.yml
deleted file mode 100644
index 5d078517c..000000000
--- a/.github/workflows/build_usermgmt.yml
+++ /dev/null
@@ -1,71 +0,0 @@
-name: build_usermgmt
-env:
- AWS_REGION: eu-west-2
-permissions:
- id-token: write
- contents: read
-
-on:
- push:
- branches: [ dev, markmac99, master ]
- pull_request:
- # The branches below must be a subset of the branches above
- branches: [ master ]
-jobs:
- local_tests:
- name: local test stage
- runs-on: win10
- steps:
- - name: "☁️ checkout repository"
- uses: actions/checkout@v4
- - name: testing
- run: |
- echo "hello from test"
- conda activate ukmon-admin
- dir usermgmt\windows
- build:
- name: build stage
- runs-on: win10
- needs: local_tests
- steps:
- - name: building
- run: |
- echo "running build step"
- conda activate ukmon-admin
- cd usermgmt\windows
- pyinstaller ./stationMaint2.py --noconsole --onefile --windowed --icon .\camera.ico
- dir dist
- package:
- name: package app
- runs-on: win10
- needs:
- - local_tests
- - build
- steps:
- - name: packaging
- run: |
- echo "packaging the app now"
- cd usermgmt\windows\dist
- copy ../camera.ico .
- ((get-content ..\stationmaint.cfg) -replace "adm_mark","") -replace "~/.ssh/ukmonhelper","" | set-content -path ./stationmaint.cfg
- compress-archive -path . -update -destinationpath c:\temp\ukmon_usermgmt.zip
- release:
- name: release package
- runs-on: win10
- needs:
- - local_tests
- - build
- - package
- steps:
- - name: release
- uses: xresloader/upload-to-github-release@v1
- env:
- GITHUB_TOKEN: ${{ secrets.MM_PAT }}
- with:
- file: c:\temp\ukmon_usermgmt.zip
- #tags: true
- draft: true
- overwrite: true
- tag_name: 2024.04.02
- default_release_name: User Management Tool
- default_release_body_path: usermgmt\windows\release_notes.md
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 86dbe2e6b..77e99e8a1 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -13,12 +13,12 @@ name: "CodeQL"
on:
push:
- branches: [ master ]
+ branches: [master]
pull_request:
# The branches below must be a subset of the branches above
- branches: [ master ]
+ branches: [master]
schedule:
- - cron: '26 14 * * 6'
+ - cron: "26 14 * * 6"
jobs:
analyze:
@@ -32,40 +32,40 @@ jobs:
strategy:
fail-fast: false
matrix:
- language: [ 'javascript', 'python' ]
+ language: ["javascript", "python"]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
# Learn more:
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
steps:
- - name: Checkout repository
- uses: actions/checkout@v3
+ - name: Checkout repository
+ uses: actions/checkout@v3
- # Initializes the CodeQL tools for scanning.
- - name: Initialize CodeQL
- uses: github/codeql-action/init@v2
- with:
- languages: ${{ matrix.language }}
- # If you wish to specify custom queries, you can do so here or in a config file.
- # By default, queries listed here will override any specified in a config file.
- # Prefix the list here with "+" to use these queries and those in the config file.
- # queries: ./path/to/local/query, your-org/your-repo/queries@main
+ # Initializes the CodeQL tools for scanning.
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@v2
+ with:
+ languages: ${{ matrix.language }}
+ # If you wish to specify custom queries, you can do so here or in a config file.
+ # By default, queries listed here will override any specified in a config file.
+ # Prefix the list here with "+" to use these queries and those in the config file.
+ # queries: ./path/to/local/query, your-org/your-repo/queries@main
- # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
- # If this step fails, then you should remove it and run the build manually (see below)
- - name: Autobuild
- uses: github/codeql-action/autobuild@v2
+ # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
+ # If this step fails, then you should remove it and run the build manually (see below)
+ - name: Autobuild
+ uses: github/codeql-action/autobuild@v2
- # ℹ️ Command-line programs to run using the OS shell.
- # 📚 https://git.io/JvXDl
+ # ℹ️ Command-line programs to run using the OS shell.
+ # 📚 https://git.io/JvXDl
- # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
- # and modify them (or add more) to build your code if your project
- # uses a compiled language
+ # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+ # and modify them (or add more) to build your code if your project
+ # uses a compiled language
- #- run: |
- # make bootstrap
- # make release
+ #- run: |
+ # make bootstrap
+ # make release
- - name: Perform CodeQL Analysis
- uses: github/codeql-action/analyze@v2
+ - name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v2
diff --git a/.gitignore b/.gitignore
index e46843f57..6e4b5f8b2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -614,3 +614,6 @@ tests/testing/RMS/
tests/testing/WesternMeteorPyLib/
archive/ukmon_pylib/tests/testdata.tar.gz
servercopybkp/*
+fbcollector/config.ini
+usermgmt/windows/stationmaint.ini
+usermgmt/windows/README.md
diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json
index c0662c555..1e99cf94d 100644
--- a/.vscode/c_cpp_properties.json
+++ b/.vscode/c_cpp_properties.json
@@ -5,8 +5,7 @@
"includePath": [
"${workspaceFolder}/**",
"${vcpkgRoot}/x64-windows/include",
- "${vcpkgRoot}/x86-windows/include",
- "e:/dev/opencv3/opencv-3.4.11/build/install/include"
+ "${vcpkgRoot}/x86-windows/include"
],
"defines": [
"_DEBUG",
diff --git a/archive/containers/trajsolver/Dockerfile b/archive/containers/trajsolver/Dockerfile
index 276e5a11b..ff8b6ff5b 100644
--- a/archive/containers/trajsolver/Dockerfile
+++ b/archive/containers/trajsolver/Dockerfile
@@ -1,18 +1,20 @@
# Copyright (C) 2018-2023 Mark McIntyre
-FROM continuumio/miniconda3:4.11.0
+#FROM continuumio/miniconda3:4.11.0
+FROM continuumio/miniconda3:24.7.1-0
-RUN conda install -y -c conda-forge "numpy<1.25.0" scipy cython pytz pyqt
+#RUN conda install -y -c conda-forge "numpy<1.25.0" scipy cython pytz pyqt
+RUN conda install -y -c conda-forge "numpy<2" scipy cython pytz pyqt
RUN conda install -y -c conda-forge jplephem ephem
RUN conda install -y -c conda-forge basemap basemap-data-hires
RUN conda install -y -c conda-forge pandas cartopy
-RUN conda install -y matplotlib=3.4.3
+RUN conda install -y matplotlib
RUN pip install boto3
RUN apt-get update && apt-get -y upgrade
RUN conda install -y -c conda-forge numba
COPY WesternMeteorPyLib/ ./WesternMeteorPyLib
-ENV PYTHONPATH ./WesternMeteorPyLib
-ENV PROJ_LIB ./
+ENV PYTHONPATH=./WesternMeteorPyLib
+ENV PROJ_LIB=./
COPY *.py ./
#ENV NUMPY_EXPERIMENTAL_DTYPE_API 1
diff --git a/archive/containers/trajsolver/build.ps1 b/archive/containers/trajsolver/build.ps1
index 8d63c8877..4a4dac048 100644
--- a/archive/containers/trajsolver/build.ps1
+++ b/archive/containers/trajsolver/build.ps1
@@ -17,7 +17,7 @@ write-output "building $imagename in $accid"
$yn=read-host -prompt "update WMPL?"
if ($yn.tolower() -eq "y") { bash -c "./update_wmpl.sh $env" }
-if ($env -eq "test") { copy-item awskeys.test awskeys}
+if ($env -eq "test") { move-item awskeys.test awskeys}
docker build . -t ${imagename}
if (! $?)
@@ -26,7 +26,7 @@ if (! $?)
exit
}
set-location $loc
-if ($env -eq "test") { Remove-Item awskeys}
+if ($env -eq "test") { move-item awskeys awskeys.test }
$yn=read-host -prompt "upload to ECR?"
if ($yn.tolower() -eq "y") {
diff --git a/archive/containers/trajsolver/trajsolver.py b/archive/containers/trajsolver/trajsolver.py
index 7155a2359..2d42835f7 100644
--- a/archive/containers/trajsolver/trajsolver.py
+++ b/archive/containers/trajsolver/trajsolver.py
@@ -36,7 +36,7 @@ def runCorrelator(dir_path, time_beg, time_end):
trajectory_constraints.geometric_uncert = not uncerttime
# Clock for measuring script time
- t1 = datetime.datetime.utcnow()
+ t1 = datetime.datetime.now(datetime.timezone.utc)
# If auto run is enabled, compute the time range to use
event_time_range = None
@@ -115,7 +115,7 @@ def runCorrelator(dir_path, time_beg, time_end):
tc = TrajectoryCorrelator(dh, trajectory_constraints, velpart, data_in_j2000=True, distribute=distribute, enableOSM=True)
tc.run(event_time_range=event_time_range)
- print("Total run time: {:s}".format(str(datetime.datetime.utcnow() - t1)))
+ print("Total run time: {:s}".format(str(datetime.datetime.now(datetime.timezone.utc) - t1)))
return
@@ -136,9 +136,7 @@ def getSourceAndTargets():
webbucket = webpth[:webpth.find('/')]
webpth = webpth[webpth.find('/')+1:]
- oldoutb = os.getenv('OLDOUTB', default=None)
- oldwebb = os.getenv('OLDWEBB', default=None)
- return srcbucket, srcpth, outbucket, outpth, webbucket, webpth, oldwebb, oldoutb
+ return srcbucket, srcpth, outbucket, outpth, webbucket, webpth
# extra args for setting the MIME type when uploading to S3.
@@ -168,10 +166,31 @@ def getExtraArgs(fname):
return extraargs
+def checkIfFileNeeded(filename):
+
+ # update this if there's an additional file created by WMPL that we want
+ # to use on the website
+ # NB THIS ALSO HAS TO BE CHANGED in maintenance.recreateOrbitPagesif more files needed
+ if 'orbit_top.png' in filename or 'orbit_side.png' in filename or 'ground_track.png' in filename:
+ return True
+ if 'velocities.png' in filename or 'lengths.png' in filename or 'lags_all.png' in filename:
+ return True
+ if 'abs_mag.png' in filename or 'abs_mag_ht.png' in filename or 'report.txt' in filename:
+ return True
+ if 'all_angular_residuals.png' in filename or 'all_spatial_total_residuals_height.png' in filename:
+ return True
+ if 'trajectory.pickle' in filename:
+ return True
+ return False
+
+
#push solutions to the website, and push the pickle and report to shared bucket
-def pushToWebsite(s3, localfldr, webbucket, webfldr, outbucket, outpth, oldwebb=None, oldoutb=None):
+def pushToWebsite(s3, localfldr, webbucket, webfldr, outbucket, outpth):
for root, _, files in os.walk(localfldr):
for fil in files:
+ if not checkIfFileNeeded(fil):
+ print(f'skipping {fil}')
+ continue
fullname = os.path.join(localfldr, root, fil)
fname = os.path.split(fil)[1]
orbname = os.path.split(root)[1]
@@ -181,26 +200,10 @@ def pushToWebsite(s3, localfldr, webbucket, webfldr, outbucket, outpth, oldwebb=
targkey = f'{webfldr}/{yr}/orbits/{ym}/{ymd}/{orbname}/{fname}'
print(f'uploading {fname} to s3://{webbucket}/{targkey}')
s3.meta.client.upload_file(fullname, webbucket, targkey, ExtraArgs = getExtraArgs(fname))
- if oldwebb:
- print(f' and uploading to {oldwebb}')
- try:
- s3.meta.client.upload_file(fullname, oldwebb, targkey, ExtraArgs = getExtraArgs(fname))
- except:
- print('unable to push to old website')
- else:
- print('not pushing to old website')
if 'report' in fname or 'pickle' in fname:
targkey = f'{outpth}/trajectories/{yr}/{ym}/{ymd}/{orbname}/{fname}'
print(f'uploading {fname} to s3://{outbucket}/{targkey}')
s3.meta.client.upload_file(fullname, outbucket, targkey, ExtraArgs = getExtraArgs(fname))
- if oldoutb:
- print(f' and uploading to {oldoutb}')
- try:
- s3.meta.client.upload_file(fullname, oldoutb, targkey, ExtraArgs = getExtraArgs(fname))
- except:
- print('unable to push to old share')
- else:
- print('not pushing to old share')
return
@@ -238,7 +241,7 @@ def startup(srcfldr, startdt, enddt, isTest=False):
os.makedirs(canddir, exist_ok = True)
s3 = getS3Client()
- srcbucket, srcpth, outbucket, outpth, webbucket, webpth, oldwebb, oldoutb = getSourceAndTargets()
+ srcbucket, srcpth, outbucket, outpth, webbucket, webpth = getSourceAndTargets()
if isTest is True:
outpth = os.path.join(outpth, 'test')
@@ -263,7 +266,7 @@ def startup(srcfldr, startdt, enddt, isTest=False):
# reacquire tokens just in case the 1hour time limit on chained roles is exceeded
s3 = getS3Client()
- pushToWebsite(s3, trajfldr, webbucket, webpth, outbucket, outpth, oldwebb=oldwebb, oldoutb=oldoutb)
+ pushToWebsite(s3, trajfldr, webbucket, webpth, outbucket, outpth)
fname = f'{srcfldr}.json'
jsonfile = os.path.join(localfldr, 'processed_trajectories.json')
diff --git a/archive/cronjobs/mergeNewOrbit.sh b/archive/cronjobs/mergeNewOrbit.sh
index cfdf264b6..09335d201 100644
--- a/archive/cronjobs/mergeNewOrbit.sh
+++ b/archive/cronjobs/mergeNewOrbit.sh
@@ -5,7 +5,11 @@ source $here/../config.ini >/dev/null 2>&1
conda activate $HOME/miniconda3/envs/${WMPL_ENV}
mkdir -p $DATADIR/manualuploads
-cd $DATADIR/manualuploads
+cd $DATADIR/manualuploadsif [ -f ./running ] ; then
+ echo "already running"
+ exit
+fi
+touch .running
aws s3 sync $UKMONSHAREDBUCKET/fireballs/uploads . --exclude "*" --include "*.zip" --exclude "*.done"
newfiles=$(ls -1 *.zip 2> /dev/null)
@@ -57,4 +61,5 @@ EOD
#else
# echo nothing to process
fi
+rm $DATADIR/manualuploads/.running
find $SRC/logs -name "mergeNewOrbit*" -mtime +10 -exec rm -f {} \;
\ No newline at end of file
diff --git a/archive/deployment/pylib.yml b/archive/deployment/pylib.yml
index 162f4ab67..46a4afc71 100644
--- a/archive/deployment/pylib.yml
+++ b/archive/deployment/pylib.yml
@@ -37,6 +37,7 @@
- {src: '{{srcdir}}/ukmon_pylib/converters/__init__.py', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no }
- {src: '{{srcdir}}/ukmon_pylib/converters/README.md', dest: '{{destdir}}/ukmon_pylib/converters', mode: '644', backup: no }
+ - {src: '{{srcdir}}/ukmon_pylib/maintenance/compressOldArchiveData.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no }
- {src: '{{srcdir}}/ukmon_pylib/maintenance/cleanDdbTables.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no }
- {src: '{{srcdir}}/ukmon_pylib/maintenance/dataMaintenance.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no }
- {src: '{{srcdir}}/ukmon_pylib/maintenance/deDuplicate.py', dest: '{{destdir}}/ukmon_pylib/maintenance', mode: '644', backup: no }
diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile b/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile
index 801df9512..26f9034fa 100644
--- a/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile
+++ b/archive/samfunctions/getExtraFilesV2/pythoncode/Dockerfile
@@ -1,16 +1,17 @@
# Copyright (C) 2018-2023 Mark McIntyre
-FROM public.ecr.aws/lambda/python:3.9
+
+FROM public.ecr.aws/lambda/python:3.13
COPY requirements.txt ./
RUN python -m pip install -r requirements.txt -t .
RUN python -m pip install numba watchdog basemap-data-hires -t .
RUN python -m pip install cartopy -t .
-RUN yum update -y
-ENV LD_LIBRARY_PATH ./
+#RUN yum update -y
+ENV LD_LIBRARY_PATH=./
COPY WesternMeteorPyLib/ ./WesternMeteorPyLib
#COPY *.npy ./WesternMeteorPyLib/wmpl/share/
COPY *.py ./
-ENV PYTHONPATH ./WesternMeteorPyLib
-ENV PROJ_LIB ./
+ENV PYTHONPATH=./WesternMeteorPyLib
+ENV PROJ_LIB=./
CMD ["getExtraFiles.lambda_handler"]
diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py b/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py
index 35f5497ac..ab4e24184 100644
--- a/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py
+++ b/archive/samfunctions/getExtraFilesV2/pythoncode/createOrbitPageIndex.py
@@ -31,9 +31,9 @@ def createOrbitPageIndex(fldr, websitebucket, s3):
lis = sumf.readlines()
idxf.writelines(lis)
- zipf = orbitname + '.zip'
- if os.path.isfile(os.path.join(fldr, zipf)):
- idxf.write(f"Click here to download a zip of the raw and processed data.\n")
+ #zipf = orbitname + '.zip'
+ #if os.path.isfile(os.path.join(fldr, zipf)):
+ # idxf.write(f"Click here to download a zip of the raw and processed data.\n")
idxf.write("\n")
idxf.write("Detailed report below graphs
\n")
idxf.write("Click on an image to see a larger view
\n")
diff --git a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py b/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py
index 6d4827a72..8c71bd7b3 100644
--- a/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py
+++ b/archive/samfunctions/getExtraFilesV2/pythoncode/getExtraFiles.py
@@ -5,7 +5,6 @@
import sys
import shutil
import json
-from zipfile import ZipFile, ZIP_DEFLATED
from datetime import datetime, timedelta
from boto3.dynamodb.conditions import Key
@@ -81,12 +80,15 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3):
try:
js = json.loads(obs.comment)
ffname = js['ff_name']
- print(ffname)
+ if ffname[0] == '[':
+ ffname = ffname[2:-2].split(',')
+ if isinstance(ffname, list):
+ ffname = ffname[0]
# case when filename is nonstandard
if 'FF_' not in ffname and 'FR_' not in ffname:
ffname = 'FF_' + ffname
ffname = ffname.replace('FR_', 'FF_').replace('.bin', '.fits')
- if '.bin' not in ffname and '.fits' not in ffname:
+ if '.bin' not in ffname and '.fits' not in ffname and '.jpg' not in ffname:
ffname = ffname + '.fits'
gotff = True
except Exception:
@@ -98,13 +100,15 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3):
else:
pass
if gotff is True:
+ print(ffname)
spls = ffname.split('_')
id = spls[1]
dtstr = spls[2]
tmstr = spls[3]
- jpgname=f'img/single/{dtstr[:4]}/{dtstr[:6]}/{ffname}'.replace('fits','jpg')
- mp4name=f'img/mp4/{dtstr[:4]}/{dtstr[:6]}/{ffname}'.replace('fits','mp4')
+ jpgname=f'img/single/{dtstr[:4]}/{dtstr[:6]}/{ffname}'.replace('.fits','.jpg')
+ mp4name=f'img/mp4/{dtstr[:4]}/{dtstr[:6]}/{ffname}'.replace('.fits','.mp4').replace('.jpg','.mp4')
print('about to get a list of available jpgs')
+ print(jpgname)
res = s3.meta.client.list_objects_v2(Bucket=websitebucket,Prefix=jpgname)
if res['KeyCount'] > 0:
jpgf.write(f'{jpgname}\n')
@@ -157,7 +161,6 @@ def generateExtraFiles(key, archbucket, websitebucket, ddb, s3):
print('no extrampgs.html')
pass
- # pushFilesBack creates the zipfile so we need to do this first
print('pushing files back')
pushFilesBack(outdir, archbucket, websitebucket, fuloutdir, s3)
print('updating the index page')
@@ -239,9 +242,8 @@ def findOtherFiles(evtdt, archbucket, websitebucket, outdir, fldr, s3):
def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3):
- # get filelist before creating the zipfile!
- flist = os.listdir(outdir)
+ flist = os.listdir(outdir)
_, pth =os.path.split(outdir)
yr = pth[:4]
ym = pth[:6]
@@ -251,13 +253,9 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3):
else:
webpth = f'reports/{yr}/orbits/{ym}/{pth}/'
- zipfname = os.path.join(outdir, pth +'.zip')
- zipf = ZipFile(zipfname, 'w', compression=ZIP_DEFLATED, compresslevel=9)
-
for f in flist:
- #print(f)
+ print(f)
locfname = os.path.join(outdir, f)
- zipf.write(locfname)
# some files need to be pushed to the website, some to the archive bucket
if '3dtrack' in f or '2dtrack' in f:
key = os.path.join(webpth, f)
@@ -277,11 +275,6 @@ def pushFilesBack(outdir, archbucket, websitebucket, fldr, s3):
extraargs = getExtraArgs(locfname)
s3.meta.client.upload_file(locfname, archbucket, key, ExtraArgs=extraargs)
- zipf.close()
- # now we push the zipfile
- key = os.path.join(webpth, pth + '.zip')
- extraargs = getExtraArgs(zipfname)
- s3.meta.client.upload_file(zipfname, websitebucket, key, ExtraArgs=extraargs)
return
diff --git a/archive/samfunctions/getExtraFilesV2/template.yaml b/archive/samfunctions/getExtraFilesV2/template.yaml
index a2d87dfe4..54dd99243 100644
--- a/archive/samfunctions/getExtraFilesV2/template.yaml
+++ b/archive/samfunctions/getExtraFilesV2/template.yaml
@@ -38,4 +38,4 @@ Resources:
Metadata:
Dockerfile: Dockerfile
DockerContext: ./pythoncode
- DockerTag: python3.9-v1
+ DockerTag: python3.13-v1
diff --git a/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1 b/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1
index 154cf55c2..6c33151be 100644
--- a/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1
+++ b/archive/samfunctions/getExtraFilesV2/tests/localTest.ps1
@@ -1,4 +1,4 @@
# Copyright (C) 2018-2023 Mark McIntyre
-sam build
+#sam build
sam local invoke --profile ukmda_admin -e tests\testEvent.json
#sam local invoke --profile ukmda_admin -e tests\testEvent2.json
\ No newline at end of file
diff --git a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh b/archive/samfunctions/getExtraFilesV2/update_wmpl.sh
index e06a917b2..57d3fb6fa 100644
--- a/archive/samfunctions/getExtraFilesV2/update_wmpl.sh
+++ b/archive/samfunctions/getExtraFilesV2/update_wmpl.sh
@@ -14,3 +14,4 @@ rm -Rf ./WesternMeteorPyLib/wmpl/MetSim
rm -Rf ./WesternMeteorPyLib/wmpl/CAMO
rm -f ./WesternMeteorPyLib/wmpl/Utils/DynamicMassFit.py
rm -f ./WesternMeteorPyLib/wmpl/Utils/PlotMap_OSM.py
+cp ./wmpl__init__.py_fixed.py ./WesternMeteorPyLib/wmpl/__init__.py
diff --git a/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py b/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py
new file mode 100644
index 000000000..dc34c35dc
--- /dev/null
+++ b/archive/samfunctions/getExtraFilesV2/wmpl__init__.py_fixed.py
@@ -0,0 +1,45 @@
+# This is a fixed-up version of WesternMeteorPyLib/wmpl/__init__.py which is python 3.12+ compatible
+
+""" Import all submodules. """
+
+import importlib
+import pkgutil
+import sys
+
+# Excluded packages
+exclude = ["MetSim.ML", "GUI"]
+
+__all__ = []
+for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
+
+ # Skip the config module
+ if 'Config' in module_name:
+ continue
+
+ # Skip Supracenter
+ if 'Supracenter' in module_name:
+ continue
+
+
+ ### Skip exluded packages ###
+ skip_package = False
+ for exclude_test in exclude:
+ if exclude_test in module_name:
+ skip_package = True
+ break
+
+ if skip_package:
+ continue
+
+ ### ###
+
+
+ __all__.append(module_name)
+ if sys.version_info.major > 2 and sys.version_info.minor > 12:
+ module_spec = loader.find_spec(module_name)
+ if module_spec is not None:
+ module = importlib.util.module_from_spec(module_spec)
+ module_spec.loader.exec_module(module)
+ else:
+ module = loader.find_module(module_name).load_module(module_name)
+ exec('%s = module' % module_name)
diff --git a/archive/samfunctions/getLiveImages/getLiveImages.py b/archive/samfunctions/getLiveImages/getLiveImages.py
index 16202cd1a..164ecc605 100644
--- a/archive/samfunctions/getLiveImages/getLiveImages.py
+++ b/archive/samfunctions/getLiveImages/getLiveImages.py
@@ -53,7 +53,7 @@ def getTrueImgs(dtstr, dtstr2, statid, ddb=None):
if d1.hour < 13:
capnight=(d1 + datetime.timedelta(days=-1)).strftime('%Y%m%d')
else:
- capnight = dtstr[:8]
+ capnight = d1.strftime('%Y%m%d')
lv=Decimal(f'{d1.timestamp():.0f}')
hv=Decimal(f'{d2.timestamp():.0f}')
resp = table.query(KeyConditionExpression=Key('CaptureNight').eq(int(capnight)) & Key('Timestamp').between(lv, hv),
diff --git a/archive/samfunctions/matchDataApi/localTest.ps1 b/archive/samfunctions/matchDataApi/localTest.ps1
new file mode 100644
index 000000000..c71174dee
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/localTest.ps1
@@ -0,0 +1,3 @@
+# Copyright (C) 2018-2023 Mark McIntyre
+# sam build --profile ukmonshared --region eu-west-1
+sam local invoke --profile ukmonshared -e $args[0] --region eu-west-1
diff --git a/archive/samfunctions/matchDataApi/matchDataApi.py b/archive/samfunctions/matchDataApi/matchDataApi.py
new file mode 100644
index 000000000..b7c2023ef
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/matchDataApi.py
@@ -0,0 +1,169 @@
+# Copyright (C) 2018-2023 Mark McIntyre
+#
+# python code behind the MatchData API
+#
+
+import os
+import boto3
+import json
+import pymysql.cursors
+
+
+def getSqlLoginDetails():
+ # retrieve password and host from SSM. This allows me to manage them from Terraform
+ ssm = boto3.client('ssm', region_name='eu-west-1')
+ res = ssm.get_parameter(Name='prod_dbpw', WithDecryption=True)
+ password = res['Parameter']['Value']
+ res = ssm.get_parameter(Name='prod_dbhost')
+ host = res['Parameter']['Value']
+ # should really do these too but they won't change often if at all
+ user = 'batch'
+ db = 'ukmon'
+ return host, user, password, db
+
+
+def getStationData(statid, dtstr):
+ host, user, passwd, db = getSqlLoginDetails()
+ connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor)
+ try:
+ with connection.cursor() as cursor:
+ if statid is None:
+ sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%'"
+ else:
+ sql = f"SELECT s.orbname from matches s where s._localtime like '_{dtstr}%' and s.stations like '%{statid}%'"
+ cursor.execute(sql)
+ result = cursor.fetchall()
+ finally:
+ connection.close()
+ res=''
+ for event in result:
+ res = res + json.dumps(event) + '\n'
+ return res
+
+
+def getSummaryData(dtstr):
+ host, user, passwd, db = getSqlLoginDetails()
+ connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor)
+ fieldlist = '_localtime,_mjd,_sol,_ID1,_amag,_ra_o,_dc_o,_ra_t,_dc_t,_elng,_elat,_vo,_vi,_vg,_vs,_a,_q,_e,_p,_peri,_node,_incl,'\
+ '_stream,_mag,_dur,_lng1,_lat1,_H1,_lng2,_lat2,_H2,_LD21,_az1r,_ev1r,_Nts,_Nos,_leap,_tme,_dt,'\
+ 'dtstamp,orbname,iau,shwrname as name,mass,pi,Q,true_anom,EA,MA,Tj,T,last_peri,jacchia1,Jacchia2,numstats,stations'
+ expr = f"SELECT {fieldlist} from matches s where s._localtime like '_{dtstr}%'"
+ result=[]
+ try:
+ with connection.cursor() as cursor:
+ cursor.execute(expr)
+ result = cursor.fetchall()
+ finally:
+ connection.close()
+ if len(result) > 0:
+ res = json.dumps(result)
+ else:
+ res=json.dumps({'result': 'no data'})
+ res = res.replace('}, {','},\n{')
+ return res
+
+
+def getDetailData(orbname):
+ host, user, passwd, db = getSqlLoginDetails()
+ connection = pymysql.connect(host=host, user=user, password=passwd, db=db, cursorclass=pymysql.cursors.DictCursor)
+ expr = f"SELECT * from matches s where s.orbname like '{orbname}%'"
+ try:
+ with connection.cursor() as cursor:
+ cursor.execute(expr)
+ result = cursor.fetchall()
+ finally:
+ connection.close()
+ if len(result) > 0:
+ res = json.dumps(result[0])
+ else:
+ res=json.dumps({'result': 'no data'})
+ res = res.replace('}, {','},\n{')
+ return res
+
+
+def lambda_handler(event, context):
+ webbuck = os.getenv('WEBBUCKET', default='ukmda-website')
+ #print('received event', json.dumps(event))
+ qs = event['queryStringParameters']
+ if qs is None:
+ return {
+ 'statusCode': 200,
+ 'body': 'usage: detections?reqtyp=xxx&reqval=yyyy[&points=1]'
+ }
+ reqtyp = qs['reqtyp']
+ points = False
+ if 'points' in qs:
+ points = True
+
+ if reqtyp == 'station':
+ statid = qs['statid']
+ dtstr = qs['reqval']
+ print(f'station data requested for {statid} on {dtstr}')
+ res = getStationData(statid, dtstr)
+ elif reqtyp == 'summary':
+ dtstr = qs['reqval']
+ print(f'summary data requested for {dtstr}')
+ res = getSummaryData(dtstr)
+ elif reqtyp == 'matches':
+ dtstr = qs['reqval']
+ print(f'match data requested for {dtstr}')
+ res = getStationData(None, dtstr)
+ elif reqtyp == 'detail':
+ orbname = qs['reqval']
+ print(f'detail requested for {orbname}')
+ res = getDetailData(orbname)
+ if points:
+ print(f'points requested for {orbname}')
+ if 'imgstr' in res:
+ url = json.loads(res)['imgstr']
+ url = url.replace('ground_track.png','report.txt')
+ reppth = url.split('//', 1)[1].split('/',1)[1]
+ _, repname = os.path.split(reppth)
+ try:
+ tmpfname = f'/tmp/{repname}'
+ s3 = boto3.client('s3')
+ s3.download_file(webbuck, reppth, tmpfname)
+ flis = open(tmpfname, 'r').readlines()
+ os.remove(tmpfname)
+ res = fileToJsonString(flis)
+ except Exception:
+ print('report file unavailable')
+ res = '{"points": "unavailable"}'
+ else:
+ res = '{"points": "unavailable"}'
+
+ else:
+ res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}'
+
+ print(res)
+ return {
+ 'statusCode': 200,
+ 'body': res
+ }
+
+
+def fileToJsonString(flis):
+ hdr = ['No','statid','ign','t','jd','m1','m2','az','alt','azl','altl','rao',
+ 'deco','ral','decl','X','Y','Z','lat','lon','H','range','length','svd',
+ 'lag','vel','pvel', 'hres','vres','ares','vmag','amag']
+ ptsarray='['
+ gotpts = False
+ for fli in flis:
+ if 'Points' in fli:
+ gotpts = True
+ continue
+ elif '------' in fli or ' No' in fli:
+ continue
+ elif gotpts is True and (len(fli) < 2 or 'Notes' in fli):
+ gotpts=False
+ break
+ elif gotpts is True:
+ spls = fli.split(',')
+ thisrow = '{'
+ for h, s in zip(hdr, spls):
+ thisrow = thisrow + f'"{h}": "{s.strip()}",'
+ thisrow = thisrow[:-1] + '},'
+ ptsarray = ptsarray + thisrow
+ ptsarray = ptsarray[:-1] + ']'
+ ptsarray = '{' + f'"points": {ptsarray}' + '}'
+ return ptsarray
diff --git a/archive/samfunctions/matchDataApi/requirements.txt b/archive/samfunctions/matchDataApi/requirements.txt
new file mode 100644
index 000000000..917ec643c
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/requirements.txt
@@ -0,0 +1,6 @@
+#
+# requirements for matchDataApi
+# Copyright (C) 2025 Mark McIntyre
+#
+# mysql-connector-python
+pymysql
diff --git a/archive/samfunctions/matchDataApi/samconfig.toml b/archive/samfunctions/matchDataApi/samconfig.toml
new file mode 100644
index 000000000..074b92a09
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/samconfig.toml
@@ -0,0 +1,12 @@
+# Copyright (C) 2018-2023 Mark McIntyre
+version = 0.1
+[default]
+[default.deploy]
+[default.deploy.parameters]
+stack_name = "matchDataApi"
+s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-1cy2k1achj9nv"
+s3_prefix = "matchDataApi"
+region = "eu-west-1"
+capabilities = "CAPABILITY_IAM"
+image_repositories = []
+profile = "ukmda_admin"
diff --git a/archive/samfunctions/matchDataApi/template.yml b/archive/samfunctions/matchDataApi/template.yml
new file mode 100644
index 000000000..d07529b3a
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/template.yml
@@ -0,0 +1,38 @@
+# SAM build file for searchArchive
+# Copyright (C) 2018-2023 Mark McIntyre
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: 'AWS::Serverless-2016-10-31'
+Resources:
+ matchDataApi:
+ Type: AWS::Serverless::Function
+ Properties:
+ Handler: matchDataApi.lambda_handler
+ Runtime: python3.11
+ FunctionName: matchDataApi
+ Description: API backend that fetches match data
+ Timeout: 120
+ Policies: [AmazonS3ReadOnlyAccess, AmazonSSMReadOnlyAccess ]
+ Environment:
+ Variables:
+ ARCHBUCKET: ukmda-shared
+ MPLCONFIGDIR: /tmp/mpl
+ Events:
+ HttpGet:
+ Type: Api
+ Properties:
+ Path: '/'
+ Method: get
+ RequestParameters:
+ - method.request.querystring.reqtyp:
+ Required: true
+ - method.request.querystring.reqval:
+ Required: true
+ - method.request.querystring.points:
+ Required: false
+ - method.request.querystring.statid:
+ Required: false
+ Tags:
+ billingtag: "ukmda"
+ typetag: "api"
+ Metadata:
+ BuildMethod: python3.11
diff --git a/archive/samfunctions/matchDataApi/testDetail.json b/archive/samfunctions/matchDataApi/testDetail.json
new file mode 100644
index 000000000..ee4ab4132
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/testDetail.json
@@ -0,0 +1,7 @@
+{
+ "httpMethod": "GET",
+ "queryStringParameters": {
+ "reqtyp": "detail",
+ "reqval": "20251008_235928.502_UK"
+ }
+}
\ No newline at end of file
diff --git a/archive/samfunctions/matchDataApi/testMatches.json b/archive/samfunctions/matchDataApi/testMatches.json
new file mode 100644
index 000000000..67a03d178
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/testMatches.json
@@ -0,0 +1,7 @@
+{
+ "httpMethod": "GET",
+ "queryStringParameters": {
+ "reqtyp": "matches",
+ "reqval": "20251008"
+ }
+}
\ No newline at end of file
diff --git a/archive/samfunctions/matchDataApi/testPoints.json b/archive/samfunctions/matchDataApi/testPoints.json
new file mode 100644
index 000000000..afcac2367
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/testPoints.json
@@ -0,0 +1,8 @@
+{
+ "httpMethod": "GET",
+ "queryStringParameters": {
+ "reqtyp": "detail",
+ "reqval": "20251008_235928.502_UK",
+ "points": 1
+ }
+}
\ No newline at end of file
diff --git a/archive/samfunctions/matchDataApi/testStation.json b/archive/samfunctions/matchDataApi/testStation.json
new file mode 100644
index 000000000..22e338fa0
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/testStation.json
@@ -0,0 +1,8 @@
+{
+ "httpMethod": "GET",
+ "queryStringParameters": {
+ "reqtyp": "station",
+ "statid": "UK0006",
+ "reqval": "20251008"
+ }
+}
\ No newline at end of file
diff --git a/archive/samfunctions/matchDataApi/testSummary.json b/archive/samfunctions/matchDataApi/testSummary.json
new file mode 100644
index 000000000..785013bcb
--- /dev/null
+++ b/archive/samfunctions/matchDataApi/testSummary.json
@@ -0,0 +1,7 @@
+{
+ "httpMethod": "GET",
+ "queryStringParameters": {
+ "reqtyp": "summary",
+ "reqval": "20251008"
+ }
+}
\ No newline at end of file
diff --git a/archive/samfunctions/updateDnsRecords/lambda_function.py b/archive/samfunctions/updateDnsRecords/lambda_function.py
new file mode 100644
index 000000000..255058f61
--- /dev/null
+++ b/archive/samfunctions/updateDnsRecords/lambda_function.py
@@ -0,0 +1,130 @@
+import os
+import boto3
+import logging
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+""" Default values """
+DNS_TAG_KEY = 'Route53FQDN'
+DNS_RECORD_TYPE = 'CNAME'
+DNS_RECORD_TTL = 60
+
+
+def update_route53_record(region, instance_id, dns_tag_key, dns_record_type, dns_record_ttl):
+ """
+ update function
+ """
+ dns_record_type = dns_record_type.upper()
+
+ """ Get EC2 Description """
+ ec2 = boto3.client('ec2', region_name=region)
+ ec2_desc_res = ec2.describe_instances(InstanceIds=[instance_id])
+ if ('Reservations' not in ec2_desc_res) or (len(ec2_desc_res['Reservations']) < 1) or (len(ec2_desc_res['Reservations'][0]['Instances']) < 1):
+ logger.warning('Error hetting EC2 instance information failure. Instance Id: ' + instance_id)
+ return
+ if (ec2_desc_res['Reservations'][0]['Instances'][0]['State']['Name'] != 'running'):
+ logger.warning('EC2 state is not running. Current state: ' + ec2_desc_res['Reservations'][0]['Instances'][0]['State']['Name'])
+ return
+ ec2_public_dns = ec2_desc_res['Reservations'][0]['Instances'][0]['PublicDnsName']
+ ec2_public_ip = ec2_desc_res['Reservations'][0]['Instances'][0]['PublicIpAddress']
+
+ """ Get EC2 Tags """
+ ec2_tags_res = ec2.describe_tags(Filters=[
+ {'Name': 'resource-id', 'Values': [instance_id]},
+ {'Name': 'key', 'Values': [dns_tag_key]}])
+
+ if ('Tags' not in ec2_tags_res) or (len(ec2_tags_res['Tags']) < 1):
+ logger.warning('EC2 instance has no DNS tag. Instance Id: ' + instance_id)
+ return
+
+ dns_record_name = ec2_tags_res['Tags'][0]['Value']
+ if (dns_record_name[-1] == '.'):
+ dns_record_name = dns_record_name[:-1]
+
+ ec2_tags_res = ec2.describe_tags(Filters=[
+ {'Name': 'resource-id', 'Values': [instance_id]},
+ {'Name': 'key', 'Values': ['DNSRecordType']}])
+ if ('Tags' not in ec2_tags_res) or (len(ec2_tags_res['Tags']) < 1):
+ logger.warning('EC2 instance has no Record Type. Instance Id: ' + instance_id)
+ else:
+ dns_record_type = ec2_tags_res['Tags'][0]['Value']
+
+ """ Check DNS name """
+ dns_record_names = dns_record_name.split('.')
+ if (len(dns_record_names) < 3):
+ logger.warning('The FQDN must over 3 levels at least. Instance Id: ' + instance_id + ', ' + dns_tag_key + ': ' + dns_record_name)
+ return
+ """ Get all hosted zones from Route53 """
+ route53 = boto3.client('route53', region_name=region)
+ zones_res = route53.list_hosted_zones()
+ all_zones = zones_res['HostedZones']
+ if (zones_res['IsTruncated']):
+ while zones_res and (not zones_res['IsTruncated']):
+ zones_res = route53.list_hosted_zones(Marker=zones_res['NextMarker'])
+ all_zones = all_zones + zones_res['HostedZones']
+
+ """ Find Route53 zone """
+ zone_id = None
+ for i in range(1, len(dns_record_names)-1):
+ cur_zone_name = '.'.join(dns_record_names[i:len(dns_record_names)+1]) + '.'
+ cur_zone_matches = [zone for zone in all_zones if (zone['Name']==cur_zone_name)]
+ if (len(cur_zone_matches) > 0):
+ zone_id = cur_zone_matches[0]['Id']
+ break
+ if (not zone_id):
+ print('Route53 zone is not found. DNS name: ' + dns_record_name)
+ return
+
+ """ Update Route53 record record """
+ route53.change_resource_record_sets(
+ HostedZoneId=zone_id,
+ ChangeBatch={
+ 'Changes': [
+ {
+ 'Action': 'UPSERT',
+ 'ResourceRecordSet': {
+ 'Name': dns_record_name + '.',
+ 'Type': dns_record_type,
+ 'ResourceRecords': [{'Value': (ec2_public_dns if (dns_record_type=='CNAME') else ec2_public_ip)}],
+ 'TTL': dns_record_ttl
+ }
+ }
+ ]
+ }
+ )
+ logger.info('The Route53 record updated successfully. ' + dns_record_name + ': [' + dns_record_type + '] ' + (ec2_public_dns if (dns_record_type=='CNAME') else ec2_public_ip))
+ return
+
+
+def lambda_handler(event, context):
+ """
+ main function
+ """
+ region = os.environ.get("region")
+ instance_id = os.environ.get("instance_id")
+ dns_tag_key = os.environ.get("dns_tag_key") if os.environ.get("dns_tag_key") else DNS_TAG_KEY
+ dns_record_type = os.environ.get("dns_record_type") if os.environ.get("dns_record_type") else DNS_RECORD_TYPE
+ dns_record_ttl = int(os.environ.get("dns_record_ttl")) if os.environ.get("dns_record_ttl") else DNS_RECORD_TTL
+
+ if (('region' in event) and event['region']):
+ region = str(event['region'])
+ if (not region):
+ print('Parameter "region" is not defined.')
+ return
+
+ if (('detail' in event) and ('instance-id' in event['detail']) and event['detail']['instance-id']):
+ instance_id = str(event['detail']['instance-id'])
+ if (not instance_id):
+ print('Parameter "instance-id" is not defined.')
+ return
+
+ if (('state' in event['detail']) and event['detail']['state'] and (event['detail']['state'] != 'running')):
+ logger.warning('EC2 state is not running. Current state: ' + event['detail']['state'])
+ return
+
+ try:
+ update_route53_record(region, instance_id, dns_tag_key, dns_record_type, dns_record_ttl)
+ except Exception as error:
+ logger.exception(error)
+ return
diff --git a/archive/samfunctions/updateDnsRecords/samconfig.toml b/archive/samfunctions/updateDnsRecords/samconfig.toml
new file mode 100644
index 000000000..2703b7f39
--- /dev/null
+++ b/archive/samfunctions/updateDnsRecords/samconfig.toml
@@ -0,0 +1,14 @@
+# Copyright (C) 2018-2023 Mark McIntyre
+version = 0.1
+[default]
+[default.deploy]
+[default.deploy.parameters]
+stack_name = "updateDnsRecords"
+s3_bucket = "aws-sam-cli-managed-default-samclisourcebucket-1ulna2hoktme4"
+s3_prefix = "updateDnsRecords"
+region = "eu-west-2"
+confirm_changeset = true
+capabilities = "CAPABILITY_IAM"
+disable_rollback = true
+image_repositories = []
+profile = "ukmda_admin"
\ No newline at end of file
diff --git a/archive/samfunctions/updateDnsRecords/template.yaml b/archive/samfunctions/updateDnsRecords/template.yaml
new file mode 100644
index 000000000..f23b62b13
--- /dev/null
+++ b/archive/samfunctions/updateDnsRecords/template.yaml
@@ -0,0 +1,33 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: AWS Lambda function for updating Route53 record when public ip of EC2 instance.
+
+Resources:
+ updateDnsRecords:
+ Type: AWS::Serverless::Function
+ Properties:
+ FunctionName: updateDnsRecords
+ MemorySize: 128
+ Handler: lambda_function.lambda_handler
+ Timeout: 10
+ Policies:
+ - Statement:
+ - Action:
+ - ec2:DescribeInstances
+ - ec2:DescribeTags
+ - route53:ListHostedZones
+ - route53:ChangeResourceRecordSets
+ Resource: '*'
+ Effect: Allow
+ Runtime: python3.11
+ Events:
+ EC2StateChangeEvent:
+ Type: CloudWatchEvent
+ Properties:
+ Pattern:
+ source:
+ - aws.ec2
+ detail:
+ state:
+ - running
+ Description: AWS Lambda function for updating Route53 record when public ip of EC2 instance.
\ No newline at end of file
diff --git a/archive/terraform/mjmm/crossacct-role.tf b/archive/terraform/mjmm/crossacct-role.tf
index 90b8f135f..fabcd4ecd 100644
--- a/archive/terraform/mjmm/crossacct-role.tf
+++ b/archive/terraform/mjmm/crossacct-role.tf
@@ -24,8 +24,12 @@ data "aws_iam_policy_document" "terraformpoldoc" {
resource "aws_iam_role" "terraformrole" {
name = "TerraformRole"
assume_role_policy = data.aws_iam_policy_document.terraformpoldoc.json
- managed_policy_arns = [data.aws_iam_policy.terraformpol.arn]
tags = {
billingtag = "ukmon"
}
}
+
+resource "aws_iam_role_policy_attachment" "tfrole_pol_attachment" {
+ role = aws_iam_role.terraformrole.name
+ policy_arn = data.aws_iam_policy.terraformpol.arn
+}
diff --git a/archive/terraform/mjmm/ec2.tf b/archive/terraform/mjmm/ec2.tf
index 2e396ea28..e5979d6e9 100644
--- a/archive/terraform/mjmm/ec2.tf
+++ b/archive/terraform/mjmm/ec2.tf
@@ -2,10 +2,10 @@
# built in aws_infra project as shared resources
data "aws_security_group" "ec2publicsg" {
- name = "ec2PublicSG"
+ name = "ec2PublicSG"
}
data "aws_key_pair" "marks_key" {
- key_name = "markskey"
+ key_name = "markskey"
}
@@ -15,6 +15,7 @@ resource "aws_instance" "ukmonhelper_g" {
iam_instance_profile = aws_iam_instance_profile.S3FullAccess.name
key_name = data.aws_key_pair.marks_key.key_name
security_groups = [data.aws_security_group.ec2publicsg.name]
+ force_destroy = false
root_block_device {
tags = {
@@ -24,16 +25,18 @@ resource "aws_instance" "ukmonhelper_g" {
}
volume_size = 50
volume_type = "gp3"
- throughput = 125
- iops = 3000
- encrypted = true
- kms_key_id = aws_kms_key.container_key.arn
+ throughput = 125
+ iops = 3000
+ encrypted = true
+ kms_key_id = aws_kms_key.container_key.arn
}
tags = {
- "Name" = "UKMonHelper2"
- "billingtag" = "ukmon"
- "project" = "UKMonHelper2"
+ "Name" = "UKMonHelper2"
+ "billingtag" = "ukmon"
+ "project" = "UKMonHelper2"
+ "Route53FQDN" = "ukmonhelper.markmcintyreastro.co.uk"
+ "DNSRecordType" = "A"
}
}
diff --git a/archive/terraform/mjmm/s3.tf b/archive/terraform/mjmm/s3.tf
index a23697722..740ba4cc6 100644
--- a/archive/terraform/mjmm/s3.tf
+++ b/archive/terraform/mjmm/s3.tf
@@ -56,6 +56,7 @@ data "aws_iam_policy_document" "websiteacesspolicy" {
########################################################################
resource "aws_s3_bucket" "mjmm-ukmon-shared" {
bucket = "mjmm-ukmon-shared"
+ timeouts {}
tags = {
"billingtag" = "ukmon"
}
@@ -79,7 +80,8 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "mjmm_ukmon_shared
}
resource "aws_s3_bucket_lifecycle_configuration" "shared_lifecycle_rule" {
- bucket = aws_s3_bucket.mjmm-ukmon-shared.id
+ bucket = aws_s3_bucket.mjmm-ukmon-shared.id
+ transition_default_minimum_object_size = "varies_by_storage_class"
rule {
id = "MoveToArchive"
status = "Enabled"
@@ -87,18 +89,16 @@ resource "aws_s3_bucket_lifecycle_configuration" "shared_lifecycle_rule" {
days = 30
storage_class = "STANDARD_IA"
}
+ filter {}
}
rule {
id = "PurgeOldVersions"
status = "Enabled"
expiration {
- days = 0
- expired_object_delete_marker = true
- }
-
- filter {
+ days = 0
}
+ filter {}
noncurrent_version_expiration {
newer_noncurrent_versions = "1"
@@ -140,7 +140,9 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "mjmm_ukmon_live_e
}
resource "aws_s3_bucket_lifecycle_configuration" "live_lifecycle_rule" {
- bucket = aws_s3_bucket.mjmm-ukmon-live.id
+ bucket = aws_s3_bucket.mjmm-ukmon-live.id
+ transition_default_minimum_object_size = "varies_by_storage_class"
+
rule {
id = "MoveToArchive"
status = "Enabled"
@@ -148,18 +150,17 @@ resource "aws_s3_bucket_lifecycle_configuration" "live_lifecycle_rule" {
days = 30
storage_class = "STANDARD_IA"
}
+ filter {}
}
rule {
id = "PurgeOldVersions"
status = "Enabled"
expiration {
- days = 0
- expired_object_delete_marker = true
+ days = 0
}
- filter {
- }
+ filter {}
noncurrent_version_expiration {
newer_noncurrent_versions = "1"
diff --git a/archive/terraform/mjmm/secgrp.tf b/archive/terraform/mjmm/secgrp.tf
index 290136180..859d8cff6 100644
--- a/archive/terraform/mjmm/secgrp.tf
+++ b/archive/terraform/mjmm/secgrp.tf
@@ -14,6 +14,7 @@ resource "aws_security_group" "default" {
description = "default VPC security group"
vpc_id = data.aws_vpc.main_vpc.id
revoke_rules_on_delete = false
+ timeouts {}
ingress = [
{
description = "http in"
@@ -60,9 +61,9 @@ resource "aws_security_group" "default" {
self = false
}
]
-egress = [
+ egress = [
{
- cidr_blocks = ["0.0.0.0/0",]
+ cidr_blocks = ["0.0.0.0/0", ]
description = ""
from_port = 0
ipv6_cidr_blocks = []
@@ -120,13 +121,13 @@ resource "aws_security_group" "launch-wizard-4" {
cidr_blocks = []
description = "IPv6 SSH for Admin"
from_port = 22
- ipv6_cidr_blocks = ["::/0",]
+ ipv6_cidr_blocks = ["::/0", ]
prefix_list_ids = []
protocol = "tcp"
security_groups = []
self = false
to_port = 22
- },
+ },
]
egress = [
{
diff --git a/archive/terraform/ukmda/apigw-domain.tf b/archive/terraform/ukmda/apigw-domain.tf
index 3cb7995ea..1baacdf31 100644
--- a/archive/terraform/ukmda/apigw-domain.tf
+++ b/archive/terraform/ukmda/apigw-domain.tf
@@ -40,6 +40,11 @@ data "aws_api_gateway_rest_api" "matchpickleapi" {
provider = aws.eu-west-1-prov
}
+data "aws_api_gateway_rest_api" "matchdataapi" {
+ name = "matchDataApi"
+ provider = aws.eu-west-1-prov
+}
+
data "aws_api_gateway_rest_api" "liveimagesapi" {
name = "getLiveImages"
provider = aws.eu-west-1-prov
@@ -71,6 +76,14 @@ resource "aws_api_gateway_base_path_mapping" "camdetapi" {
provider = aws.eu-west-1-prov
}
+resource "aws_api_gateway_base_path_mapping" "matchdataapi" {
+ api_id = data.aws_api_gateway_rest_api.matchdataapi.id
+ stage_name = "Prod"
+ domain_name = aws_api_gateway_domain_name.apigwdomain.domain_name
+ base_path = "matches"
+ provider = aws.eu-west-1-prov
+}
+
resource "aws_api_gateway_base_path_mapping" "pickleapi" {
api_id = data.aws_api_gateway_rest_api.matchpickleapi.id
stage_name = "Prod"
diff --git a/archive/terraform/ukmda/cloudwatch.tf b/archive/terraform/ukmda/cloudwatch.tf
index 10e038755..61ab690a0 100644
--- a/archive/terraform/ukmda/cloudwatch.tf
+++ b/archive/terraform/ukmda/cloudwatch.tf
@@ -3,33 +3,6 @@
#
# Copyright (C) 2018-2023 Mark McIntyre
-resource "aws_cloudwatch_metric_alarm" "calcServerDiskSpace" {
- alarm_name = "CalcServer diskspace"
- comparison_operator = "GreaterThanThreshold"
- evaluation_periods = "1"
- metric_name = "disk_used_percent"
- namespace = "CWAgent"
- period = "300"
- statistic = "Average"
- threshold = "90"
- alarm_description = "CalcServer diskspace has gone over 90%"
- insufficient_data_actions = []
- datapoints_to_alarm = 1
- alarm_actions = [aws_sns_topic.ukmdaalerts.arn, ]
- dimensions = {
- "ImageId" = aws_instance.calc_server.ami
- "InstanceId" = aws_instance.calc_server.id
- "InstanceType" = aws_instance.calc_server.instance_type
- "device" = "nvme0n1p1"
- "fstype" = "xfs"
- "path" = "/"
- }
- tags = {
- "billingtag" = "ukmda"
- }
- actions_enabled = true
-}
-
resource "aws_cloudwatch_metric_alarm" "calcServerIdle" {
alarm_name = "CalcServer idle shutdown"
comparison_operator = "LessThanOrEqualToThreshold"
diff --git a/archive/terraform/ukmda/ec2.tf b/archive/terraform/ukmda/ec2.tf
index 8bcb950e3..29c5fd71c 100644
--- a/archive/terraform/ukmda/ec2.tf
+++ b/archive/terraform/ukmda/ec2.tf
@@ -1,13 +1,16 @@
# Copyright (C) 2018-2023 Mark McIntyre
resource "aws_instance" "calc_server" {
- ami = "ami-0df2d8f6def0bd716"
- instance_type = "c6g.4xlarge"
- iam_instance_profile = aws_iam_instance_profile.calcserverrole.name
- key_name = aws_key_pair.marks_key.key_name
+ ami = "ami-0df2d8f6def0bd716"
+ instance_type = "c8g.2xlarge"
+ iam_instance_profile = aws_iam_instance_profile.calcserverrole.name
+ key_name = aws_key_pair.marks_key.key_name
+ force_destroy = false
tags = {
- "Name" = "Calcengine"
- "billingtag" = "ukmda"
+ "Name" = "Calcengine"
+ "billingtag" = "ukmda"
+ "Route53FQDN" = "calcengine.ukmeteors.co.uk"
+ "DNSRecordType" = "A"
}
root_block_device {
tags = {
@@ -16,9 +19,12 @@ resource "aws_instance" "calc_server" {
}
volume_size = 100
}
- network_interface {
+ primary_network_interface {
network_interface_id = aws_network_interface.calcserver_if.id
- device_index = 0
+ }
+
+ metadata_options {
+ http_tokens = "required"
}
}
@@ -40,13 +46,15 @@ resource "aws_network_interface" "calcserver_if" {
################################################
resource "aws_instance" "ubuntu_calc_server" {
- ami = "ami-0bdf149a42243bde8"
- instance_type = "c6g.4xlarge"
- iam_instance_profile = aws_iam_instance_profile.calcserverrole.name
- key_name = aws_key_pair.marks_key.key_name
+ ami = "ami-0bdf149a42243bde8"
+ instance_type = "c6g.4xlarge"
+ iam_instance_profile = aws_iam_instance_profile.calcserverrole.name
+ key_name = aws_key_pair.marks_key.key_name
tags = {
- "Name" = "Calcengine2"
- "billingtag" = "ukmda"
+ "Name" = "calcengine_ub"
+ "billingtag" = "ukmda"
+ "Route53FQDN" = "calcengine_ub.ukmeteors.co.uk"
+ "DNSRecordType" = "A"
}
root_block_device {
tags = {
@@ -55,9 +63,11 @@ resource "aws_instance" "ubuntu_calc_server" {
}
volume_size = 100
}
- network_interface {
+ primary_network_interface {
network_interface_id = aws_network_interface.ubuntu_calcserver_if.id
- device_index = 0
+ }
+ metadata_options {
+ http_tokens = "required"
}
}
@@ -81,13 +91,15 @@ resource "aws_network_interface" "ubuntu_calcserver_if" {
resource "aws_instance" "admin_server" {
- ami = "ami-0c1ce90bf42d9802b"
- instance_type = "t2.micro"
- iam_instance_profile = aws_iam_instance_profile.S3FullAccess.name
- key_name = aws_key_pair.marks_key.key_name
+ ami = "ami-0c1ce90bf42d9802b"
+ instance_type = "t2.micro"
+ iam_instance_profile = aws_iam_instance_profile.S3FullAccess.name
+ key_name = aws_key_pair.marks_key.key_name
tags = {
- "Name" = "AdminServer"
- "billingtag" = "ukmda"
+ "Name" = "AdminServer"
+ "billingtag" = "ukmda"
+ "Route53FQDN" = "adminserver.ukmeteors.co.uk"
+ "DNSRecordType" = "A"
}
root_block_device {
tags = {
@@ -96,14 +108,8 @@ resource "aws_instance" "admin_server" {
}
volume_size = 8
}
-}
-
-# elastic IP attached to the calc_server
-resource "aws_eip" "adminserver_eip" {
- instance = aws_instance.calc_server.id
- domain = "vpc"
- tags = {
- billingtag = "ukmda"
+ metadata_options {
+ http_tokens = "required"
}
-}
+}
diff --git a/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py b/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py
index 6b1b2e966..57ffa44ce 100644
--- a/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py
+++ b/archive/terraform/ukmda/files/matchDataApi/matchDataApiHandler.py
@@ -24,7 +24,7 @@ def lambda_handler(event, context):
if reqtyp not in ['matches','detail','station','summary']:
res = '{"invalid request type - must be one of \'matches\', \'details\', \'station\',\'summary\'"}'
else:
- print(f'{reqtyp} {reqval} {points}')
+ print(f'reqtype {reqtyp} reqval {reqval} points {points}')
if reqtyp == 'summary':
d1 = datetime.datetime.strptime(reqval, '%Y%m%d')
idxfile = 'matches/matched/matches-full-{:04d}.csv'.format(d1.year)
@@ -47,6 +47,7 @@ def lambda_handler(event, context):
fhi = {"FileHeaderInfo": "Use"}
elif reqtyp == 'station':
statid = qs['statid']
+ print(f'statid {statid}')
d1 = datetime.datetime.strptime(reqval, '%Y%m%d')
idxfile = 'matches/matched/matches-full-{:04d}.csv'.format(d1.year)
res = '{"no matches"}'
@@ -78,6 +79,7 @@ def lambda_handler(event, context):
print('report file unavailable')
res = '{"points": "unavailable"}'
+ print(res)
return {
'statusCode': 200,
'body': res
diff --git a/archive/terraform/ukmda/loggingbucket.tf b/archive/terraform/ukmda/loggingbucket.tf
index 7938ec4a1..adbffcf74 100644
--- a/archive/terraform/ukmda/loggingbucket.tf
+++ b/archive/terraform/ukmda/loggingbucket.tf
@@ -10,12 +10,14 @@ resource "aws_s3_bucket" "logbucket" {
resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" {
bucket = aws_s3_bucket.logbucket.id
+ transition_default_minimum_object_size = "all_storage_classes_128K"
rule {
status = "Enabled"
id = "purge old versions"
noncurrent_version_expiration {
noncurrent_days = 30
}
+ filter {}
}
rule {
id = "purge old ukmda-shared logs"
@@ -23,7 +25,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" {
expiration {
days = 10
- expired_object_delete_marker = false
}
filter {
@@ -40,7 +41,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" {
expiration {
days = 10
- expired_object_delete_marker = false
}
filter {
@@ -57,7 +57,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp" {
expiration {
days = 10
- expired_object_delete_marker = false
}
filter {
@@ -106,12 +105,14 @@ resource "aws_s3_bucket" "logbucket_w1" {
resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp_w1" {
bucket = aws_s3_bucket.logbucket_w1.id
provider = aws.eu-west-1-prov
+ transition_default_minimum_object_size = "varies_by_storage_class"
rule {
status = "Enabled"
id = "purge old versions"
noncurrent_version_expiration {
noncurrent_days = 30
}
+ filter {}
}
rule {
id = "purge old ukmda-live logs"
@@ -119,7 +120,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalogslcp_w1" {
expiration {
days = 10
- expired_object_delete_marker = false
}
filter {
diff --git a/archive/terraform/ukmda/matchApiLambda.tf b/archive/terraform/ukmda/matchApiLambda.tf
index 30348402c..68be39136 100644
--- a/archive/terraform/ukmda/matchApiLambda.tf
+++ b/archive/terraform/ukmda/matchApiLambda.tf
@@ -57,7 +57,7 @@ resource "aws_api_gateway_base_path_mapping" "matchapi" {
api_id = aws_api_gateway_rest_api.matchapi_apigateway.id
stage_name = "prod"
domain_name = aws_api_gateway_domain_name.apigwdomain.domain_name
- base_path = "matches"
+ base_path = "matchesv1"
provider = aws.eu-west-1-prov
depends_on = [aws_api_gateway_stage.matchapistage]
}
diff --git a/archive/terraform/ukmda/ssm_variables.tf b/archive/terraform/ukmda/ssm_variables.tf
new file mode 100644
index 000000000..29fba5ac6
--- /dev/null
+++ b/archive/terraform/ukmda/ssm_variables.tf
@@ -0,0 +1,43 @@
+# Copyright (C) 2018-2023 Mark McIntyre
+
+# SSM parameters for use in Lambdas
+
+resource "aws_ssm_parameter" "prod_dbhost" {
+ provider = aws.eu-west-1-prov
+ name = "prod_dbhost"
+ type = "String"
+ value = "3.11.55.160"
+ tags = {
+ "billingtag" = "ukmon"
+ }
+}
+
+resource "aws_ssm_parameter" "prod_dbname" {
+ provider = aws.eu-west-1-prov
+ name = "prod_dbname"
+ type = "String"
+ value = "ukmon"
+ tags = {
+ "billingtag" = "ukmon"
+ }
+}
+
+resource "aws_ssm_parameter" "prod_dbpw" {
+ provider = aws.eu-west-1-prov
+ name = "prod_dbpw"
+ type = "SecureString"
+ value = "Batch33mdl"
+ tags = {
+ "billingtag" = "ukmon"
+ }
+}
+
+resource "aws_ssm_parameter" "prod_dbuser" {
+ provider = aws.eu-west-1-prov
+ name = "prod_dbuser"
+ type = "String"
+ value = "batch"
+ tags = {
+ "billingtag" = "ukmon"
+ }
+}
diff --git a/archive/terraform/ukmda/ukmdaadmin_bucket.tf b/archive/terraform/ukmda/ukmdaadmin_bucket.tf
index c588da602..a36f9a433 100644
--- a/archive/terraform/ukmda/ukmdaadmin_bucket.tf
+++ b/archive/terraform/ukmda/ukmdaadmin_bucket.tf
@@ -3,17 +3,18 @@ resource "aws_s3_bucket" "adminbucket" {
force_destroy = false
tags = {
"billingtag" = "ukmda"
+ "ukmdatype" = "admin"
}
}
resource "aws_s3_bucket_lifecycle_configuration" "adminbucketlcp" {
bucket = aws_s3_bucket.adminbucket.id
+ transition_default_minimum_object_size = "varies_by_storage_class"
rule {
id = "purge old emails"
status = "Enabled"
expiration {
days = 30
- expired_object_delete_marker = false
}
filter {
prefix = "email/"
diff --git a/archive/terraform/ukmda/ukmdalive_bucket.tf b/archive/terraform/ukmda/ukmdalive_bucket.tf
index 2889ce61c..7e1c405bb 100644
--- a/archive/terraform/ukmda/ukmdalive_bucket.tf
+++ b/archive/terraform/ukmda/ukmdalive_bucket.tf
@@ -65,6 +65,7 @@ resource "aws_s3_bucket_policy" "ukmdalivebp" {
resource "aws_s3_bucket_lifecycle_configuration" "ukmdalivelcp" {
bucket = aws_s3_bucket.ukmdalive.id
provider = aws.eu-west-1-prov
+ transition_default_minimum_object_size = "varies_by_storage_class"
rule {
status = "Enabled"
id = "1 months delete"
@@ -76,8 +77,8 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdalivelcp" {
}
expiration {
days = 30
- expired_object_delete_marker = false
}
+ filter {}
}
}
diff --git a/archive/terraform/ukmda/ukmdashared_bucket.tf b/archive/terraform/ukmda/ukmdashared_bucket.tf
index 7e992f090..bad938fe0 100644
--- a/archive/terraform/ukmda/ukmdashared_bucket.tf
+++ b/archive/terraform/ukmda/ukmdashared_bucket.tf
@@ -24,9 +24,9 @@ resource "aws_s3_bucket_policy" "ukmdasharedbp" {
Effect = "Allow"
Principal = {
AWS = [
- "arn:aws:iam::${var.remote_account_id}:role/S3FullAccess", # role used by lambdas
+ "arn:aws:iam::${var.remote_account_id}:role/S3FullAccess", # role used by lambdas
"arn:aws:iam::${var.remote_account_id}:role/lambda-s3-full-access-role", # role used by SAM functions
- "arn:aws:iam::${var.remote_account_id}:role/ecsTaskExecutionRole", # role used by ECS tasks
+ "arn:aws:iam::${var.remote_account_id}:role/ecsTaskExecutionRole", # role used by ECS tasks
"arn:aws:iam::${var.remote_account_id}:user/Mary",
"arn:aws:iam::${var.remote_account_id}:user/Mark"
]
@@ -38,22 +38,22 @@ resource "aws_s3_bucket_policy" "ukmdasharedbp" {
Sid = "DelegateS3Access"
},
{
- "Sid": "BlockAccessToAdmin",
- "Effect": "Deny",
- "Principal": "*",
- "Action": "s3:*",
- "Resource": "arn:aws:s3:::ukmda-shared/admin/*",
- "Condition": {
- "StringNotLike": {
- "aws:userId": [ # update with new relevant IDs
- "AIDASVSZXPTTB3UZT4E2B", # MarkMcIntyreUKM
- "AROAUUCG4WH4GFCTQIKH3", # S3FullAccess in MJMM account
- "AROAUUCG4WH4GFCTQIKH3:*", # S3FullAccess in MJMM account
- "${data.aws_caller_identity.current.account_id}", # root account
- "AROA36ZZGKDHYW6XYFNJD:*"
- ]
- }
+ "Sid" : "BlockAccessToAdmin",
+ "Effect" : "Deny",
+ "Principal" : "*",
+ "Action" : "s3:*",
+ "Resource" : "arn:aws:s3:::ukmda-shared/admin/*",
+ "Condition" : {
+ "StringNotLike" : {
+ "aws:userId" : [ # update with new relevant IDs
+ "AIDASVSZXPTTB3UZT4E2B", # MarkMcIntyreUKM
+ "AROAUUCG4WH4GFCTQIKH3", # S3FullAccess in MJMM account
+ "AROAUUCG4WH4GFCTQIKH3:*", # S3FullAccess in MJMM account
+ "${data.aws_caller_identity.current.account_id}", # root account
+ "AROA36ZZGKDHYW6XYFNJD:*"
+ ]
}
+ }
}
]
Version = "2012-10-17"
@@ -79,12 +79,14 @@ resource "aws_s3_bucket_acl" "ukmdasharedacl" {
resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" {
bucket = aws_s3_bucket.ukmdashared.id
+ transition_default_minimum_object_size = "all_storage_classes_128K"
rule {
status = "Enabled"
id = "purge old versions"
noncurrent_version_expiration {
noncurrent_days = 30
}
+ filter {}
}
rule {
status = "Enabled"
@@ -94,7 +96,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" {
}
transition {
- days = 45
+ days = 30
storage_class = "STANDARD_IA"
}
}
@@ -104,7 +106,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" {
expiration {
days = 2
- expired_object_delete_marker = false
}
filter {
@@ -121,7 +122,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" {
expiration {
days = 30
- expired_object_delete_marker = false
}
filter {
@@ -138,7 +138,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" {
expiration {
days = 60
- expired_object_delete_marker = false
}
filter {
@@ -155,7 +154,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" {
expiration {
days = 30
- expired_object_delete_marker = false
}
filter {
@@ -172,7 +170,6 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" {
expiration {
days = 30
- expired_object_delete_marker = false
}
filter {
@@ -183,6 +180,18 @@ resource "aws_s3_bucket_lifecycle_configuration" "ukmdasharedlcp" {
noncurrent_days = 10
}
}
+ rule {
+ status = "Enabled"
+ id = "Transition to Glacier"
+ filter {
+ prefix = "archive/"
+ }
+
+ transition {
+ days = 180
+ storage_class = "GLACIER_IR"
+ }
+ }
}
diff --git a/archive/terraform/ukmda/ukmdawebsite_bucket.tf b/archive/terraform/ukmda/ukmdawebsite_bucket.tf
index 17e9083b1..e1d2de677 100644
--- a/archive/terraform/ukmda/ukmdawebsite_bucket.tf
+++ b/archive/terraform/ukmda/ukmdawebsite_bucket.tf
@@ -4,6 +4,7 @@ resource "aws_s3_bucket" "archsite" {
force_destroy = false
tags = {
"billingtag" = "ukmda"
+ "ukmdatype" = "website"
}
}
@@ -86,12 +87,14 @@ resource "aws_s3_bucket_logging" "ukmdawebsitelogs" {
resource "aws_s3_bucket_lifecycle_configuration" "archsitelcp" {
bucket = aws_s3_bucket.archsite.id
+ transition_default_minimum_object_size = "varies_by_storage_class"
rule {
status = "Enabled"
id = "purge old versions"
noncurrent_version_expiration {
noncurrent_days = 30
}
+ filter {}
}
rule {
status = "Enabled"
diff --git a/archive/terraform/ukmda/vpc.tf b/archive/terraform/ukmda/vpc.tf
index 79ba622d4..455e6c3a8 100644
--- a/archive/terraform/ukmda/vpc.tf
+++ b/archive/terraform/ukmda/vpc.tf
@@ -73,11 +73,16 @@ resource "aws_default_route_table" "ec2_rtbl" {
vpc_peering_connection_id = aws_vpc_peering_connection.ukmdatommpeering.id
}
tags = {
- "Name" = "ec2_rtbl"
+ "Name" = "public_rtbl"
billingtag = "Management"
}
}
+resource "aws_route_table_association" "ec2_rtbl_assoc" {
+ subnet_id = aws_subnet.ec2_subnet.id
+ route_table_id = aws_default_route_table.ec2_rtbl.id
+}
+
# internet gateway for this account
resource "aws_internet_gateway" "main_igw" {
vpc_id = aws_vpc.ec2_vpc.id
diff --git a/archive/ukmon_pylib/converters/gmnTxtToPandas.py b/archive/ukmon_pylib/converters/gmnTxtToPandas.py
index 958d64b8e..ad98886f9 100644
--- a/archive/ukmon_pylib/converters/gmnTxtToPandas.py
+++ b/archive/ukmon_pylib/converters/gmnTxtToPandas.py
@@ -17,7 +17,8 @@
'Lat1','Lat1sd','Lon1','Lon1sd','H1','H1sd','Lat2','Lat2sd','Lon2','Lon2sd','H2','H2sd',
'Dur','Amag','PkHt','F1','mass','Qc','MedianFitErr','BegIn','EndIn','NumStat','stats']
-dirpath='F:/videos/MeteorCam/gmndata'
+datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data')
+dirpath = os.path.join(datadir, 'gmndata')
def loadOneFile(fname):
diff --git a/archive/ukmon_pylib/maintenance/compressOldArchiveData.py b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py
new file mode 100644
index 000000000..ed04fced5
--- /dev/null
+++ b/archive/ukmon_pylib/maintenance/compressOldArchiveData.py
@@ -0,0 +1,345 @@
+# copyright 2023- Mark McIntyre
+# all rights reserved
+
+"""
+ a set of functions to
+ - compress the ukmda-shared/archive data from before the current year
+ - remove unused but easily recreated files from the website
+ - remove links to the old downloadable zip file
+
+usage:
+ 1) python compressOldArchiveData.py - will scan and compress ALL data from prior years
+ 2) python compressOldArchiveData.py "archive/somefolder" - will scan and compress only the named location folder
+ 3) python compressOldArchiveData.py prune_website - will scan and remove unused files from the website
+ 3) python compressOldArchiveData.py prune_website reports/2024/202401 - will scan and remove unused files
+ from the named location and below
+
+note that any MP4 or AVI files that have been uploaded to this area are removed by this process. The videos should be
+on the website in ukmda-website/img/mp4s in mp4 format
+"""
+
+import boto3
+import zipfile
+import io
+import os
+import datetime
+import argparse
+
+maxyear = datetime.datetime.now().year - 1
+
+s3 = boto3.client('s3')
+s3res = boto3.resource('s3')
+
+
+def listLocations(srcbucket='ukmda-shared', prefixstr='archive/'):
+ """
+ list all top level folders in a particular prefix
+ parameters:
+ srcbucket: [string] bucket name, default ukmda-shared
+ prefixstr: [string] prefix, default archive/ - note must have terminal slash
+
+ returns:
+ list of prefixes
+ """
+ response = s3.list_objects_v2(Bucket=srcbucket, Prefix=prefixstr, Delimiter='/')
+ if 'CommonPrefixes' in response:
+ return response['CommonPrefixes']
+ else:
+ return []
+
+
+def listCameras(srcbucket='ukmda-shared', location='archive/Tackley/'):
+ response = s3.list_objects_v2(Bucket=srcbucket, Prefix=location, Delimiter='/')
+ if 'CommonPrefixes' in response:
+ return response['CommonPrefixes']
+ else:
+ return []
+
+
+def listYears(srcbucket='ukmda-shared', camid='archive/Tackley/c1/'):
+ response = s3.list_objects_v2(Bucket=srcbucket, Prefix=camid, Delimiter='/')
+ if 'CommonPrefixes' in response:
+ return response['CommonPrefixes']
+ else:
+ return []
+
+
+"""
+Not used
+def getAllS3Objects(s3, **base_kwargs):
+ continuation_token = None
+ while True:
+ list_kwargs = dict(MaxKeys=1000, **base_kwargs)
+ if continuation_token:
+ list_kwargs['ContinuationToken'] = continuation_token
+ response = s3.list_objects_v2(**list_kwargs)
+ yield from response.get('Contents', [])
+ if not response.get('IsTruncated'): # At the end of the list?
+ break
+ continuation_token = response.get('NextContinuationToken')
+"""
+
+
+def deleteFiles(flist, srcbucket='ukmda-shared'):
+ chunk_size = 900
+ chunked_list = [flist[i:i + chunk_size] for i in range(0, len(flist), chunk_size)]
+ for ch in chunked_list:
+ delete_keys = {'Objects': []}
+ delete_keys['Objects'] = [{'Key': k} for k in ch]
+ s3.delete_objects(Bucket=srcbucket, Delete=delete_keys)
+ return
+
+
+def compressObjects(srcbucket='ukmda-shared', camprefix=None):
+ lcmprefixes = []
+ if camprefix is None:
+ locations = listLocations()
+ for location in locations:
+ camids = listCameras(location=location['Prefix'])
+ for camid in camids:
+ years = listYears(camid=camid['Prefix'])
+ for year in years:
+ lcmprefix = year['Prefix'][:-1]
+ lcmprefixes.append(lcmprefix)
+ else:
+ spls = camprefix[:-1].split('/')
+ print(spls)
+ if len(spls) < 3:
+ camids = listCameras(location=camprefix)
+ for camid in camids:
+ years = listYears(camid=camid['Prefix'])
+ for year in years:
+ lcmprefix = year['Prefix'][:-1]
+ lcmprefixes.append(lcmprefix)
+ else:
+ years = listYears(camid=camprefix)
+ for year in years:
+ lcmprefix = year['Prefix'][:-1]
+ lcmprefixes.append(lcmprefix)
+
+ for lcmprefix in lcmprefixes:
+ spls = lcmprefix.split('/')
+ zfname = f'{spls[0]}/{spls[1]}/{spls[1]}_{spls[2]}_{spls[3]}.zip'
+ try:
+ if int(spls[3]) > maxyear or 'miscellan' in spls[3] or 'profiles' in spls[3]:
+ continue
+ except Exception:
+ pass
+ if 'testpi' in lcmprefix:
+ continue
+ bucket = s3res.Bucket(srcbucket)
+ files = [os.key for os in bucket.objects.filter(Prefix=lcmprefix)]
+ mp4files = [file for file in files if '.mp4' in file]
+ deleteFiles(mp4files)
+ avifiles = [file for file in files if '.avi' in file]
+ deleteFiles(avifiles)
+ zip_buffer = io.BytesIO()
+ with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zipf:
+ for key in files:
+ if '.mp4' not in key and '.avi' not in key:
+ print(key)
+ data = s3.get_object(Bucket=srcbucket, Key=key)
+ content = data['Body'].read()
+ zipf.writestr(key, content)
+
+ # Upload the zip file to the target bucket
+ zip_buffer.seek(0)
+ print(f'uploading {zfname}')
+ s3.upload_fileobj(zip_buffer, srcbucket, zfname)
+ # now delete the files
+ print(f'deleting files from {lcmprefix}')
+ deleteFiles(files)
+
+
+def moveJpgsAndMp4s(source_bucket, yr, ym):
+ print(f'moving jpgs and mp4s from {ym}')
+ bucket = s3res.Bucket(source_bucket)
+ files = [os.key for os in bucket.objects.filter(Prefix=f'reports/{yr}/orbits/{ym}/')]
+ jpgs = [x for x in files if '.jpg' in x or '.mp4' in x]
+ print(f'moving {len(jpgs)} jpgs or mp4s')
+ for jpg in jpgs:
+ print(jpg)
+ keyname = os.path.split(jpg)[1]
+ if '.jpg' in keyname:
+ newkey = f'img/single/{yr}/{ym}/{keyname}'
+ else:
+ newkey = f'img/mp4/{yr}/{ym}/{keyname}'
+ s3.copy_object(Bucket=source_bucket,
+ CopySource={'Bucket':source_bucket,'Key':jpg},
+ Key=newkey, MetadataDirective='COPY')
+ s3.delete_object(Bucket=source_bucket, Key=jpg)
+ jpgfldrs = [os.path.split(x)[0] for x in jpgs]
+ jpgfldrs = list(set(jpgfldrs))
+ jpgfldrs.sort()
+ print(f'updating {len(jpgfldrs)} indexes')
+ for fldr in jpgfldrs:
+ print(fldr)
+ idx = f'{fldr}/index.html'
+ rootdir = os.path.split(idx)[0]
+ try:
+ s3.download_file(source_bucket, idx, '/tmp/oldindex.html')
+ except Exception:
+ continue
+ lis = open('/tmp/oldindex.html', 'r').readlines()
+ # skip files that dont have the line in them
+ mtch = [li for li in lis if '.jpg' in li or '.mp4' in li]
+ if len(mtch) == 0:
+ continue
+ with open('/tmp/newindex.html', 'w') as outf:
+ for li in lis:
+ if '.jpg' in li and '/img/single/' not in li:
+ newpth=f'/img/single/{yr}/{ym}/'
+ li = li.replace('a href="', f'a href="{newpth}')
+ li = li.replace('img src="', f'img src="{newpth}')
+ if '.mp4' in li and '/img/mp4/' not in li:
+ newpth=f'/img/mp4/{yr}/{ym}/'
+ li = li.replace('a href="', f'a href="{newpth}')
+ li = li.replace('source src="', f'source src="{newpth}')
+ li = li.replace(rootdir, '')
+ li = li.replace('//', '/')
+ li = li.replace('//', '/')
+ outf.write(li)
+ extraargs = {'ContentType': 'text/html'}
+ s3.upload_file('/tmp/newindex.html', source_bucket, idx, ExtraArgs=extraargs)
+ print('done')
+
+
+def fixBrokenIndexes(source_bucket, yr, ym):
+ bucket = s3res.Bucket(source_bucket)
+ files = [os.key for os in bucket.objects.filter(Prefix=f'reports/{yr}/orbits/{ym}/')]
+ idxs = [x for x in files if 'index.html' in x]
+ for idx in idxs:
+ print(idx)
+ s3.download_file(source_bucket, idx, '/tmp/oldindex.html')
+ lis = open('/tmp/oldindex.html', 'r').readlines()
+ rootdir = os.path.split(idx)[0]
+ # skip files that dont have the line in them
+ mtch = [li for li in lis if '.jpg' in li or '.mp4' in li]
+ if len(mtch) == 0:
+ continue
+ with open('/tmp/newindex.html', 'w') as outf:
+ for li in lis:
+ if '.jpg' in li and '/img/single/' not in li:
+ newpth=f'/img/single/{yr}/{ym}/'
+ li = li.replace('a href="', f'a href="{newpth}')
+ li = li.replace('img src="', f'img src="{newpth}')
+ if '.mp4' in li and '/img/mp4/' not in li:
+ newpth=f'/img/mp4/{yr}/{ym}/'
+ li = li.replace('a href="', f'a href="{newpth}')
+ li = li.replace('source src="', f'source src="{newpth}')
+ li = li.replace(rootdir, '')
+ li = li.replace('//', '/')
+ li = li.replace('//', '/')
+ outf.write(li)
+# for li in lis:
+# if '.jpg' in li:
+# oldpth = f'/img/single/{yr}/{ym}'
+# newpth = f'/img/single/{yr}/{ym}/'
+# li = li.replace(f'a href="{oldpth}', f'a href="{newpth}')
+# li = li.replace(f'img src="{oldpth}', f'img src="{newpth}')
+# if '.mp4' in li:
+# oldpth = f'/img/single/{yr}/{ym}'
+# newpth=f'/img/mp4/{yr}/{ym}/'
+# li = li.replace(f'a href="{oldpth}', f'a href="{newpth}')
+# li = li.replace(f'source src="{oldpth}', f'source src="{newpth}')
+# li = li.replace(rootdir, '')
+# li = li.replace('//', '/')
+# li = li.replace('//', '/')
+# outf.write(li)
+ extraargs = {'ContentType': 'text/html'}
+ s3.upload_file('/tmp/newindex.html', source_bucket, idx, ExtraArgs=extraargs)
+
+
+def updateIndexes(idxs, source_bucket):
+ for idx in idxs:
+ if len(idx.split('/')) < 7:
+ continue
+ s3.download_file(source_bucket, idx, '/tmp/oldindex.html')
+ lis = open('/tmp/oldindex.html', 'r').readlines()
+ # skip files that dont have the line in them
+ mtch = [li for li in lis if 'download a zip of the' in li]
+ if len(mtch) == 0:
+ continue
+ with open('/tmp/newindex.html', 'w') as outf:
+ for li in lis:
+ if 'download a zip of the' in li:
+ continue
+ outf.write(li)
+ extraargs = {'ContentType': 'text/html'}
+ s3.upload_file('/tmp/newindex.html', source_bucket, idx, ExtraArgs=extraargs)
+ return
+
+
+def pruneObjects(source_bucket, prefix_str, force_reindex=False):
+ spls = prefix_str.split('/')
+ bucket = s3res.Bucket(source_bucket)
+ if len(spls) < 3:
+ years = listYears(source_bucket, prefix_str)
+ years = [x['Prefix'] for x in years if 'reports/20' in x['Prefix']]
+ else:
+ years = [f'reports/{spls[1]}/']
+ for yr in years:
+ print(f'processing {yr}')
+ yr_prefix = f'{yr}orbits/'
+ if len(spls) > 3:
+ mths = [f'{yr_prefix}{spls[2]}/']
+ else:
+ mths = listYears(source_bucket, yr_prefix)
+ mths = [x['Prefix'] for x in mths if 'csv/' not in x['Prefix'] and 'plots/' not in x['Prefix']]
+ for mth in mths:
+ print(f'processing {mth}')
+ files = [os.key for os in bucket.objects.filter(Prefix=mth)]
+ print('purging zip files')
+ zipfs = [file for file in files if '.zip' in file]
+ deleteFiles(zipfs, source_bucket)
+ if len(zipfs) > 0 or force_reindex:
+ print('updating indexes')
+ idxs = [file for file in files if '/index.html' in file]
+ updateIndexes(idxs, source_bucket)
+ print('purging pngs')
+ spatres = [file for file in files if '_spatial_residuals.png' in file and '_all' not in file]
+ deleteFiles(spatres, source_bucket)
+ print('purging kml and ftpdetect')
+ others = [file for file in files if '.kml' in file or 'FTPdetectinfo' in file]
+ deleteFiles(others, source_bucket)
+ print('purging csv')
+ csvs = [file for file in files if '.csv' in file]
+ deleteFiles(csvs, source_bucket)
+ return
+
+
+if __name__ == '__main__':
+ prefix_str = None
+
+ arg_parser = argparse.ArgumentParser(description='Compress, prune or clear down older data')
+
+ arg_parser.add_argument('command_str', nargs=1, metavar='COMMAND_STR', type=str,
+ help='action to take. Options are COMPRESS or PRUNE to compress archive files or prune the website.')
+
+ arg_parser.add_argument('-f', '--folder', help="""area to act on, eg "Tackley" or "2025/202504".""")
+
+ arg_parser.add_argument('-i', '--reindex', action='store_true', help="""force recreation of indexes".""")
+
+ args = arg_parser.parse_args()
+
+ if args.command_str[0].upper() == 'COMPRESS':
+ source_bucket = 'ukmda-shared'
+ if args.folder:
+ prefix_str = 'archive/' + prefix_str
+ prefix_str = args.folder
+ if prefix_str[-1] != '/':
+ prefix_str = prefix_str + '/'
+ print(f'compressing {prefix_str}')
+ compressObjects(source_bucket, prefix_str)
+
+ if args.command_str[0].upper() == 'PRUNE':
+ source_bucket = 'ukmda-website'
+ if args.folder:
+ prefix_str = 'reports/' + args.folder
+ if prefix_str[-1] != '/':
+ prefix_str = prefix_str + '/'
+ force_reindex = False
+ if args.reindex:
+ force_reindex = True
+ print(f'pruning {prefix_str} and forcing reindex')
+ pruneObjects(source_bucket, prefix_str, force_reindex)
diff --git a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py
index 62af22c17..f1b477283 100644
--- a/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py
+++ b/archive/ukmon_pylib/maintenance/getUserAndKeyInfo.py
@@ -111,12 +111,17 @@ def getKeyStatuses(excludeadm=False):
except ClientError as error:
err = error.response['Error']['Code']
if err == 'CredentialReportNotPresentException' or err == 'CredentialReportExpiredException':
+ print('creating credentials report')
res = iamc.generate_credential_report()
+ time.sleep(60)
elif err == 'CredentialReportNotReadyException':
+ print('report not ready, sleeping 30s')
time.sleep(30)
gotit = False
while not gotit:
try:
+ res = iamc.generate_credential_report()
+ time.sleep(60)
res = iamc.get_credential_report()
gotit = True
print('got the report')
@@ -342,7 +347,10 @@ def rollKeysForUsers(nusers=3):
_, stalekeys, _, _ = getKeyStatuses(excludeadm=True)
if len(stalekeys) == 0:
return
- sampledf = stalekeys.sample(n=nusers)
+ if len(stalekeys) > nusers:
+ sampledf = stalekeys.sample(n=nusers)
+ else:
+ sampledf = stalekeys
for _, rw in sampledf.iterrows():
uid = rw.user
if 'testtraj' in uid:
diff --git a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py
index cb74b0163..4f5f050cf 100644
--- a/archive/ukmon_pylib/maintenance/recreateOrbitPages.py
+++ b/archive/ukmon_pylib/maintenance/recreateOrbitPages.py
@@ -136,6 +136,26 @@ def fixupTrajComments(traj, availableimages, outdir, picklename):
return
+def checkIfFileNeeded(filename):
+
+ # update this if there's an additional file created by WMPL that we want
+ # to use on the website
+ # NB THIS ALSO HAS TO BE CHANGED in TRAJSOLVER if more files needed
+ if 'orbit_top.png' in filename or 'orbit_side.png' in filename or 'ground_track.png' in filename:
+ return True
+ if 'velocities.png' in filename or 'lengths.png' in filename or 'lags_all.png' in filename:
+ return True
+ if 'abs_mag.png' in filename or 'abs_mag_ht.png' in filename or 'report.txt' in filename:
+ return True
+ if 'all_angular_residuals.png' in filename or 'all_spatial_total_residuals_height.png' in filename:
+ return True
+ if 'trajectory.pickle' in filename:
+ return True
+ if 'extra' in filename:
+ return True
+ return False
+
+
def recreateOrbitFiles(outdir, pickname, doupload=False):
traj = loadPickle(outdir, pickname)
traj.save_results = True
@@ -149,8 +169,12 @@ def recreateOrbitFiles(outdir, pickname, doupload=False):
print('created additional output')
basename = pickname[:15]
repname = basename + '_report.txt'
- traj.saveReport(outdir, repname, None, False)
- traj.savePlots(outdir, basename, show_plots=False, ret_figs=False)
+ try:
+ traj.saveReport(outdir, repname, None, False)
+ traj.savePlots(outdir, basename, show_plots=False, ret_figs=False)
+ except:
+ print('unable to process pickle properly')
+ pass
print('created reports and figures')
orbparent, orbfldr = os.path.split(outdir)
yr = orbfldr[:4]
@@ -161,7 +185,12 @@ def recreateOrbitFiles(outdir, pickname, doupload=False):
availableimages = getImgList(outdir, traj)
createExtraJpgtxt(outdir, traj, availableimages)
createExtraJpgHtml(outdir, orbparent, yr, ym)
- fixupTrajComments(traj, availableimages, outdir, pickname)
+ try:
+ fixupTrajComments(traj, availableimages, outdir, pickname)
+ except:
+ print('unable to fixup traj comments')
+ pass
+
if doupload:
files = os.listdir(outdir)
@@ -172,6 +201,9 @@ def recreateOrbitFiles(outdir, pickname, doupload=False):
else:
webfldr = f'reports/{yr}/orbits/{ym}'
for fil in files:
+ if not checkIfFileNeeded(fil):
+ print(f'skipping {fil}')
+ continue
locfname = f'{outdir}/{fil}'
if 'summary' in fil or 'extrajpgs.txt' or 'html' in fil:
keyname = f"{webfldr}/{orbfldr}/{fil}"
diff --git a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py
index 186be1274..feb2a5079 100644
--- a/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py
+++ b/archive/ukmon_pylib/maintenance/rerunFailedLambdas.py
@@ -83,17 +83,17 @@ def checkFails(fails):
s3bucket=os.getenv('WEBSITEBUCKET', default='s3://ukmda-website')[5:]
realfails = []
for f in fails:
- # example f : matches/RMSCorrelate/trajectories/2022/202202/20220227/20220227_011240.522_UK
+ # example f : matches/RMSCorrelate/trajectories/2022/202202/20220227/20220227_011240.522_UK/20220227_011240_trajectory.pickle
orb = f[f.find('/20')+1:]
+
orb = 'reports/' + orb[:5] + 'orbits' + orb[4:]
pth, _ = os.path.split(orb)
- _, zipname = os.path.split(pth)
- keyf = pth + '/' + zipname + '.zip'
+ keyf = f'{pth}/index.html'
try:
_ = s3.head_object(Bucket=s3bucket, Key=keyf)
#print(f'{pth} already done')
except:
- print(f'adding {keyf}')
+ print(f'adding {f}')
realfails.append(f)
return realfails
diff --git a/archive/ukmon_pylib/reports/CameraDetails.py b/archive/ukmon_pylib/reports/CameraDetails.py
index 49e5680ec..74ce7ac86 100644
--- a/archive/ukmon_pylib/reports/CameraDetails.py
+++ b/archive/ukmon_pylib/reports/CameraDetails.py
@@ -126,7 +126,7 @@ def createStatOptsHtml(sitedets, datadir, active=False, locs=False):
def findSite(stationid, camdets=None, ddb=None):
if camdets is None:
camdets = loadLocationDetails(ddb=ddb)
- cc = camdets[camdets.stationid==stationid]
+ cc = camdets[camdets.stationid==stationid[:6]]
if len(cc) > 0:
res = cc.iloc[0]['site']
else:
@@ -137,7 +137,7 @@ def findSite(stationid, camdets=None, ddb=None):
def findEmail(stationid, camdets=None, ddb=None):
if camdets is None:
camdets = loadLocationDetails(ddb=ddb)
- cc = camdets[camdets.stationid==stationid]
+ cc = camdets[camdets.stationid==stationid[:6]]
if len(cc) > 0:
res = cc.iloc[0]['eMail']
else:
diff --git a/archive/ukmon_pylib/reports/findBestMp4s.py b/archive/ukmon_pylib/reports/findBestMp4s.py
index bd0af136c..a07611ddd 100644
--- a/archive/ukmon_pylib/reports/findBestMp4s.py
+++ b/archive/ukmon_pylib/reports/findBestMp4s.py
@@ -5,9 +5,81 @@
import shutil
import sys
import os
+import datetime
+import requests
from traj.pickleAnalyser import getAllMp4s
+def getBestNMatches(reqdate=None, numtoget=10):
+ if reqdate is None:
+ tod = datetime.datetime.now()
+ tod = tod.replace(hour=12, minute=0, second=0, microsecond=0)
+ reqdate = tod + datetime.timedelta(days=-1)
+ else:
+ reqdate = reqdate.replace(hour=12, minute=0, second=0, microsecond=0)
+ tod = reqdate + datetime.timedelta(days=1)
+
+ yr = reqdate.year
+ datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data')
+ mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap')
+
+ # select only the columns we need
+ cols=['_localtime', '_mag','url']
+ matches = pd.read_parquet(mf, columns=cols)
+ matches['dt'] = [datetime.datetime.strptime(x,'_%Y%m%d_%H%M%S') for x in matches._localtime]
+
+ matches = matches[matches.dt >= reqdate]
+ matches = matches[matches.dt <= tod]
+ sepdata = matches.sort_values(by=['_mag'])
+ sorteddata = sepdata.head(numtoget)
+ sorteddata.drop_columns(['_localtime'], inplace=True)
+
+
+def getBestNSingles(reqdate=None, numtoget=20, shwr=None, outdir=None):
+ if reqdate is None:
+ tod = datetime.datetime.now()
+ tod = tod.replace(hour=12, minute=0, second=0, microsecond=0)
+ reqdate = tod + datetime.timedelta(days=-1)
+ else:
+ print(reqdate)
+ reqdate = datetime.datetime.strptime(reqdate, '%Y%m%d')
+ reqdate = reqdate.replace(hour=12, minute=0, second=0, microsecond=0)
+ tod = reqdate + datetime.timedelta(days=1)
+ yr = reqdate.year
+ url = f'https://archive.ukmeteors.co.uk/browse/parquet/singles-{yr}.parquet.snap'
+
+ # select only the columns we need
+ cols=['Mag','Shwr','Filename','Dtstamp']
+ matches = pd.read_parquet(url, columns=cols)
+
+ matches = matches[matches.Dtstamp >= reqdate.timestamp()]
+ matches = matches[matches.Dtstamp <= tod.timestamp()]
+ sepdata = matches.sort_values(by=['Mag'])
+ sepdata['url'] = [getUrlFromFilename(x) for x in sepdata.Filename]
+ if shwr:
+ sepdata = sepdata[sepdata.Shwr==shwr]
+ sorteddata = sepdata.head(numtoget)
+ if outdir:
+ outdir = os.path.join(outdir, reqdate.strftime('%Y%m%d'))
+ os.makedirs(outdir, exist_ok=True)
+ for _, rw in sorteddata.iterrows():
+ fname = (f'{rw.Shwr}_{rw.Mag}_{rw.Filename}').replace('.fits','.jpg')
+ print(fname)
+ res = requests.get(rw.url)
+ if res.status_code == 200:
+ open(os.path.join(outdir, fname),'wb').write(res.content)
+
+ filtereddata = sorteddata.drop(columns=['Dtstamp','Filename'])
+ return filtereddata
+
+
+def getUrlFromFilename(fname):
+ ymd = fname.split('_')[2]
+ jpgname = fname.replace('.fits','.jpg')
+ url = f'https://archive.ukmeteors.co.uk/img/single/{ymd[:4]}/{ymd[:6]}/{jpgname}'
+ return url
+
+
def getBestNMp4s(yr, mth, numtoget):
datadir=os.getenv('DATADIR', default='/home/ec2-user/prod/data')
mf = os.path.join(datadir, 'matched', f'matches-full-{yr}.parquet.snap')
diff --git a/archive/ukmon_pylib/traj/README.md b/archive/ukmon_pylib/traj/README.md
index 8b6280a2e..7cb699d70 100644
--- a/archive/ukmon_pylib/traj/README.md
+++ b/archive/ukmon_pylib/traj/README.md
@@ -12,7 +12,7 @@ routines associated with trajectory solving and analysis of results
* plotOSMGroundTrack.py - creates an OS-style ground map from a trajectory pickle.
* showerAssociation.py - creates shower association information from a trajectory pickle.
-* clusdetails-ee.txt - files autogenerated by Terraform containing cluster details for the distributed solver
+* clusdetails-mda.txt - files autogenerated by Terraform containing cluster details for the solver
* clusdetails-mm.txt
* clusdetails.txt
diff --git a/archive/ukmon_pylib/traj/distributeCandidates.py b/archive/ukmon_pylib/traj/distributeCandidates.py
index 1c0254082..3030c9b08 100644
--- a/archive/ukmon_pylib/traj/distributeCandidates.py
+++ b/archive/ukmon_pylib/traj/distributeCandidates.py
@@ -19,9 +19,7 @@
def getClusterDetails(templdir):
sts = boto3.client('sts')
accid = sts.get_caller_identity()['Account']
- if accid == '822069317839':
- clusdetails = os.path.join(templdir, 'clusdetails-ee.txt')
- elif accid == '183798037734':
+ if accid == '183798037734':
clusdetails = os.path.join(templdir, 'clusdetails-mda.txt')
else:
clusdetails = os.path.join(templdir, 'clusdetails-mm.txt')
diff --git a/archive/utils/checkCalcServerRunTime.sh b/archive/utils/checkCalcServerRunTime.sh
new file mode 100644
index 000000000..8380d69f3
--- /dev/null
+++ b/archive/utils/checkCalcServerRunTime.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# script to extract the start/stop times of the Calculation Engine server from the logs
+# I have a spreadsheet that consumes this and works out the cost for various models of server (c8g.2xlarge, c8g.4xlarge etc)
+
+cd ~/prod/logs
+
+csl=/tmp/calcserver_timings.txt
+csl2=/tmp/csl.txt
+[ -f $csl ] && rm $csl
+touch $csl
+
+ls -1tr nightly*.gz | while read i
+do
+ mfz=matchJob${i:10:50}
+ gunzip $i
+ gunzip $mfz
+ nf=$(basename $i .gz)
+ mf=$(basename $mfz .gz)
+ egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl
+ gzip $nf
+ gzip $mf
+done
+ls -1tr nightlyJob* | grep -v gz | while read nf
+do
+ mf=matchJob${nf:10:50}
+ egrep "start correlation server|stop the server again|restarting server to|stopping calcserver again" $mf $nf >> $csl
+done
+[ -f $csl2 ] && rm $csl2
+touch $csl2
+cat $csl | while read i
+do
+ parta=$(echo $i | awk -F ">" '{print $2}')
+ partb=$(echo $parta | awk -F " " '{printf("%s, %s, %s\n",$1,$2,$3)}')
+ echo $partb >> $csl2
+done
+[ -f $csl ] && rm $csl
+python << EOD
+lis = open('$csl2','r').readlines()
+data = [x.strip().split(',') for x in lis]
+currdt = None
+results = []
+res = []
+col = 0
+with open('/tmp/computeservertimes.csv','w') as outf:
+ for i in range(len(data)):
+ mth = data[i][0]
+ dy = data[i][1]
+ tim = data[i][2]
+ dtstr = f'{mth} {dy}'
+ if currdt is None:
+ currdt = dtstr
+ if currdt != dtstr:
+ outf.write(','.join(res) + '\n')
+ res = []
+ col = 0
+ if col == 0:
+ res.append(dtstr)
+ res.append(tim)
+ else:
+ res.append(tim)
+ col += 1
+ currdt = dtstr
+ outf.write(','.join(res) + '\n')
+EOD
+[ -f $csl2 ] && rm $csl2
diff --git a/archive/utils/getGmnData.sh b/archive/utils/getGmnData.sh
new file mode 100644
index 000000000..bfbaf9398
--- /dev/null
+++ b/archive/utils/getGmnData.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+source $here/../config.ini >/dev/null 2>&1
+conda activate $HOME/miniconda3/envs/${WMPL_ENV}
+
+cd $DATADIR/gmndata
+
+y1=$(date +%Y)
+y2=$(date +%Y --date 'yesterday')
+rsync -vtz gmn.uwo.ca:/srv/meteor-ro/rms/gmn/extracted_data/auto_output/traj_summary_data/*$y1.txt . --exclude traj_summary_all.txt
+if [ "$ym1" != "$ym2" ] ; then
+ rsync -vtz gmn.uwo.ca:/srv/meteor-ro/rms/gmn/extracted_data/auto_output/traj_summary_data/*$y2.txt . --exclude traj_summary_all.txt
+fi
+ym1=$(date +%Y%m)
+ym2=$(date +%Y%m --date 'yesterday')
+rsync -vtz gmn.uwo.ca:/srv/meteor-ro/rms/gmn/extracted_data/auto_output/traj_summary_data/monthly/*${ym1}.txt ./monthly --exclude traj_summary_all.txt
+if [ "$ym1" != "$ym2" ] ; then
+ rsync -vtz gmn.uwo.ca:/srv/meteor-ro/rms/gmn/extracted_data/auto_output/traj_summary_data/monthly/*${ym2}.txt ./monthly --exclude traj_summary_all.txt
+fi
\ No newline at end of file
diff --git a/archive/utils/statsToMqtt.sh b/archive/utils/statsToMqtt.sh
index 50b58a699..3235281a2 100644
--- a/archive/utils/statsToMqtt.sh
+++ b/archive/utils/statsToMqtt.sh
@@ -5,7 +5,7 @@ if [ "$(hostname)" == "wordpresssite" ] ; then
source /home/bitnami/venvs/openhabstuff/bin/activate
python $here/statsToMqtt.py
diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g')
- if [ $diskpct -gt 90 ] ; then
+ if [ $diskpct -gt 9 ] ; then
/home/bitnami/src/maintenance/mysql_checks.sh
msg="$(hostname) diskspace was ${diskpct}%"
subj="Diskspace alert for $(hostname)"
@@ -27,7 +27,7 @@ else
export PYTHONPATH=$SRC/ukmon_pylib:$PYTHONPATH
python -m metrics.statsToMqtt
diskpct=$(df / |tail -1| awk '{print $5 }' | sed 's/%//g')
- if [ $diskpct -gt 90 ] ; then
+ if [ $diskpct -gt 95 ] ; then
msg="$(hostname) diskspace is ${diskpct}%"
subj="Diskspace alert for $(hostname)"
hn="$(hostname)@aws"
diff --git a/archive/website/createSummaryTable.sh b/archive/website/createSummaryTable.sh
index 33577ded1..fab46348f 100644
--- a/archive/website/createSummaryTable.sh
+++ b/archive/website/createSummaryTable.sh
@@ -1,6 +1,6 @@
#!/bin/bash
# Copyright (C) 2018-2023 Mark McIntyre
-# Creates the table and bar chart that appear on the website homepage.
+# Create the archive website homepage, and the table and bar chart that appear on the page.
# Also creates the coverage map, and copies the logfile to the site.
#
# Parameters
diff --git a/archive/website/templates/frontpage.html b/archive/website/templates/frontpage.html
index 4ff72d8d6..c1af50676 100644
--- a/archive/website/templates/frontpage.html
+++ b/archive/website/templates/frontpage.html
@@ -63,7 +63,7 @@ Coverage
We currently have #NUMCAMS# cameras in the network, the locations of which are shown at the foot of
this page. Camera Coverage can be seen here.
Camera status can be quickly checked here or
- in more detail on the GMN site here.
+ in more detail on the GMN site here.
Summary Data
The graph below shows activity year to date and the table summarises our entire dataset.
You can also see an excellent map of all detections here,
diff --git a/fbCollector/README.md b/fbCollector/README.md
new file mode 100644
index 000000000..4887015bc
--- /dev/null
+++ b/fbCollector/README.md
@@ -0,0 +1,48 @@
+# A Fireball Collector and Analyser for UKMON and GMN
+
+This tool allows authorised users to collect fireball data from UKMON and GMN and then to reduce and solve it.
+
+## Prerequisites
+
+* You have installed [WMPL](https://github.com/wmpg/WesternMeteorPyLib/), used to solve trajectories.
+* You have installed [RMS](https://github.com/CroatianMeteorNetwork/RMS), used to reduce raw data.
+* A local folder where you will store fireball data. This is called `basedir` in this documentation.
+
+The above are sufficient to collect data from UKMON and to analyse data either collected from UKMON or by other means. There are additional configuration options that can be used to collect data from GMN and upload solutions to UKMON. These are described inline below.
+
+## Installation
+* Clone this repository to a location of your choice
+* Install WMPL and RMS, and activate the WMPL python virtual environment.
+* Change directory into the location of this code then install the additional requiremnts with `pip install -r requirements.txt`
+* Copy `config.ini.sample` to `config.ini` and update the values of `basedir`, `rms_loc`, `rms_env`, `wmpl_loc`, and `wmpl_env` as appropriate.
+
+## Using the App
+* Launch the app by running `fbCollector.ps1` in a Powershell window. After a few seconds the GUI window should appear.
+* In the box labelled `Image Selection` enter a date and time in the format `YYYYMMDD_HHMMSS`, then click "Get Images".
+* After a few seconds, the listbox below should be populated with images from around the time you selected, provided the UKMON live feed captured something.
+ * If nothing appears, check the UKMON [livestream](https://archive.ukmeteors.co.uk/live/index.html) to make sure you chose the correct time.
+
+* Go through the image list and click `Remove` to delete any that are not of the event you are interested in.
+* Once you've whittled the list down to just the interesting events, you can select `Get ECSVs` from the `Raw` menu. This will attempt to get raw data for each image. You can also click `Get Videos` to collect any video data thats available.
+* Now you can click `Solve` from the `Solve` menu. This will invoke WMPL and will attempt to find a trajectory that matches the raw data. It may take some time.
+* If the solver is successful, you can view the solution by selecting `View Solution` from the `Solve` menu. The left-hand list will now show the output of the solver so you can examine it. You can switch back to viewing the raw images by selecting `Review Images` from the `Review` menu.
+* If the solve process fails or if the solution seems very poor then try excluding some detections. To do this, select `Excl/Incl ECSV` from the `Raw` menu then rerun the Solver.
+* If the solution was bad its also worth deleting it before attempting a rerun via `Delete Solution` from the `Solve` menu.
+
+## Manually Reducing an Image
+If you've installed RMS then if you have a FITS or FR file from RMS along with the camera's config and platepar files, you can run RMS's `SkyFit2` tool to analyse the image - select and image then select `Reduce Selected Image` from the `Raw` menu.
+
+## Sending Solutions to UKMON
+Once you have a solution, select `Upload orbit` from the `Solve` menu. This will create a Zip file that bundles the created trajectory pickle file with any images and videos. You can then upload the file to Dropbox, Google Drive or another file-sharing site and email a link to fireballdata@ukmeteornetwork.org where one of our team will validate and upload it to our Archive.
+
+#### API Key
+Members of the UKMON team who frequently create solutions can request an API key to upload solutions less than 10 MB in size. We'll contact you if we think this is applicable.
+
+### Collecting data from GMN
+Members of the GMN coordinators group who have permission from Denis Vida can use this tool to collect data directly from GMN or using the GMN Watchlist. If you fall into this category you can fill in the `[gmnanalysis]` section of the config file with the name of your SSH private key and other details. This will activate additional menu options to `Get GMN Raw Data` and to use the `Watchlist`.
+If you're running on Windows, you will need WSL2 enabled and the rsync tool installed in WSL2.
+
+## Logs
+The programme creates logs in your system's `TEMP` folder.
+
+
diff --git a/fbCollector/config.ini.sample b/fbCollector/config.ini.sample
new file mode 100644
index 000000000..2128f65dd
--- /dev/null
+++ b/fbCollector/config.ini.sample
@@ -0,0 +1,32 @@
+# tailor this config file to your needs
+# Note that $HOME will be translated into your homedir (~ on linux, $env:userprofile on windows)
+
+[Fireballs]
+# the location to which fireball data will be downloaded eg c:\fireballs.
+basedir=
+
+[reduction]
+# To use the reduction options RMS must be installed and an RMS python conda environment created
+rms_loc=
+rms_env=RMS
+
+[solver]
+# To use the solver options WMPL must be installed and a WMPL python conda environment created
+wmpl_loc=
+wmpl_env=wmpl
+
+[sharing]
+# create a folder in Dropbox or similar if you want to share raw data with others and then put
+# its *local* path here eg c:\users\yourname\dropbox\meteors
+shrfldr=
+
+[gmn]
+# the GMN SSH key needs to be configured with Denis Vida
+# if you have permission you can then collect data from the GMN server
+gmnkey=
+gmnuser=
+gmnserver=gmn.uwo.ca
+
+[ukmon]
+apikey=
+
diff --git a/fbCollector/download_events.sh b/fbCollector/download_events.sh
new file mode 100644
index 000000000..d75b9a430
--- /dev/null
+++ b/fbCollector/download_events.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+dt=$1
+basedir=$(grep basedir $here/config.ini | awk -F= '{print $2}'|tr -d '\r' | sed 's|f:|/mnt/f|g' | sed 's|c:|/mnt/c|g')
+echo $basedir $dt
+sleep 30
+dtns=$dt
+mkdir -p $basedir/$dtns
+pushd $basedir/$dtns
+
+rsync -avz gmn.uwo.ca:/home/uk*/files/event_monitor/*${dt}*.bz2 .
+rsync -avz gmn.uwo.ca:/home/be*/files/event_monitor/*${dt}*.bz2 .
+rsync -avz gmn.uwo.ca:/home/ie*/files/event_monitor/*${dt}*.bz2 .
+rsync -avz gmn.uwo.ca:/home/nl*/files/event_monitor/*${dt}*.bz2 .
+if [ "$2" == "all" ] ; then
+ rsync -avz gmn.uwo.ca:/home/fr*/files/event_monitor/*${dt}*.bz2 .
+ rsync -avz gmn.uwo.ca:/home/de*/files/event_monitor/*${dt}*.bz2 .
+ rsync -avz gmn.uwo.ca:/home/es*/files/event_monitor/*${dt}*.bz2 .
+ rsync -avz gmn.uwo.ca:/home/ch*/files/event_monitor/*${dt}*.bz2 .
+ rsync -avz gmn.uwo.ca:/home/it*/files/event_monitor/*${dt}*.bz2 .
+ rsync -avz gmn.uwo.ca:/home/cz*/files/event_monitor/*${dt}*.bz2 .
+ rsync -avz gmn.uwo.ca:/home/hr*/files/event_monitor/*${dt}*.bz2 .
+ rsync -avz gmn.uwo.ca:/home/sk*/files/event_monitor/*${dt}*.bz2 .
+fi
+
+for f in *.bz2 ; do tar -xvf $f ; done
+if [ -d UKMON ] ; then
+ sites=$(ls -1 UKMON)
+ for site in $sites ; do
+ cams=$(ls -1 UKMON/$site)
+ for cam in $cams ; do
+ if [ -d ./$cam ] ; then rm -Rf $cam ; fi
+ mv -f UKMON/$site/$cam .
+ done
+ done
+ rm -Rf UKMON
+fi
+if [ -d NEMETODE ] ; then
+ sites=$(ls -1 NEMETODE)
+ for site in $sites ; do
+ cams=$(ls -1 NEMETODE/$site)
+ for cam in $cams ; do
+ if [ -d ./$cam ] ; then rm -Rf $cam ; fi
+ mv -f NEMETODE/$site/$cam .
+ done
+ done
+ rm -Rf NEMETODE
+fi
+if [ -d "UKMON,NEMETODE" ] ; then
+ sites=$(ls -1 "UKMON,NEMETODE")
+ for site in $sites ; do
+ cams=$(ls -1 "UKMON,NEMETODE/$site")
+ for cam in $cams ; do
+ if [ -d ./$cam ] ; then rm -Rf $cam ; fi
+ mv -f "UKMON,NEMETODE/$site/$cam" .
+ done
+ done
+ rm -Rf "UKMON,NEMETODE"
+fi
+mkdir -p ./stacks
+mkdir -p ./jpgs
+mkdir -p ./mp4s
+cp -f */*.jpg ./jpgs
+mv -f ./jpgs/*captured_stack* ./stacks
+cp -f */*.mp4 ./mp4s
+popd
\ No newline at end of file
diff --git a/fbCollector/fbCollector.ps1 b/fbCollector/fbCollector.ps1
new file mode 100644
index 000000000..52d6e2c17
--- /dev/null
+++ b/fbCollector/fbCollector.ps1
@@ -0,0 +1,25 @@
+#
+# powershell script to launch the fireball data collector tool
+# Copyright (C) 2018-2023 Mark McIntyre
+#
+push-location $PSScriptRoot
+
+$wmpl_env=((select-string .\config.ini -pattern "wmpl_env" -list).line).split('=')[1]
+$wmpl_loc=((select-string .\config.ini -pattern "wmpl_loc" -list).line).split('=')[1]
+
+conda activate $wmpl_env
+
+$wmpl_loc = $wmpl_loc.replace('$HOME',$env:userprofile)
+$wmpl_loc = $wmpl_loc.replace('\','/')
+
+$env:pythonpath="$wmpl_loc"
+
+if ($args.count -lt 1) {
+ python $PSScriptRoot/fireballCollector.py
+}else {
+ python $PSScriptRoot/fireballCollector.py -d $args[0]
+}
+
+Pop-Location
+
+
diff --git a/fbCollector/fbcollector.yml b/fbCollector/fbcollector.yml
new file mode 100644
index 000000000..83463e685
--- /dev/null
+++ b/fbCollector/fbcollector.yml
@@ -0,0 +1,24 @@
+---
+- hosts: ukmonhelper2
+ gather_facts: no
+ tasks:
+ - name: import variables
+ include_vars: ./vars.yml
+ tags: [dev,prod]
+ - name: Ensures {{destdir}} exists
+ file: path={{destdir}} state=directory
+ delegate_to: localhost
+ tags: [dev,prod]
+ - name: Copy files
+ copy: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }}
+ tags: [dev,prod]
+ delegate_to: localhost
+ with_items:
+ - {src: '{{srcdir}}/fbCollector.ps1', dest: '{{destdir}}/', mode: '755', backup: no, directory_mode: no }
+ - {src: '{{srcdir}}/fireballCollector.py', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no }
+ - {src: '{{srcdir}}/download_events.sh', dest: '{{destdir}}/', mode: '755', backup: no, directory_mode: no }
+ - {src: '{{srcdir}}/config.ini.sample', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no }
+ - {src: '{{srcdir}}/noimage.jpg', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no }
+ - {src: '{{srcdir}}/ukmda.ico', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no }
+ - {src: '{{srcdir}}/requirements.txt', dest: '{{destdir}}/', mode: '644', backup: no, directory_mode: no }
+
diff --git a/fbCollector/fireballCollector.py b/fbCollector/fireballCollector.py
new file mode 100644
index 000000000..078f07ad5
--- /dev/null
+++ b/fbCollector/fireballCollector.py
@@ -0,0 +1,1146 @@
+#
+# UI to manage fireball data collection
+# Copyright (C) 2018-2023 Mark McIntyre
+#
+import os
+import sys
+import argparse
+import logging
+import logging.handlers
+import datetime
+import configparser
+import shutil
+import platform
+import subprocess
+import glob
+import xmltodict
+from PIL import Image
+import requests
+import pandas as pd
+
+import paramiko
+from scp import SCPClient
+
+try:
+ from wmpl.Formats.ECSV import loadECSVs
+ from wmpl.Formats.GenericFunctions import solveTrajectoryGeneric
+ solveravailable = 'enabled'
+except Exception:
+ solveravailable = 'disabled'
+
+import tkinter as tk
+import tkinter.filedialog as tkFileDialog
+import tkinter.messagebox as tkMessageBox
+from tkinter.simpledialog import askstring
+from tkinter import StringVar, Frame, ACTIVE, END, Listbox, Menu, Entry, Button
+from tkinter.ttk import Label, Style, LabelFrame, Scrollbar
+
+from PIL import Image as img
+from PIL import ImageTk
+
+
+config_file = ''
+noimg_file = ''
+global_bg = "Black"
+global_fg = "Gray"
+
+
+def quitApp():
+ # Cleanly exits the app
+ root.quit()
+ root.destroy()
+
+
+def log_timestamp():
+ """ Returns timestamp for logging.
+ """
+ return datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
+
+
+def showConfig():
+ if platform.system() == 'Darwin': # macOS
+ procid = subprocess.Popen(('open', config_file))
+ elif platform.system() == 'Windows': # Windows
+ procid = subprocess.Popen(('cmd','/c',config_file))
+ else: # linux variants
+ procid = subprocess.Popen(('xdg-open', config_file))
+ procid.wait()
+
+
+class StyledButton(Button):
+ """ Button with style.
+ """
+ def __init__(self, *args, **kwargs):
+ Button.__init__(self, *args, **kwargs)
+
+ self.configure(foreground = global_fg, background = global_bg, borderwidth = 3)
+
+
+class StyledEntry(Entry):
+ """ Entry box with style.
+ """
+ def __init__(self, *args, **kwargs):
+ Entry.__init__(self, *args, **kwargs)
+
+ self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg, disabledbackground = global_bg, disabledforeground = "DimGray")
+
+
+class ConstrainedEntry(StyledEntry):
+ """ Entry box with constrained values which can be input (e.g. 0-255).
+ """
+ def __init__(self, *args, **kwargs):
+ StyledEntry.__init__(self, *args, **kwargs)
+ self.maxvalue = 255
+ vcmd = (self.register(self.on_validate), "%P")
+ self.configure(validate="key", validatecommand=vcmd)
+ # self.configure(foreground = global_fg, background = global_bg, insertbackground = global_fg)
+
+ def disallow(self):
+ """ Pings a bell on values which are out of bound.
+ """
+ self.bell()
+
+ def update_value(self, maxvalue):
+ """ Updates values in the entry box.
+ """
+ self.maxvalue = maxvalue
+ vcmd = (self.register(self.on_validate), "%P")
+ self.configure(validate="key", validatecommand=vcmd)
+
+ def on_validate(self, new_value):
+ """ Checks if entered value is within bounds.
+ """
+ try:
+ if new_value.strip() == "":
+ return True
+ value = int(new_value)
+ if value < 0 or value > self.maxvalue:
+ self.disallow()
+ return False
+ except ValueError:
+ self.disallow()
+ return False
+
+ return True
+
+
+def getECSVs(stationID, dateStr, savefiles=False, outdir='.'):
+ """
+ Retrieve a detection in ECSV format for the specified date
+ """
+ apiurl='https://api.ukmeteors.co.uk/getecsv?stat={}&dt={}'
+ res = requests.get(apiurl.format(stationID, dateStr))
+ ecsvlines=''
+ if res.status_code == 200:
+ rawdata=res.text.strip()
+ if len(rawdata) > 10:
+ ecsvlines=rawdata.split('\n') # convert the raw data into a python list
+ if savefiles is True:
+ os.makedirs(outdir, exist_ok=True)
+ fnamebase = dateStr.replace(':','_').replace('.','_') # create an output filename
+ j=0
+ outf = False
+ for li in ecsvlines:
+ if 'issue getting data' in li:
+ print(li)
+ return li
+ if '# %ECSV' in li:
+ if outf is not False:
+ outf.close()
+ j=j+1
+ fname = fnamebase + f'_ukmda_{stationID}_M{j:03d}.ecsv'
+ outf = open(os.path.join(outdir, fname), 'w')
+ print('saving to ', os.path.join(outdir,fname))
+ if outf:
+ outf.write(f'{li}\n')
+ else:
+ print('no ECSV marker found in data')
+ else:
+ print('no error, but no data returned')
+ else:
+ print(res.status_code)
+ return ecsvlines
+
+
+def _download(url, outdir, fname=None):
+ get_response = requests.get(url, stream=True)
+ if fname is None:
+ fname = url.split('?')[0].split("/")[-1]
+ with open(os.path.join(outdir, fname), 'wb') as f:
+ for chunk in get_response.iter_content(chunk_size=4096):
+ if chunk: # filter out keep-alive new chunks
+ f.write(chunk)
+ return fname
+
+
+def getLiveJpgs(dtstr, outdir=None):
+ """
+ Retrieve live images from the ukmon website that match a pattern
+ """
+ if outdir is None:
+ outdir = dtstr
+ os.makedirs(outdir, exist_ok=True)
+
+ apiurl = 'https://api.ukmeteors.co.uk/liveimages/getlive'
+
+ while len(dtstr) < 15:
+ dtstr = dtstr + '0'
+ isodt1 = datetime.datetime.strptime(dtstr[:15],'%Y%m%d_%H%M%S')
+ fromdstr = isodt1.isoformat()[:19]+'.000Z'
+ isodt2 = isodt1 + datetime.timedelta(minutes=1)
+ todstr = isodt2.isoformat()[:19]+'.000Z'
+ liveimgs = pd.read_json(f'{apiurl}?dtstr={fromdstr}&enddtstr={todstr}&fmt=withxml')
+
+ for _, thisimg in liveimgs.iterrows():
+ try:
+ jpgurl = thisimg .urls['url']
+ fname = _download(jpgurl, outdir)
+ log.info(f'retrieved {fname}')
+ except:
+ log.warning(f'{img.image_name} unavailable')
+
+
+class fbCollector(Frame):
+
+ def __init__(self, parent, patt=None):
+ Frame.__init__(self, parent, bg = global_bg)
+ parent.configure(bg = global_bg) # Set backgound color
+ parent.grid_columnconfigure(0, weight=1)
+ parent.grid_rowconfigure(0, weight=1)
+
+ self.grid(sticky="NSEW") # Expand frame to all directions
+ self.parent = parent
+
+ self.fb_dir = ''
+ self.gmn_key = ''
+ self.gmn_user = ''
+ self.gmn_server = ''
+ self.wmpl_loc = ''
+ self.wmpl_env = ''
+ self.rms_loc = ''
+ self.rms_env = ''
+ self.api_key = None
+ self.share_loc = ''
+ self.selected = {}
+ self.evtMonTriggered = None
+ self.review_stack = False
+ self.soln_outputdir = None
+ self.log_files_to_keep = 30
+ self.script_loc = os.path.split(config_file)[0]
+
+ self.readConfig()
+
+ self.patt = patt
+ if patt is None:
+ self.dir_path = self.fb_dir.strip()
+ else:
+ self.dir_path = os.path.join(self.fb_dir, patt)
+ log.info(f'Fireball folder is {self.fb_dir}')
+ log.info(f'Scripts folder is {self.script_loc}')
+
+ self.initUI()
+ if self.dir_path != self.fb_dir:
+ bin_list = self.get_bin_list()
+ for b in bin_list:
+ self.selected[b] = (0, '')
+ self.update_listbox(bin_list)
+
+ # Update UI changes
+ parent.update_idletasks()
+ parent.update()
+
+ return
+
+ def readConfig(self):
+ localcfg = configparser.ConfigParser()
+ localcfg.read(config_file)
+ self.fb_dir = os.path.expanduser(localcfg['Fireballs']['basedir'].replace('$HOME','~')).replace('\\','/')
+ os.makedirs(self.fb_dir, exist_ok=True)
+
+ try:
+ self.gmn_key = localcfg['gmn']['gmnkey']
+ self.gmn_user = localcfg['gmn']['gmnuser']
+ self.gmn_server = localcfg['gmn']['gmnserver']
+ except:
+ self.gmn_key = None
+
+ try:
+ self.api_key = localcfg['ukmon']['apikey']
+ if os.path.isfile(os.path.expanduser(self.api_key)):
+ key = open(os.path.expanduser(self.api_key), 'r').readlines()[0].strip()
+ else:
+ key = self.api_key
+ if len(key) < 40:
+ key = None
+ self.api_key = key
+ except:
+ self.api_key = None
+ # log.info(f'apikey "{self.api_key}"')
+
+ try:
+ self.wmpl_loc = os.path.expanduser(localcfg['solver']['wmpl_loc'].replace('$HOME','~')).replace('\\','/')
+ self.wmpl_env= localcfg['solver']['wmpl_env']
+ except:
+ log.error('WMPL location and environment name must be configured')
+ quitApp()
+ try:
+ self.rms_loc = os.path.expanduser(localcfg['reduction']['rms_loc'].replace('$HOME','~')).replace('\\','/')
+ self.rms_env = localcfg['reduction']['rms_env']
+ except:
+ self.rms_loc = None
+
+ try:
+ self.share_loc = os.path.expanduser(localcfg['sharing']['shrfldr'].replace('$HOME','~')).replace('\\','/')
+ except:
+ self.share_loc = None
+ return
+
+ def quitApplication(self):
+ print('quitting')
+ logdir = os.path.join(os.getenv('TMP'), 'fbcollector')
+ logfiles = os.listdir(logdir)
+ numtokeep = self.log_files_to_keep
+ if len(logfiles) > numtokeep:
+ logfiles.sort()
+ for i in range(len(logfiles)-numtokeep):
+ try:
+ os.remove(os.path.join(logdir, logfiles[i]))
+ except:
+ pass
+ quitApp()
+
+ def initUI(self):
+ """ Initialize GUI elements.
+ """
+
+ self.parent.title("Fireball Downloader")
+
+ # Configure the style of each element
+ s = Style()
+ s.configure("TButton", padding=(0, 5, 0, 5), font='serif 10')
+ s.configure('TLabelframe.Label', foreground=global_fg, background=global_bg)
+ s.configure('TLabelframe', foreground =global_fg, background=global_bg, padding=(3, 3, 3, 3))
+ s.configure("TRadiobutton", foreground = global_fg, background = global_bg)
+ s.configure("TLabel", foreground = global_fg, background = global_bg)
+ s.configure("TCheckbutton", foreground = global_fg, background = global_bg)
+ s.configure("Vertical.TScrollbar", background=global_bg, troughcolor = global_bg)
+
+ self.columnconfigure(0, pad=3)
+ self.columnconfigure(1, pad=3)
+ self.columnconfigure(2, pad=3)
+ self.columnconfigure(3, pad=3)
+ self.columnconfigure(4, pad=3)
+ self.columnconfigure(5, pad=3)
+ self.columnconfigure(6, pad=3)
+ self.columnconfigure(7, pad=3)
+ self.columnconfigure(8, pad=3)
+
+ self.rowconfigure(0, pad=3)
+ self.rowconfigure(1, pad=3)
+ self.rowconfigure(2, pad=3)
+ self.rowconfigure(3, pad=3)
+ self.rowconfigure(4, pad=3)
+ self.rowconfigure(5, pad=3)
+ self.rowconfigure(6, pad=3)
+ self.rowconfigure(7, pad=3)
+ self.rowconfigure(8, pad=3)
+ self.rowconfigure(9, pad=3)
+ self.rowconfigure(10, pad=3)
+
+ # Make menu
+ self.menuBar = Menu(self.parent)
+ self.parent.config(menu=self.menuBar)
+
+ # File menu
+ fileMenu = Menu(self.menuBar, tearoff=0)
+ fileMenu.add_command(label="Load Folder", command=self.loadFolder)
+ fileMenu.add_command(label="Archive This Folder", command=self.archiveFolder)
+ fileMenu.add_command(label="Open in Explorer", command=self.openFolder)
+ fileMenu.add_separator()
+ fileMenu.add_command(label="Delete This Folder", command=self.delFolder)
+ fileMenu.add_separator()
+ fileMenu.add_command(label="Configuration", command=self.reviewConfig)
+ fileMenu.add_command(label="View Logs", command=self.viewLogs)
+ fileMenu.add_separator()
+ fileMenu.add_command(label="Exit", command=self.quitApplication)
+ self.menuBar.add_cascade(label="File", underline=0, menu=fileMenu)
+
+ if self.rms_loc:
+ rmsavailable = 'active'
+ else:
+ rmsavailable = 'disabled'
+ if self.share_loc:
+ shareavailable = 'active'
+ else:
+ shareavailable = 'disabled'
+ if self.gmn_key:
+ gmnavailable ='active'
+ else:
+ gmnavailable ='disabled'
+
+ rawMenu = Menu(self.menuBar, tearoff=0)
+ rawMenu.add_command(label="Get Live Images", command=self.get_data)
+ rawMenu.add_command(label="Get Videos", command=self.get_vids)
+ rawMenu.add_separator()
+ rawMenu.add_command(label="Get ECSVs", command=self.getRequestedECSVs)
+ rawMenu.add_command(label="Excl/Incl ECSV", command=self.ignoreCamera)
+ rawMenu.add_command(label="Reduce Selected Image", command=self.reduceCamera, state=rmsavailable)
+ rawMenu.add_separator()
+ rawMenu.add_command(label="Share Raw Data", command=self.uploadRaw, state=shareavailable)
+ rawMenu.add_separator()
+ watchMenu = Menu(self.menuBar, tearoff=0)
+ watchMenu.add_command(label="Get GMN Raw Data", command=self.getGMNData, state=gmnavailable)
+ watchMenu.add_separator()
+ watchMenu.add_command(label="Get Watchlist", command=self.getWatchlist, state=gmnavailable)
+ watchMenu.add_command(label="View Watchlist", command=self.viewWatchlist, state=gmnavailable)
+ watchMenu.add_command(label="Upload Watchlist", command=self.putWatchlist, state=gmnavailable)
+ watchMenu.add_separator()
+ watchMenu.add_command(label="Fetch Watchlist Event", command=self.getEventData, state=gmnavailable)
+ rawMenu.add_cascade(label='GMN', menu=watchMenu, state=gmnavailable)
+
+ self.menuBar.add_cascade(label="Raw", underline=0, menu=rawMenu)
+
+ revMenu = Menu(self.menuBar, tearoff=0)
+ revMenu.add_command(label="Review Stacks", command=self.checkStacks)
+ revMenu.add_command(label="Review Images", command=self.viewData)
+ revMenu.add_command(label="Clean Folder", command=self.clean_folder)
+ self.menuBar.add_cascade(label="Review", underline=0, menu=revMenu)
+
+ solveMenu = Menu(self.menuBar, tearoff=0)
+ solveMenu.add_command(label="Solve", command=self.solveOrbit, state=solveravailable)
+ solveMenu.add_separator()
+ solveMenu.add_command(label="View Solution", command=self.viewSolution)
+ solveMenu.add_command(label="Delete Solution", command=self.removeSolution)
+ solveMenu.add_separator()
+ solveMenu.add_command(label="Upload Solution", command=self.uploadOrbit)
+ self.menuBar.add_cascade(label="Solve", underline=0, menu=solveMenu)
+
+ otherMenu = Menu(self.menuBar, tearoff=0)
+ otherMenu.add_command(label="Get Traj Pickle", command=self.get_trajpickle)
+ self.menuBar.add_cascade(label="Other", underline=0, menu=otherMenu)
+
+ # buttons
+ self.save_panel = LabelFrame(self, text=' Image Selection ')
+ self.save_panel.grid(row = 1, columnspan = 2, sticky='WE')
+
+ self.newpatt = StringVar()
+ self.newpatt.set(self.patt)
+
+ self.patt_entry = StyledEntry(self.save_panel, textvariable = self.newpatt, width = 20)
+ self.patt_entry.grid(row = 1, column = 1, columnspan = 2, sticky = "W")
+ save_bmp = StyledButton(self.save_panel, text="Get Images", width = 8, command = lambda: self.get_data())
+ save_bmp.grid(row = 1, column = 3)
+
+ save_bmp = StyledButton(self.save_panel, text="Remove", width = 8, command = lambda: self.remove_image())
+ save_bmp.grid(row = 1, column = 4)
+
+
+ # Listbox
+ self.scrollbar = Scrollbar(self)
+ self.listbox = Listbox(self, width = 47, yscrollcommand=self.scrollbar.set, exportselection=0, activestyle = "none", bg = global_bg, fg = global_fg)
+ self.listbox.config(height = 25) # Listbox size
+ self.listbox.grid(row = 4, column = 0, rowspan = 7, columnspan = 2, sticky = "NS") # Listbox position
+ self.scrollbar.grid(row = 4, column = 2, rowspan = 7, sticky = "NS") # Scrollbar size
+
+ self.listbox.bind('<>', self.update_image)
+ self.scrollbar.config(command = self.listbox.yview)
+
+ # IMAGE
+ try:
+ # Show the TV test card image on program start
+ #noimage_data = img.open('noimage.bin').resize((640,360))
+ noimgdata = img.open(noimg_file).resize((640,360))
+ noimage = ImageTk.PhotoImage(noimgdata)
+ except:
+ noimage = None
+
+ self.imagelabel = Label(self, image = noimage)
+ self.imagelabel.image = noimage
+ self.imagelabel.grid(row=3, column=3, rowspan = 4, columnspan = 3)
+
+ # Timestamp label
+ self.timestamp_label = Label(self, text = "CCNNNN YYYY-MM-DD HH:MM.SS.mms", font=("Courier", 12))
+ self.timestamp_label.grid(row = 7, column = 3, sticky = "E")
+
+ def reviewConfig(self):
+ showConfig()
+ self.readConfig()
+ self.initUI()
+
+ def reduceCamera(self):
+ current_image = self.listbox.get(ACTIVE)
+ if current_image == '':
+ return
+ camid = current_image[3:9]
+ print('selected camera is', camid)
+ dirname = os.path.join(self.dir_path, camid)
+ tmpscr = os.path.join(os.getenv('TEMP'), 'reduce.ps1')
+ with open(tmpscr, 'w') as outf:
+ outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {dirname} -c {dirname}/.config\n')
+ _ = subprocess.run(['powershell.exe', tmpscr])
+ frs = glob.glob(os.path.join(dirname, 'FR*.bin'))
+ if len(frs) > 0:
+ if tkMessageBox.askyesno("Rerun", f'{len(frs)} FR files detected - rerun?'):
+ for fr in frs:
+ with open(tmpscr, 'w') as outf:
+ outf.write(f'cd {self.rms_loc}\nconda activate {self.rms_env}\npython -m Utils.SkyFit2 {fr} -c {dirname}/.config\n')
+ _ = subprocess.run(['powershell.exe', tmpscr])
+ try:
+ os.remove(tmpscr)
+ except:
+ pass
+ os.makedirs(os.path.join(self.dir_path,'ecsvs'), exist_ok=True)
+ ecsvfs = glob.glob1(dirname, '*.ecsv')
+ for ecsv in ecsvfs:
+ shutil.copyfile(os.path.join(dirname, ecsv), os.path.join(self.dir_path, 'ecsvs', ecsv))
+
+ return
+
+ def ignoreCamera(self):
+ current_image = self.listbox.get(ACTIVE)
+ if current_image == '':
+ return
+ camid = current_image[3:9]
+ log.info(f'selected camera is {camid}')
+ dirname = os.path.join(self.dir_path, camid)
+ allecsvs = glob.glob(os.path.join(dirname, '*.ecsv'))
+ ecsvs = [e for e in allecsvs if 'REJECT' not in e.upper()]
+ rejs = [e for e in allecsvs if 'REJECT' in e.upper()]
+ if len(ecsvs) == 0 and len(rejs) == 0:
+ log.info('no files to reject or include')
+ return
+ elif len(ecsvs) > 0 and len(rejs) == 0:
+ for ecsv in ecsvs:
+ os.rename(ecsv, ecsv.replace('.ecsv','_REJECT.ecsv'))
+ jpgnames = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image))
+ for jpgname in jpgnames:
+ os.rename(jpgname, jpgname.replace('.jpg','_REJECT.jpg'))
+ elif len(ecsvs) == 0 and len(rejs) > 0:
+ for rej in rejs:
+ os.rename(rej, rej.replace('_REJECT.ecsv','.ecsv'))
+ jpgnames = glob.glob(os.path.join(self.dir_path, 'jpgs', current_image))
+ for jpgname in jpgnames:
+ os.rename(jpgname, jpgname.replace('_REJECT.jpg','.jpg'))
+ else:
+ # more than one ECSV or REJ file to handle, urk
+ log.info('urk')
+ return
+ bin_list = [line for line in os.listdir(os.path.join(self.dir_path, 'jpgs')) if self.correct_datafile_name(line)]
+ #print(bin_list)
+ for b in bin_list:
+ self.selected[b] = (0, '')
+ self.update_listbox(bin_list)
+ return
+
+ def solveOrbit(self):
+ log.info('Using ECSV files:')
+ ecsv_names = []
+ ecsv_paths = []
+ os.makedirs(os.path.join(self.dir_path,'ecsvs'), exist_ok=True)
+ for entry in sorted(os.walk(self.dir_path), key=lambda x: x[0]):
+ dir_name, _, file_names = entry
+ if 'ecsvs' in dir_name:
+ continue
+ for fn in file_names:
+ if fn.lower().endswith(".ecsv"):
+ shutil.copyfile(os.path.join(dir_name, fn), os.path.join(self.dir_path, 'ecsvs', fn))
+ if fn.lower().endswith(".ecsv") and 'REJECT' not in dir_name.upper() and 'REJECT' not in fn.upper():
+ # Add ECSV file, but skip duplicates
+ if fn not in ecsv_names:
+ ecsv_paths.append(os.path.join(dir_name, fn))
+ ecsv_names.append(fn)
+ log.info(fn)
+ if len(ecsv_paths) < 2:
+ tkMessageBox.showinfo('Warning', 'Need at least two ECSV files')
+ return
+ jdt_ref, meteor_list = loadECSVs(ecsv_paths)
+ mcruns = 20
+ max_toffset = 15.0
+ velpart = None
+ vinitht = None
+ plotallspatial = True
+ uncertgeom = False
+ jacchia = False
+ traj = solveTrajectoryGeneric(jdt_ref, meteor_list, self.dir_path,
+ max_toffset=max_toffset, monte_carlo=True, mc_runs=mcruns,
+ geometric_uncert=uncertgeom, plot_all_spatial_residuals=plotallspatial,
+ show_plots=False, v_init_part=velpart, v_init_ht=vinitht,
+ show_jacchia=jacchia, enable_OSM_plot=True, mc_cores=8)
+ tkMessageBox.showinfo('Info', 'Solver Finished')
+ if traj is not None:
+ self.soln_outputdir = traj.output_dir
+ return
+
+ def viewSolution(self):
+ self.review_stack = False
+ if not self.soln_outputdir:
+ solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*')
+ if len(solndir) == 0:
+ tkMessageBox.showinfo('Warning', 'No solution to review')
+ return
+ solndir = os.path.join(self.dir_path, solndir[0])
+ self.soln_outputdir = solndir
+ bin_list = [line for line in os.listdir(self.soln_outputdir) if self.correct_datafile_name(line)]
+ for b in bin_list:
+ self.selected[b] = (0, '')
+ self.update_listbox(bin_list)
+ return
+
+ def removeSolution(self):
+ if not self.soln_outputdir:
+ solndir = glob.glob1(self.dir_path, os.path.split(self.dir_path)[1][:8]+'*')
+ if len(solndir) == 0:
+ tkMessageBox.showinfo('Warning', 'No solution to remove')
+ return
+ solndir = os.path.join(self.dir_path, solndir[0])
+ self.soln_outputdir = solndir
+ if not tkMessageBox.askyesno("Delete file", f"delete {self.soln_outputdir}?"):
+ return
+ shutil.rmtree(os.path.join(self.dir_path, self.soln_outputdir))
+ return
+
+ def uploadOrbit(self):
+ pickles=[]
+ for path, _, files in os.walk(self.dir_path):
+ for name in files:
+ if '.pickle' in name and '_mc_' not in name and 'tmpzip' not in path:
+ log.info(f'adding pickle {name}')
+ pickles.append(os.path.join(path, name))
+ if name.lower().endswith(".ecsv") and 'ecsvs' not in path:
+ log.info(f'copying {name}')
+ shutil.copyfile(os.path.join(path, name), os.path.join(self.dir_path, 'ecsvs', name))
+
+ if len(pickles) == 0:
+ return
+ pickles = list(set(pickles))
+ if len(pickles) == 1:
+ pickfile = pickles[0]
+ else:
+ pickfile = tkFileDialog.askopenfilename(title='Select Orbit Pickle', defaultextension='*.pickle',
+ initialdir=self.dir_path, initialfile='*.pickle',
+ filetypes=[('pickles','*.pickle')])
+ if not pickfile:
+ return
+ orbname = os.path.split(pickfile)[1]
+ tmpdir = os.path.join(self.dir_path, 'tmpzip')
+ os.makedirs(tmpdir, exist_ok=True)
+ shutil.copyfile(pickfile, os.path.join(tmpdir, orbname))
+ if os.path.isdir(os.path.join(self.dir_path, 'jpgs')):
+ shutil.copytree(os.path.join(self.dir_path, 'jpgs'), os.path.join(tmpdir, 'jpgs'), dirs_exist_ok=True)
+ if os.path.isdir(os.path.join(self.dir_path, 'mp4s')):
+ shutil.copytree(os.path.join(self.dir_path, 'mp4s'), os.path.join(tmpdir, 'mp4s'), dirs_exist_ok=True)
+ if os.path.isdir(os.path.join(self.dir_path, 'ecsvs')):
+ shutil.copytree(os.path.join(self.dir_path, 'ecsvs'), os.path.join(tmpdir, 'ecsvs'), dirs_exist_ok=True)
+ for path, _, files in os.walk(self.dir_path):
+ for name in files:
+ if '_dyn_mass_fit' in name:
+ im = Image.open(os.path.join(path, name)).convert("RGB")
+ im.save(os.path.join(tmpdir,'jpgs', name[:-4] + '.jpg'))
+ zfname = os.path.join(self.dir_path, orbname[:15])
+ shutil.make_archive(zfname,'zip',tmpdir)
+ try:
+ shutil.rmtree(tmpdir)
+ except Exception:
+ pass
+
+ if self.api_key:
+ headers = {'Content-type': 'application/zip', 'Slug': orbname[:15], 'apikey': self.api_key}
+ url = f'https://api.ukmeteors.co.uk/fireballfiles?orbitfile={orbname[:15]}.zip'
+ r = requests.put(url, data=open(zfname+'.zip', 'rb'), headers=headers) #, auth=('username', 'pass'))
+ #print(r.text)
+ if r.status_code != 200:
+ tkMessageBox.showinfo('Warning', f'Problem with upload, {r.status_code}')
+ else:
+ tkMessageBox.showinfo('Info', 'Orbit Uploaded')
+ return
+ else:
+ tkMessageBox.showinfo('Info', 'Zip File created')
+
+ def uploadRaw(self):
+ zfname = os.path.join(os.getenv('TMP'), os.path.basename(self.dir_path))
+ log.info(f'zfname is {zfname}')
+ shutil.make_archive(zfname,'zip',self.dir_path)
+ try:
+ targname = os.path.join(self.share_loc, os.path.basename(zfname)+'.zip')
+ log.info(f'targname is {targname}')
+ shutil.copyfile(zfname+'.zip', targname)
+ tkMessageBox.showinfo('Info', 'Raw Data Uploaded to Dropbox')
+ subprocess.Popen(f'explorer "{self.share_loc}"')
+ except Exception:
+ tkMessageBox.showinfo('Warning', 'Problem with upload')
+ return
+
+ def viewData(self):
+ self.review_stack = False
+ self.soln_outputdir = None
+ print(self.dir_path)
+ bin_list = self.get_bin_list()
+ for b in bin_list:
+ self.selected[b] = (0, '')
+ self.update_listbox(bin_list)
+ return
+
+ def getRequestedECSVs(self):
+ notgotlist=[]
+ img_list = self.get_bin_list()
+ for current_image in img_list:
+ if not self.getOneEcsv(current_image):
+ notgotlist.append(current_image)
+ if len(notgotlist) > 0:
+ tkMessageBox.showinfo('Info', f'No ECSVs for {notgotlist}')
+ return
+
+ def getOneEcsv(self, current_image):
+ if current_image[:1] == 'M':
+ datestr = current_image[1:16]
+ statid = current_image[-11:-5]
+ elif current_image[:2] == 'FF':
+ statid = current_image[3:9]
+ datestr = current_image[10:29]
+ else:
+ return
+ #dtval = datetime.datetime.strptime(datestr, '%Y%m%d_%H%M%S')
+ #datestr = dtval.strftime('%Y-%m-%dT%H:%M:%S')
+ try:
+ lis = getECSVs(statid, datestr, savefiles=True, outdir=os.path.join(self.dir_path, statid))
+ for li in lis:
+ if 'issue getting data' in li:
+ return False
+ os.makedirs(os.path.join(self.dir_path,'ecsvs'), exist_ok=True)
+ ecsvfs = glob.glob1(os.path.join(self.dir_path, statid), '*.ecsv')
+ for ecsv in ecsvfs:
+ shutil.copyfile(os.path.join(self.dir_path, statid, ecsv), os.path.join(self.dir_path, 'ecsvs', ecsv))
+
+ shutil.copyfile()
+ ## finish here
+ return True
+ except Exception:
+ return False
+
+ def get_bin_list(self):
+ """ Get a list of image files in a given directory.
+ """
+ if self.dir_path is None:
+ dirname = tkFileDialog.askdirectory(parent=root,initialdir=self.fb_dir,
+ title='Please select a directory')
+ _, thispatt = os.path.split(dirname)
+ self.dir_path = dirname
+ self.patt = thispatt
+ self.newpatt.set(self.patt)
+
+ if self.review_stack is False:
+ targdir = os.path.join(self.dir_path, 'jpgs')
+ else:
+ targdir = os.path.join(self.dir_path, 'stacks')
+ if os.path.isdir(targdir):
+ bin_list = [line for line in os.listdir(targdir) if self.correct_datafile_name(line)]
+ else:
+ log.info('no jpgs available')
+ bin_list = []
+ return bin_list
+
+ def update_listbox(self, bin_list):
+ """ Updates the listbox with the current entries.
+ """
+ self.listbox.delete(0, END)
+ for line in sorted(bin_list):
+ self.listbox.insert(END, line)
+ if line not in self.selected:
+ self.selected[line] = (0, '')
+ if self.selected[line][0] == 1:
+ self.listbox.itemconfig(END, fg = 'green')
+
+ def checkStacks(self):
+ self.review_stack = True
+ bin_list = self.get_bin_list()
+ if len(bin_list) > 0:
+ for b in bin_list:
+ self.selected[b] = (0, '')
+ self.update_listbox(bin_list)
+ else:
+ tkMessageBox.showinfo('Warning', 'No stacks to review')
+
+ def loadFolder(self):
+ self.review_stack = False
+ self.dir_path = None
+ self.soln_outputdir = None
+ bin_list = self.get_bin_list()
+ for b in bin_list:
+ self.selected[b] = (0, '')
+ self.update_listbox(bin_list)
+
+ def openFolder(self):
+ dir_path = self.dir_path.replace("/","\\")
+ os.system(f'explorer.exe {dir_path}')
+
+ def viewLogs(self):
+ logdir = os.path.join(os.getenv('TMP'), 'fbcollector')
+ os.system(f'explorer.exe {logdir}')
+
+ def archiveFolder(self):
+ noimgdata = img.open(noimg_file).resize((640,360))
+ noimage = ImageTk.PhotoImage(noimgdata)
+ self.imagelabel.configure(image = noimage)
+ self.imagelabel.image = noimage
+ if self.dir_path == self.fb_dir.strip():
+ self.loadFolder()
+ if self.dir_path is not None and self.dir_path != self.fb_dir:
+ try:
+ _, fldr = os.path.split(os.path.normpath(self.dir_path))
+ yr = fldr[:4]
+ archdir = os.path.join(self.fb_dir, yr)
+ os.makedirs(archdir, exist_ok=True)
+ zfname = os.path.join(archdir, fldr)
+ shutil.make_archive(zfname,'zip',self.fb_dir, fldr)
+ except Exception as e:
+ log.warning(f'unable to create archive {self.dir_path}')
+ log.warning(e)
+ try:
+ os.chdir(self.fb_dir)
+ shutil.rmtree(self.dir_path)
+ except Exception as e:
+ log.warning(f'unable to remove folder, please do it manually {self.dir_path}')
+ log.warning(e)
+ self.dir_path = self.fb_dir
+
+ def delFolder(self):
+ noimgdata = img.open(noimg_file).resize((640,360))
+ noimage = ImageTk.PhotoImage(noimgdata)
+ self.imagelabel.configure(image = noimage)
+ self.imagelabel.image = noimage
+ if self.dir_path is not None and self.dir_path != self.fb_dir:
+ try:
+ shutil.rmtree(self.dir_path)
+ self.dir_path = self.fb_dir
+ except Exception as e:
+ log.warning(f'unable to remove {self.dir_path}')
+ log.warning(e)
+
+ def correct_datafile_name(self, line):
+ if ('.jpg' in line or '.png' in line) and 'noimage' not in line:
+ return True
+ return False
+
+ def removeRelated(self, imgname):
+ camid = imgname[:6]
+ mp4s = os.listdir(os.path.join(self.dir_path, 'mp4s'))
+ badones = [x for x in mp4s if camid in x]
+ for ba in badones:
+ os.remove(os.path.join(self.dir_path, 'mp4s', ba))
+ jpgs = os.listdir(os.path.join(self.dir_path, 'jpgs'))
+ badones = [x for x in jpgs if camid in x]
+ for ba in badones:
+ os.remove(os.path.join(self.dir_path, 'jpgs', ba))
+ fldrs = os.listdir(self.dir_path)
+ badones = [x for x in fldrs if camid in x]
+ for ba in badones:
+ try:
+ shutil.rmtree(os.path.join(self.dir_path, ba))
+ except Exception:
+ os.remove(os.path.join(self.dir_path, ba))
+ os.remove(os.path.join(self.dir_path, 'stacks', imgname))
+
+ def remove_image(self):
+ """ Remove the selected image from disk
+ """
+ current_image = self.listbox.get(ACTIVE)
+ if current_image == '':
+ return
+ if not tkMessageBox.askyesno("Delete file", f"delete {current_image}?"):
+ return
+ log.info(f'removing {current_image}')
+ if self.review_stack:
+ self.removeRelated(current_image)
+ else:
+ try:
+ os.remove(os.path.join(self.dir_path, 'jpgs', current_image))
+ except Exception:
+ pass
+ xmlf = current_image.replace('P.jpg', '.xml')
+ try:
+ os.remove(os.path.join(self.dir_path, 'jpgs', xmlf))
+ except Exception:
+ pass
+ self.selected[current_image] = (0,'')
+ self.update_listbox(self.get_bin_list())
+
+ def update_image(self, thing):
+ """ When selected, load a new image
+ """
+ try:
+ # Check if the list is empty. If it is, do nothing.
+ if self.review_stack:
+ self.current_image = os.path.join(self.dir_path, 'stacks', self.listbox.get(self.listbox.curselection()[0]))
+ elif self.soln_outputdir:
+ self.current_image = os.path.join(self.soln_outputdir, self.listbox.get(self.listbox.curselection()[0]))
+ else:
+ self.current_image = os.path.join(self.dir_path, 'jpgs', self.listbox.get(self.listbox.curselection()[0]))
+ except:
+ return 0
+
+ with img.open(self.current_image).resize((640,360)) as imgdata:
+ thisimage = ImageTk.PhotoImage(imgdata)
+ self.imagelabel.configure(image = thisimage)
+ self.imagelabel.image = thisimage
+
+ self.timestamp_label.configure(text = os.path.split(self.current_image)[1])
+ return
+
+ def clean_folder(self):
+ stacklist = os.listdir(os.path.join(self.dir_path, 'stacks'))
+ camlist = [x[:6] for x in stacklist if 'stack.jpg' in x]
+ datalist = os.listdir(self.dir_path)
+ datalist = [x for x in datalist if 'jpgs' not in x and 'mp4s' not in x and 'stacks' not in x]
+ for d in datalist:
+ keep = False
+ for c in camlist:
+ if c in d:
+ keep = True
+ if keep is False:
+ try:
+ os.remove(d)
+ except Exception:
+ pass
+ return
+
+ def get_data(self):
+ thispatt = self.newpatt.get().strip()
+ self.patt = thispatt.ljust(15,'0')
+ self.dir_path = os.path.join(self.fb_dir, self.newpatt.get().strip())
+ log.info(f'getting data matching {thispatt}')
+ os.makedirs(os.path.join(self.dir_path, 'jpgs'), exist_ok=True)
+ reqdate = datetime.datetime.strptime(self.patt, '%Y%m%d_%H%M%S')
+ reqdate = reqdate + datetime.timedelta(seconds=-30)
+ getLiveJpgs(reqdate.strftime('%Y%m%d_%H%M%S'), outdir=os.path.join(self.dir_path, 'jpgs'))
+ self.renameImages(self.dir_path)
+ self.update_listbox(self.get_bin_list())
+
+ def get_trajpickle(self):
+ basepatt = os.path.split(self.dir_path)[1]
+ ymd = basepatt[:8]
+ fullpatt = askstring('Trajectory Name', 'eg 20240101_010203.345_UK', initialvalue=basepatt)
+ trajpick = f'{basepatt}_trajectory.pickle'
+ url = f'https://archive.ukmeteors.co.uk/reports/{ymd[:4]}/orbits/{ymd[:6]}/{ymd}/{fullpatt}/{trajpick}'
+ log.info(f'{url}')
+ get_response = requests.get(url, stream=True)
+ if get_response.status_code == 200:
+ log.info(f'retrieved {trajpick}')
+ with open(os.path.join(self.dir_path, trajpick), 'wb') as f:
+ for chunk in get_response.iter_content(chunk_size=4096):
+ if chunk: # filter out keep-alive new chunks
+ f.write(chunk)
+ else:
+ log.info(f'unable to retrieve {trajpick}, {get_response.status_code}')
+ return
+
+ def get_vids(self):
+ jpglist = glob.glob1(os.path.join(self.dir_path,'jpgs'), 'FF*.jpg')
+ os.makedirs(os.path.join(self.dir_path, 'mp4s'), exist_ok=True)
+ count = 0
+ for jpg in jpglist:
+ mp4 = jpg.replace('.jpg','.mp4')
+ ym = mp4[10:16]
+ url = f'https://archive.ukmeteors.co.uk/img/mp4/{ym[:4]}/{ym}/{mp4}'
+ get_response = requests.get(url, stream=True)
+ if get_response.status_code == 200:
+ log.info(f'retrieved {mp4}')
+ count += 1
+ with open(os.path.join(self.dir_path, 'mp4s', mp4), 'wb') as f:
+ for chunk in get_response.iter_content(chunk_size=4096):
+ if chunk: # filter out keep-alive new chunks
+ f.write(chunk)
+ log.info(f'retrieved {count} videos')
+ tkMessageBox.showinfo("Info", f'Retrieved {count} videos')
+ return
+
+ def renameImages(self, dir_path):
+ xmllist = glob.glob(os.path.join(dir_path,'jpgs', 'M*.xml'))
+ for xmlf in xmllist:
+ jpgf = xmlf.replace('.xml','P.jpg')
+ if os.path.isfile(jpgf):
+ xmld = xmltodict.parse(open(xmlf).read())
+ realfname = xmld['ufocapture_record']['@cap'].replace('.fits','.jpg')
+ if not os.path.isfile(os.path.join(dir_path, 'jpgs', realfname)):
+ os.rename(jpgf, os.path.join(dir_path, 'jpgs', realfname))
+ else:
+ os.remove(jpgf)
+ os.remove(xmlf)
+ return
+
+ def viewWatchlist(self):
+ evtfile = os.path.join(self.fb_dir,'event_watchlist.txt')
+ if platform.system() == 'Darwin': # macOS
+ procid = subprocess.Popen(('open', evtfile))
+ elif platform.system() == 'Windows': # Windows
+ procid = subprocess.Popen(('cmd','/c',evtfile))
+ else: # linux variants
+ procid = subprocess.Popen(('xdg-open', evtfile))
+ procid.wait()
+ if not tkMessageBox.askyesno("Upload File", "Upload event watchlist?"):
+ return
+ else:
+ self.putWatchlist()
+
+ def getWatchlist(self):
+ k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key))
+ c = paramiko.SSHClient()
+ server=self.gmn_server
+ user=self.gmn_user
+ log.info(f'trying {user}@{server} with {self.gmn_key}')
+ c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ c.connect(hostname = server, username = user, pkey = k)
+ scpcli = SCPClient(c.get_transport())
+ log.info('getting Watchlist')
+ scpcli.get('./event_watchlist.txt', self.fb_dir)
+ self.viewWatchlist()
+
+ def putWatchlist(self):
+ k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key))
+ c = paramiko.SSHClient()
+ server=self.gmn_server
+ user=self.gmn_user
+ log.info(f'trying {user}@{server} with {self.gmn_key}')
+ c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ c.connect(hostname = server, username = user, pkey = k)
+ scpcli = SCPClient(c.get_transport())
+ log.info('Uploading watchlist')
+ evtfile = os.path.join(self.fb_dir,'event_watchlist.txt')
+ scpcli.put(evtfile, 'event_watchlist.txt')
+ self.evtMonTriggered = datetime.datetime.now() + datetime.timedelta(minutes=5)
+ return
+
+ def getEventData(self):
+ evtdate = self.newpatt.get().strip()
+ if len(evtdate) < 15:
+ tkMessageBox.showinfo("Warning", f'Need seconds in the event date field {evtdate}')
+ return
+ cmd = os.path.join(self.script_loc, 'download_events.sh') + f' {evtdate} 1'
+ if ':' in cmd:
+ drv = cmd[0].lower()
+ cmd = '/mnt/' + drv + cmd[2:]
+ cmd = cmd.replace('\\','/')
+ log.info(f'executing {cmd}')
+ if self.evtMonTriggered is None:
+ ret = tkMessageBox.askyesno("Warning", 'Event Monitor has not been triggered, continue?')
+ if ret is False:
+ return
+ elif datetime.datetime.now() < self.evtMonTriggered:
+ ret = tkMessageBox.askyesno("Wait", f'Should wait till {self.evtMonTriggered.strftime("%H:%M:%S")} - continue anyway?')
+ if ret is False:
+ return
+ os.chdir(self.fb_dir)
+ log.info(f'getting data for {evtdate}')
+ procid = subprocess.Popen(('bash','-c', cmd))
+ procid.wait()
+ tkMessageBox.showinfo("Info", 'Done')
+ return
+
+ def getGMNData(self):
+ print(self.dir_path)
+ camlist = [line for line in os.listdir(os.path.join(self.dir_path,'jpgs')) if self.correct_datafile_name(line)]
+ dts=[]
+ camids=[]
+ for cam in camlist:
+ camid,_ = os.path.splitext(cam)
+ spls = camid.split('_')
+ if camid[:2] == 'FF':
+ camids.append(spls[1])
+ dts.append(camid[10:25])
+ else:
+ camids.append(spls[-1][:6])
+ dts.append(camid[1:16])
+ dts.sort()
+ dtstr = f'{dts[0]}.000000'
+ stationlist = ','.join(map(str, camids))
+ server=self.gmn_server
+ user=self.gmn_user
+ log.info('getting data from GMN')
+ k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(self.gmn_key))
+ c = paramiko.SSHClient()
+ log.info(f'trying {user}@{server} with {self.gmn_key}')
+ c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ c.connect(hostname = server, username = user, pkey = k)
+ command = f'source ~/anaconda3/etc/profile.d/conda.sh && conda activate wmpl && python scripts/extract_fireball.py {dtstr} {stationlist}'
+ log.info(f'running {command}')
+
+ _, stdout, stderr = c.exec_command(command, timeout=900)
+ for line in iter(stdout.readline, ""):
+ log.info(line)
+ for line in iter(stderr.readline, ""):
+ print(line, end="")
+ scpcli = SCPClient(c.get_transport())
+ log.info('done, collecting output')
+ indir = os.path.join(f'event_extract/{dtstr}/')
+ scpcli.get(indir, self.dir_path, recursive=True)
+ command = f'rm -Rf event_extract/{dtstr}'
+ log.info(f'running {command}')
+ _, stdout, stderr = c.exec_command(command, timeout=120)
+ for line in iter(stdout.readline, ""):
+ log.info(line)
+ for line in iter(stderr.readline, ""):
+ log.info(line)
+ dirs = os.listdir(os.path.join(self.dir_path, dtstr))
+ for d in dirs:
+ srcdir = os.path.join(self.dir_path, dtstr, d)
+ targ = os.path.join(self.dir_path, d)
+ os.makedirs(targ, exist_ok=True)
+ for f in os.listdir(srcdir):
+ shutil.copy(os.path.join(srcdir, f), targ)
+ try:
+ shutil.rmtree(os.path.join(self.dir_path, dtstr))
+ except Exception:
+ pass
+ tkMessageBox.showinfo("Data Collected", 'data collected from GMN')
+ self.update_listbox(self.get_bin_list())
+ return
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-d", "--datepatt", type=str, help="date pattern to retrieve")
+ args = parser.parse_args()
+
+ dir_ = os.getcwd()
+ config_file = os.path.join(dir_, 'config.ini')
+ if not os.path.isfile(config_file):
+ shutil.copyfile(os.path.join(dir_, 'config.ini.sample'), config_file)
+ tkMessageBox.showinfo("Config Missing", 'Please configure before using')
+ showConfig()
+
+ noimg_file = os.path.join(dir_, 'noimage.jpg')
+
+ log = logging.getLogger(__name__)
+ log.setLevel(logging.INFO)
+
+ logdir = os.path.join(os.getenv('TMP'), 'fbcollector')
+ os.makedirs(logdir, exist_ok=True)
+ log_file = os.path.join(logdir, log_timestamp() + '.log')
+ handler = logging.handlers.TimedRotatingFileHandler(log_file, when='D', interval=1) # Log to a different file each day
+ handler.setLevel(logging.INFO)
+
+ formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s-%(module)s-line:%(lineno)d - %(message)s', datefmt='%Y/%m/%d %H:%M:%S')
+ handler.setFormatter(formatter)
+ log.addHandler(handler)
+
+ ch = logging.StreamHandler(sys.stdout)
+ ch.setLevel(logging.DEBUG)
+ formatter = logging.Formatter(fmt='%(asctime)s-%(levelname)s: %(message)s', datefmt='%Y/%m/%d %H:%M:%S')
+ ch.setFormatter(formatter)
+ log.addHandler(ch)
+
+ # Log program start
+ log.info("Program start")
+ log.info(f'config file is {config_file}')
+
+
+ # Initialize main window
+ root = tk.Tk()
+ root.geometry('+0+0')
+ targdir = args.datepatt
+ log.info(f'patt is {targdir}')
+
+ app = fbCollector(root, patt=targdir)
+ root.iconbitmap(os.path.join(dir_,'ukmda.ico'))
+ root.protocol('WM_DELETE_WINDOW', app.quitApplication)
+
+ root.mainloop()
diff --git a/fbCollector/noimage.jpg b/fbCollector/noimage.jpg
new file mode 100644
index 000000000..80e1f661b
Binary files /dev/null and b/fbCollector/noimage.jpg differ
diff --git a/fbCollector/requirements.txt b/fbCollector/requirements.txt
new file mode 100644
index 000000000..f8283c730
--- /dev/null
+++ b/fbCollector/requirements.txt
@@ -0,0 +1,9 @@
+# requirements for the fbCollector tool
+# Copyright (C) 2018-2023 Mark McIntyre
+Pillow
+paramiko
+scp
+pyqtgraph
+xmltodict
+requests
+pandas
\ No newline at end of file
diff --git a/fbCollector/ukmda.ico b/fbCollector/ukmda.ico
new file mode 100644
index 000000000..e788188d8
Binary files /dev/null and b/fbCollector/ukmda.ico differ
diff --git a/fbCollector/vars.yml b/fbCollector/vars.yml
new file mode 100644
index 000000000..e7e28879c
--- /dev/null
+++ b/fbCollector/vars.yml
@@ -0,0 +1,2 @@
+ srcdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/OneDrive/dev/meteorhunting/ukmda-dataprocessing/fbCollector"
+ destdir: "/mnt/c/Users/{{ lookup('env','USER' )}}/meteors/fbCollector"
diff --git a/usermgmt/README.md b/usermgmt/README.md
index 4bb751c08..d966a3530 100644
--- a/usermgmt/README.md
+++ b/usermgmt/README.md
@@ -1,6 +1,7 @@
# UKMDA User Management
-This folder contains the scripts used to add/maintain new cameras to the network.
+This Windows application is used to add/maintain new cameras to the network. Its use is restricted to members of the UKMON
+admin team.
## Basic Principles
A camera consists of an RMS ID, location and pointing direction.
@@ -13,15 +14,18 @@ Each _camera_ is allocated a unique *Unix* ID. The public ssh key provided by th
# User Management tool
-Unix and AWS User creation and configuration are managed by *stationMaint.ps1* which calls a python programme that uses native AWS and Unix libraries to execute the required commands. The tool requires AWS and SSH credentials which are restricted to administrators (see note below).
+Unix and AWS User creation and configuration are managed by *stationMaint.exe* which is a compiled python programme that uses native AWS and Unix libraries to execute the required commands.
## Setup
-Prerequisites:
-* anaconda or miniconda
-* An AWS profile with the usermaintenance IAM permissions. The default profile name is *ukmda_maint*.
-* An SSH key thats permissioned to connect to the server. The default key name is *ukmda_admin*.
-
-To install the app, copy the python files, requirements.txt and powershell script to a folder of your choosing, and update the environment variables in the powershell script as needed. Now run the powershell script which will create a suitable Conda environment, install the requirements and launch the tool.
+Reminder: To use the app you will have to be permissioned on our servers by one of the Admin team.
+
+* Download the latest zip file from [here](https://github.com/ukmda/ukmda-dataprocessing/releases) and expand it to a location of your choosing on your computer, for example `%userprofile%\ukmon\usermgmt`
+* Create a desktop shortcut pointing to `dist\stationmaint2.exe` in this folder.
+
+When you first run the app you will be prompted to provide the credentials we gave you to connect to our server.
+
+### Linux or MacOS
+In principle the code should work on Linux-like OSes. You'd need to copy the python file, icon file, sample ini file, requirements file and this README to a folder of your choosing then create a python virtual environment, activate it and install the requirements.
## Adding a new camera
To add a new camera the camera operator must supply the following:
@@ -37,28 +41,28 @@ by other cameras at the same location.
Once the information has been gathered, select Camera/Add and fill in the boxes. Note that the boxes are part-prepopulated with values from whatever row your cursor was on, so that if you're adding another camera at an existing location, you can save a bit of typing.
## Amending a camera
-The location and pointing direction cannot be changed once set. To change these values you'll need to follow the process to move a camera.
+To change location or pointing direction follow the process to move a camera.
All other values can be amended by selecting the line and updating the values.
## Moving a camera
-To move a camera to a new location, select the row containing the camera then select Camera/Move. After you fill in any new information, a new unix and AWS user will be created and the configuration files for the old user will be updated. The old camera must then be marked disabled as explained next.
+IMPORTANT NOTE: the camera owner should NOT make any changes - the changes will automatically flow down to their station.
-IMPORTANT NOTE: the camera owner should NOT make any changes - the system will automatically update their configuration.
-
-## Disabling a camera
-To disable a camera, change the Active column from 1 to today's date in YYYYMMDD format eg 20220715. This removes the camera from current reporting, but retains the details for any historical reporting.
-
-# Disabling a location
-To disable a location, we revoke the corresponding AWS user's keys, and if necessary, delete the AWS user and Unix ID. No other user will be affected. This is not currently managed by the UI as it is a drastic action that we do not want to carry out accidentally!
+To move a camera to a new location, select the row containing the camera then select Camera/Move. After you fill in any new information, a new unix and AWS user will be created and the configuration files for the old user will be updated. The old camera must then be marked disabled as explained next.
-# Permissions Needed
-This tool requires two sets of permissions:
-* an AWS profile with permissions to add/amend/delete users in IAM. This is required to manage the AWS roles.
-* an SSH key with permissions to login to the Batch server. This is required to manage the Unix accounts.
+## Disabling and Re-enabling a camera
+To disable a camera, change the Active column from 1 to the last active date in YYYYMMDD format eg 20220715. This removes the camera from current reporting and disables the unix user, but retains the details for any historical reporting. A camera can be reactivated by setting the Active column back to 1.
-# Parameters
-The powershell script lists the parameters that need to be set, such as AWS profile to use, the buckets and so on. The ansible YAML script sets these values.
+## Disabling a location
+Disable all cameras at that location, and then update the AWS key. We never remove a location as there would be a risk of losing historic data.
+## Other Functionality
+The tool also provides some other functionality:
+* Owners menu - search for station details and owner details.
+* Camera menu
+ * Check camera - this does a quick sanity test to see when the camera last connected to our server.
+ * Download the current plateper for a camera and upload a new one. The new plate will be automaticlly installed on the camera the following morning.
+ * Update the SSH key - if a user sends a new SSH key, we need to add it to the server.
+ * Update the AWS key - force an update of the location's AWS key. Note that keys are autorolled every 60 days so this would only be needed if the key had been compromised.
# Copyright
All code Copyright (C) 2018-2023 Mark McIntyre
\ No newline at end of file
diff --git a/usermgmt/server/updateAwsKey.sh b/usermgmt/server/updateAwsKey.sh
index ab53776a1..ff508560d 100644
--- a/usermgmt/server/updateAwsKey.sh
+++ b/usermgmt/server/updateAwsKey.sh
@@ -8,7 +8,14 @@ targ=$1
force=$2
here="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
-source $here/../config.ini >/dev/null 2>&1
+if [ -f ../config.ini ] ; then
+ source ~/dev/config.ini
+ keydir=/home/ec2-user/dev/keymgmt
+else
+ source ~/prod/config.ini
+ keydir=/home/ec2-user/keymgmt
+
+fi
conda activate $HOME/miniconda3/envs/${WMPL_ENV}
export AWS_PROFILE=ukmonshared
diff --git a/usermgmt/windows/release_notes.md b/usermgmt/windows/release_notes.md
index de9a86dd7..979fd1c87 100644
--- a/usermgmt/windows/release_notes.md
+++ b/usermgmt/windows/release_notes.md
@@ -3,7 +3,7 @@
This application is used to add, modify and disable RMS stations linked to UKMON. The app is intended for use only by the UKMON committee members responsible for managing users.
## Before Installation
-First you should contact me, Mark McIntyre. You will need to send me your ssh public key, and i will then create a userid for you on our server and send you the details.
+First you should contact me, Mark McIntyre so we can set up a userid for you.
## Installation
Download the zip file and expand it to a location of your choice, for example `c:\programdata\ukmon`
@@ -15,4 +15,5 @@ Save the config file and run the app.
The menus should be pretty self-explanatory.
## Release Notes
-2024.04.2 - first release.
\ No newline at end of file
+2024.04.2 - first release.
+2025.10.1 - overhaul to fix a plethora of bugs and add new features.
\ No newline at end of file
diff --git a/usermgmt/windows/stationMaint2.py b/usermgmt/windows/stationMaint2.py
index cf6a81d3e..90db60053 100644
--- a/usermgmt/windows/stationMaint2.py
+++ b/usermgmt/windows/stationMaint2.py
@@ -21,18 +21,30 @@
from configparser import ConfigParser
import logging
import logging.handlers
+import shutil
+import platform
+import subprocess
log = logging.getLogger("logger")
+config_file = ''
+
def loadConfig(cfgdir):
- cfgfile = os.path.join(cfgdir, 'stationmaint.cfg')
+ config_file = os.path.join(cfgdir, 'stationmaint.ini')
+ if not os.path.isfile(config_file):
+ tkMessageBox.showinfo("Config Missing", 'Please configure before using')
+ shutil.copyfile(f'{config_file}.sample', config_file)
+ if platform.system() == 'Darwin': # macOS
+ procid = subprocess.Popen(('open', config_file))
+ elif platform.system() == 'Windows': # Windows
+ procid = subprocess.Popen(('cmd','/c',config_file))
+ else: # linux variants
+ procid = subprocess.Popen(('xdg-open', config_file))
+ procid.wait()
cfg = ConfigParser()
- if not os.path.isfile(cfgfile):
- tkMessageBox.showinfo('Warning', f'config file {cfgfile} not found')
- exit(0)
- cfg.read(cfgfile)
+ cfg.read(config_file)
server = cfg['helper']['helperip']
user = cfg['helper']['user']
keyfile = cfg['helper']['sshkey']
@@ -297,6 +309,7 @@ def __init__(self, parent, cfgdir):
self.parent = parent
Frame.__init__(self, parent)
+ self.config_dir = cfgdir
self.cfg, awskeys = loadConfig(cfgdir)
self.conn = boto3.Session(aws_access_key_id=awskeys['key'], aws_secret_access_key=awskeys['secret'])
self.bucket_name = self.cfg['store']['srcbucket']
@@ -306,7 +319,6 @@ def __init__(self, parent, cfgdir):
os.makedirs('users', exist_ok=True)
os.makedirs('inifs', exist_ok=True)
os.makedirs('sshkeys', exist_ok=True)
- self.resyncLocalFiles()
self.ddb = self.conn.resource('dynamodb', region_name='eu-west-2')
try:
@@ -332,25 +344,27 @@ def __init__(self, parent, cfgdir):
camMenu = Menu(self.menuBar, tearoff=0)
camMenu.add_command(label = "Add Camera", command = self.addCamera)
camMenu.add_command(label = "Relocate Camera", command = self.moveCamera)
- camMenu.add_command(label = "Deactivate Camera", command = self.delCamera)
camMenu.add_separator()
- camMenu.add_command(label = "Remove Location", command = self.delOperator)
+ camMenu.add_command(label = "Check Camera", command = self.checkLastUpdate)
camMenu.add_separator()
camMenu.add_command(label = "Download Platepar", command = self.getPlate)
- camMenu.add_command(label = "Update platepar", command = self.newPlate)
+ camMenu.add_command(label = "Update Platepar", command = self.newPlate)
camMenu.add_separator()
camMenu.add_command(label = "Update SSH Key", command = self.newSSHKey)
camMenu.add_command(label = "Update AWS Key", command = self.newAWSKey)
- camMenu.add_separator()
- camMenu.add_command(label = "Check Camera", command = self.checkLastUpdate)
ownMenu = Menu(self.menuBar, tearoff=0)
ownMenu.add_command(label = "View Owner Data", command = self.viewOwnerData)
ownMenu.add_command(label = "Search Data", command = self.searchOwnerData)
+ helpMenu = Menu(self.menuBar, tearoff=0)
+ helpMenu.add_command(label = "Documentation", command = self.viewReadme)
+ helpMenu.add_command(label = "About", command = self.aboutBox)
+
self.menuBar.add_cascade(label="File", underline=0, menu=fileMenu)
self.menuBar.add_cascade(label="Camera", underline=0, menu=camMenu)
self.menuBar.add_cascade(label="Owners", underline=0, menu=ownMenu)
+ self.menuBar.add_cascade(label="Help", underline=0, menu=helpMenu)
parent.grid_columnconfigure(0, weight = 1)
parent.grid_rowconfigure(0, weight = 1)
@@ -404,6 +418,10 @@ def __init__(self, parent, cfgdir):
("end_edit_cell", self.end_edit_cell)])
self.sheet.popup_menu_add_command('Sort', self.columns_sort, table_menu = False, index_menu = False)
+ self.parent.title('Please wait, syncing data...')
+ self.resyncLocalFiles()
+ self.parent.title('Station Maintenance')
+ return
def hide_columns_right_click(self, event = None):
currently_displayed = self.sheet.display_columns()
@@ -427,6 +445,10 @@ def end_edit_cell(self, event):
'direction': data[2], 'camtype': str(data[3]), 'active': int(data[4]), 'oldcode': data[1],
'created': data[8]}
addRow(newdata, ddb=self.ddb)
+ if int(data[4]) > 1:
+ self.disableEnableUnixUser(data[0], data[2], False)
+ else:
+ self.disableEnableUnixUser(data[0], data[2], True)
return event[3]
def end_delete_rows(self, event):
@@ -489,14 +511,6 @@ def on_closing(self):
self.parent.quit()
self.parent.destroy()
- def delCamera(self):
- tk.messagebox.showinfo(title="Information", message='To remove a camera, set Active=current date yyyymmdd')
- return
-
- def delOperator(self):
- tk.messagebox.showinfo(title="Information", message='Not implemented yet')
- return
-
def viewOwnerData(self):
statOwnerDialog(self)
return
@@ -693,23 +707,23 @@ def uploadPlatepar(self, camdets, plateparfile):
log.info(f'running {command}')
_, stdout, stderr = c.exec_command(command, timeout=10)
for line in iter(stdout.readline, ""):
- log.info(line, end="")
+ log.info(line)
for line in iter(stderr.readline, ""):
- log.info(line, end="")
+ log.info(line)
command = f'sudo mv {uplfile} /var/sftp/{camname}/platepar/platepar_cmn2010.cal'
log.info(f'running {command}')
_, stdout, stderr = c.exec_command(command, timeout=10)
for line in iter(stdout.readline, ""):
- log.info(line, end="")
+ log.info(line)
for line in iter(stderr.readline, ""):
- log.info(line, end="")
+ log.info(line)
command = f'sudo chown {camname}:{camname} /var/sftp/{camname}/platepar/platepar_cmn2010.cal'
log.info(f'running {command}')
_, stdout, stderr = c.exec_command(command, timeout=10)
for line in iter(stdout.readline, ""):
- log.info(line, end="")
+ log.info(line)
for line in iter(stderr.readline, ""):
- log.info(line, end="")
+ log.info(line)
scpcli.close()
c.close()
return
@@ -762,51 +776,17 @@ def createNewAwsKey(self, location, caminfo):
c.connect(hostname = server, username = user, pkey = k)
command = f'/home/{user}/keymgmt/updateAwsKey.sh {location} force'
log.info(f'running {command}')
- _, stdout, stderr = c.exec_command(command, timeout=10)
- for line in iter(stdout.readline, ""):
- log.info(line, end="")
+ _, stdout, stderr = c.exec_command(command, timeout=60)
for line in iter(stderr.readline, ""):
- log.info(line, end="")
+ log.info(line)
+ for line in iter(stdout.readline, ""):
+ log.info(line)
+ tkMessageBox.showinfo('Updated', line.split(' to')[0])
log.info('done')
c.close()
return
- def updateKeyfile(self, caminfo, location):
- server = self.cfg['helper']['platepardir']
- user='ec2-user'
- keyf = os.path.join('jsonkeys', location + '.key')
- currkey = json.load(open(keyf, 'r'))
- archcsvf = os.path.join('csvkeys', location.lower() + '_arch.csv')
- with open(archcsvf,'w') as outf:
- outf.write('Access key ID,Secret access key\n')
- outf.write('{},{}\n'.format(currkey['AccessKey']['AccessKeyId'], currkey['AccessKey']['SecretAccessKey']))
-
- affectedcamlist = caminfo[caminfo.site==location]
- keyfile = self.cfg['helper']['sshkey']
- k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile))
- c = paramiko.SSHClient()
- c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
- c.connect(hostname = server, username = user, pkey = k)
- scpcli = SCPClient(c.get_transport())
- # push the raw keyfile
- scpcli.put(keyf, 'keymgmt/rawkeys/live/')
- scpcli.put(archcsvf, 'keymgmt/rawkeys/csvkeys/')
- scpcli.close()
- c.close()
- for _, cam in affectedcamlist.iterrows():
- cameraname = cam.site.lower() + '_' + cam.site.lower()
- keyfile = os.path.join('sshkeys', cameraname + '.pub')
- k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile))
- c = paramiko.SSHClient()
- c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
- c.connect(hostname = server, username = cameraname, pkey = k)
- scpcli = SCPClient(c.get_transport())
- scpcli.put(archcsvf, '.')
- scpcli.close()
- c.close()
-
-
def getSSHkey(self, loc, dir):
server= self.cfg['helper']['helperip']
user='ec2-user'
@@ -874,7 +854,34 @@ def addNewUnixUser(self, location, cameraname, oldcamname='', updatemode=0):
scpcli.close()
c.close()
return
-
+
+ def disableEnableUnixUser(self, loc, dir, enable):
+ cameraname = loc.lower() + '_' + dir.lower()
+ server = self.cfg['helper']['helperip']
+ user='ec2-user'
+ log.info(f'updating Unix user {cameraname}')
+ keyfile = self.cfg['helper']['sshkey']
+ k = paramiko.RSAKey.from_private_key_file(os.path.expanduser(keyfile))
+ c = paramiko.SSHClient()
+ c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ try:
+ c.connect(hostname = server, username = user, pkey = k)
+ except Exception:
+ c.connect(hostname = server+'.', username = user, pkey = k)
+ if enable is False:
+ command = f'/usr/bin/sudo /usr/sbin/usermod --expiredate 1 {cameraname}'
+ else:
+ command = f'/usr/bin/sudo /usr/sbin/usermod --expiredate "" {cameraname}'
+ log.info(f'running {command}')
+ line = ''
+ _, stdout, stderr = c.exec_command(command, timeout=60)
+ for line in iter(stderr.readline, ""):
+ log.info(line)
+ for line in iter(stdout.readline, ""):
+ log.info(line)
+ if enable is False:
+ tkMessageBox.showinfo('Disabled', f'{cameraname} {line}')
+ return
def createKeyFile(self, location):
archbucket = self.cfg['store']['srcbucket']
@@ -907,6 +914,20 @@ def createIniFile(self, cameraname):
outf.write('export UKMONKEY=~/.ssh/ukmon\n')
outf.write('export RMSCFG=~/source/RMS/.config\n')
return
+
+ def viewReadme(self):
+ docfile = os.path.join(self.config_dir, 'README.md')
+ if platform.system() == 'Darwin': # macOS
+ procid = subprocess.Popen(('open', docfile))
+ elif platform.system() == 'Windows': # Windows
+ procid = subprocess.Popen(('cmd','/c',f'notepad {docfile}'))
+ else: # linux variants
+ procid = subprocess.Popen(('xdg-open', docfile))
+ procid.wait()
+
+ def aboutBox(self):
+ tkMessageBox.showinfo('About',
+ 'UKMON user / camera maintenance tool.\nAll rights reserved, Mark McIntyre, 2024')
def log_timestamp():
@@ -917,7 +938,7 @@ def log_timestamp():
if __name__ == '__main__':
# Initialize main window
- dir_ = os.getcwd() #os.path.dirname(os.path.realpath(__file__))
+ dir_ = os.getcwd()
log.setLevel(logging.INFO)
@@ -940,6 +961,7 @@ def log_timestamp():
log.info("Program start")
root = tk.Tk()
+ root.minsize(400, 100)
app = CamMaintenance(root, dir_)
root.iconbitmap(os.path.join(dir_,'camera.ico'))
root.protocol("WM_DELETE_WINDOW", app.on_closing)
diff --git a/usermgmt/windows/stationmaint.cfg b/usermgmt/windows/stationmaint.ini.sample
similarity index 75%
rename from usermgmt/windows/stationmaint.cfg
rename to usermgmt/windows/stationmaint.ini.sample
index 210eb98de..0a6b99346 100644
--- a/usermgmt/windows/stationmaint.cfg
+++ b/usermgmt/windows/stationmaint.ini.sample
@@ -1,6 +1,6 @@
[helper]
-USER=adm_mark
-SSHKEY=~/.ssh/ukmonhelper
+USER=
+SSHKEY=
HELPERIP=3.11.55.160
REMOTEDIR=/home/ec2-user/prod/data
PLATEPARDIR=
diff --git a/utils/addVidorImg.ps1 b/utils/addVidorImg.ps1
new file mode 100644
index 000000000..7b03fa64d
--- /dev/null
+++ b/utils/addVidorImg.ps1
@@ -0,0 +1,50 @@
+# Copyright (C) 2018-2023 Mark McIntyre
+#
+# powershell script to upload an extra video or image of an event, given a trajectory ID
+
+$loc = get-location
+if ($args.count -lt 1) {
+ write-output "Usage: addVideoorImg.ps1 trajpath"
+ write-output " First create a folder for the trajectory eg 20241113_192754.345_UK"
+ write-output " Copy any additional mp4, mov or jpgs into it, then run this script"
+ exit
+}else {
+ $pth = $args[0]
+}
+set-location $PSScriptRoot
+# load the helper functions
+. .\helperfunctions.ps1
+$ini=get-inicontent analysis.ini
+$fbfolder = $ini['localdata']['fbfolder'].replace('$HOME',$home)
+$fbfolder = $fbfolder + "/fireballs"
+set-location $fbfolder
+$traj = (split-path -leaf $pth)
+
+$yr=$traj.Substring(0,4)
+$ym=$traj.Substring(0,6)
+$ymd=$traj.Substring(0,8)
+$pick=$traj.substring(0,15) + '_trajectory.pickle'
+$zipf=$traj.substring(0,15) + '.zip'
+
+wget https://archive.ukmeteors.co.uk/reports/${yr}/orbits/${ym}/${ymd}/${traj}/${pick} -O ${pth}\${pick}
+
+if ((test-path $pth\jpgs) -eq $false) { mkdir $pth\jpgs }
+if ((test-path $pth\mp4s) -eq $false) { mkdir $pth\mp4s }
+
+move-item $pth\*.jpg $pth\jpgs
+move-item $pth\*.mp4 $pth\mp4s
+
+compress-archive -path $pth\* -DestinationPath $pth\$zipf -force
+
+$apikey=(Get-Content ~/.ssh/fbuploadkey.txt)
+$headers = @{
+ 'Content-type' = 'application/zip'
+ 'Slug' = ${zipf}
+ 'apikey'= ${apikey}
+}
+$url = "https://api.ukmeteors.co.uk/fireballfiles?orbitfile=${zipf}"
+
+invoke-webrequest -uri $url -infile $pth\$zipf -Method PUT -Headers $headers
+
+Write-Output "uploaded $zipf at $(get-date)"
+set-location $loc
diff --git a/utils/analysis.ini b/utils/analysis.ini
new file mode 100644
index 000000000..3d0d32ee3
--- /dev/null
+++ b/utils/analysis.ini
@@ -0,0 +1,29 @@
+# Parameters that define your local configuration
+#
+#
+[pylib]
+pylib=$HOME/OneDrive/dev/ukmda-dataprocessing/archive/ukmon_pylib
+
+[localdata]
+FBFOLDER=$HOME/OneDrive/pictures/meteors
+MTHLYFOLDER=$HOME/OneDrive/pictures/meteors/mthlystacks
+
+[rms]
+RMS_INSTALLED=1
+RMS_LOC=$HOME/OneDrive/dev/RMS
+RMS_ENV=RMS
+
+[wmpl]
+WMPL_INSTALLED=1
+WMPL_LOC=$HOME/OneDrive/dev/WesternMeteorPyLib
+WMPL_ENV=wmpl
+
+[gmn]
+IDFILE=~/.ssh/gmnanalysis
+
+# camid to hostname mappings, yours may be different
+[stations]
+UK0006=uk0006
+UK000F=uk000f
+UK002F=uk002f
+UK001L=uk001l
diff --git a/utils/getBrightest.ps1 b/utils/getBrightest.ps1
new file mode 100644
index 000000000..ba9933b27
--- /dev/null
+++ b/utils/getBrightest.ps1
@@ -0,0 +1,41 @@
+# powershell script to get the 30 best detections from the previous night
+# for sharing on social media
+# note that you will have to go through and delete non-meteors from the images
+
+# requirements - you must clone WesternMeteorPyLib and ukmda-dataprocessing from github
+# to your local development space. I use onedrive\dev for my code - set this location in $codeloc
+
+
+# copyright (c) Mark McIntyre, 2025-
+
+set-location $PSScriptRoot
+
+# load the helper functions
+. .\helperfunctions.ps1
+$ini=get-inicontent analysis.ini
+$bdir = $ini['localdata']['fbfolder'].replace('$HOME',$home)
+$bdir = $bdir + "/brightest"
+set-location $bdir
+
+$outdir = $bdir.replace('\','/')
+
+$wmplloc = $ini['wmpl']['wmpl_loc'].replace('$HOME',$home)
+$repdir = $ini['pylib']['pylib'].replace('$HOME',$home)
+
+$env:pythonpath="$wmplloc"
+Push-Location $repdir
+
+conda activate ukmon-shared
+
+Write-Output "working..."
+if ($args.count -eq 0) {
+ python -c "from reports.findBestMp4s import getBestNSingles;getBestNSingles(numtoget=30,outdir='$outdir')"
+}else{
+ $reqdate = $args[0]
+ python -c "from reports.findBestMp4s import getBestNSingles;getBestNSingles(numtoget=30,outdir='$outdir', reqdate='$reqdate')"
+}
+
+$outdirw = $outdir.replace('/','\')
+explorer "$outdirw"
+
+Pop-Location
\ No newline at end of file
diff --git a/utils/helperfunctions.ps1 b/utils/helperfunctions.ps1
new file mode 100644
index 000000000..b914e30dc
--- /dev/null
+++ b/utils/helperfunctions.ps1
@@ -0,0 +1,125 @@
+# Copyright (C) 2018-2023 Mark McIntyre
+
+# helper functions
+
+Function Get-IniContent {
+ <#
+ .Synopsis
+ Gets the content of an INI file
+
+ .Description
+ Gets the content of an INI file and returns it as a hashtable
+
+ .Notes
+ Author : Oliver Lipkau
+ Blog : http://oliver.lipkau.net/blog/
+ Source : https://github.com/lipkau/PsIni
+ http://gallery.technet.microsoft.com/scriptcenter/ea40c1ef-c856-434b-b8fb-ebd7a76e8d91
+ Version : 1.0 - 2010/03/12 - Initial release
+ 1.1 - 2014/12/11 - Typo (Thx SLDR)
+ Typo (Thx Dave Stiff)
+
+ #Requires -Version 2.0
+
+ .Inputs
+ System.String
+
+ .Outputs
+ System.Collections.Hashtable
+
+ .Parameter FilePath
+ Specifies the path to the input file.
+
+ .Example
+ $FileContent = Get-IniContent "C:\myinifile.ini"
+ -----------
+ Description
+ Saves the content of the c:\myinifile.ini in a hashtable called $FileContent
+
+ .Example
+ $inifilepath | $FileContent = Get-IniContent
+ -----------
+ Description
+ Gets the content of the ini file passed through the pipe into a hashtable called $FileContent
+
+ .Example
+ C:\PS>$FileContent = Get-IniContent "c:\settings.ini"
+ C:\PS>$FileContent["Section"]["Key"]
+ -----------
+ Description
+ Returns the key "Key" of the section "Section" from the C:\settings.ini file
+
+ .Link
+ Out-IniFile
+ #>
+
+ [CmdletBinding()]
+ Param(
+ [ValidateNotNullOrEmpty()]
+ [ValidateScript({(Test-Path $_) -and ((Get-Item $_).Extension -eq ".ini")})]
+ [Parameter(ValueFromPipeline=$True,Mandatory=$True)]
+ [string]$FilePath
+ )
+
+ Begin
+ {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function started"}
+
+ Process
+ {
+ Write-Verbose "$($MyInvocation.MyCommand.Name):: Processing file: $Filepath"
+
+ $ini = @{}
+ switch -regex -file $FilePath
+ {
+ "^\[(.+)\]$" # Section
+ {
+ $section = $matches[1]
+ $ini[$section] = @{}
+ $CommentCount = 0
+ }
+ "^(;.*)$" # Comment
+ {
+ if (!($section))
+ {
+ $section = "No-Section"
+ $ini[$section] = @{}
+ }
+ $value = $matches[1]
+ $CommentCount = $CommentCount + 1
+ $name = "Comment" + $CommentCount
+ $ini[$section][$name] = $value
+ }
+ "(.+?)\s*=\s*(.*)" # Key
+ {
+ if (!($section))
+ {
+ $section = "No-Section"
+ $ini[$section] = @{}
+ }
+ $name,$value = $matches[1..2]
+ $ini[$section][$name] = $value
+ }
+ }
+ Write-Verbose "$($MyInvocation.MyCommand.Name):: Finished Processing file: $FilePath"
+ Return $ini
+ }
+
+ End
+ {Write-Verbose "$($MyInvocation.MyCommand.Name):: Function ended"}
+}
+
+Function Test-CommandExists
+{
+ Param ($command)
+ $oldPreference = $ErrorActionPreference
+ $ErrorActionPreference = ‘stop’
+ try {if(Get-Command $command){RETURN $true}}
+ Catch {Write-Host “$command does not exist”; RETURN $false}
+ Finally {$ErrorActionPreference=$oldPreference}
+} #end function test-CommandExists
+
+Function New-TemporaryFolder {
+ # Make a new folder based upon a TempFileName
+ $T="$($Env:temp)\tmp$([convert]::tostring((get-random 65535),16).padleft(4,'0')).tmp"
+ New-Item -ItemType Directory -Path $T
+}
\ No newline at end of file
diff --git a/utils/uploadOrbit.ps1 b/utils/uploadOrbit.ps1
new file mode 100644
index 000000000..a512a129b
--- /dev/null
+++ b/utils/uploadOrbit.ps1
@@ -0,0 +1,70 @@
+# Copyright (C) 2018-2023 Mark McIntyre
+#
+# powershell script to create a zip file of a solution and upload it
+
+$loc = get-location
+if ($args.count -lt 1) {
+ $curdir = get-location
+}else {
+ $curdir = resolve-path $args[0]
+}
+set-location $PSScriptRoot
+# load the helper functions
+. .\helperfunctions.ps1
+$ini=get-inicontent analysis.ini
+Set-Location $Loc
+
+Add-Type -AssemblyName System.Windows.Forms
+
+conda activate $ini['wmpl']['wmpl_env']
+$wmplloc = $ini['wmpl']['wmpl_loc'].replace('$HOME',$home)
+$env:pythonpath="$wmplloc;$env:pythonpath"
+
+write-output "Find the pickle"
+$picklefile= (Get-ChildItem $curdir\*.pickle -r).fullname
+if ($picklefile -is [Array] -or $picklefile.length -eq 0) {
+ $FileBrowser = New-Object System.Windows.Forms.OpenFileDialog -Property @{ InitialDirectory = $curdir; Filter = 'pickles (*.pickle)|*.pickle'; Title='Select orbit pickle' }
+ $null = $FileBrowser.ShowDialog()
+ $picklefile = $filebrowser.filename
+ if ( $picklefile -eq "" ) {
+ Write-Output "Cancelled"
+ set-location $loc
+ exit
+ }
+}
+write-output "find the orbit full name"
+$srcdir = ((get-item $picklefile).directoryname).replace('\','/')
+$pickname = (get-item $picklefile).name
+$origdir = (python -c "from wmpl.Utils.Pickling import loadPickle;pick = loadPickle('${srcdir}','${pickname}');print(pick.output_dir)")
+$orbname = split-path $origdir -leaf
+if ((test-path $curdir\$orbname) -eq 0 )
+{
+ mkdir -force $curdir\$orbname
+ move-item $picklefile $curdir\$orbname\
+}
+$picklefile = (Get-ChildItem $picklefile -r).fullname
+
+write-output "look for jpgs and mp4s"
+mkdir -force $curdir\jpgs
+$jpgdir = resolve-path $curdir\jpgs
+Move-Item $curdir\*.jpg $jpgdir
+mkdir -force $curdir\mp4s
+$mp4dir = resolve-path $curdir\mp4s
+Move-Item $curdir\*.mp4 $mp4dir
+
+write-output "Creating zip file"
+if ((test-path $env:temp\$orbname\mp4s) -eq 0 ) {mkdir $env:temp\$orbname\mp4s | out-null}
+if ((test-path $env:temp\$orbname\jpgs) -eq 0 ) {mkdir $env:temp\$orbname\jpgs | out-null}
+copy-item $picklefile $env:temp\$orbname
+copy-item $jpgdir\*.jpg $env:temp\$orbname\jpgs
+copy-item $mp4dir\*.mp4 $env:temp\$orbname\mp4s
+if ((test-path $env:temp\$orbname.zip) -eq 1 ) { remove-item $env:temp\$orbname.zip}
+compress-archive -destinationpath "$env:temp\$orbname.zip" -path "$env:temp\$orbname\*"
+remove-item $env:temp\$orbname\* -Recurse
+
+write-output "Uploading $orbname.zip"
+curl -X PUT -H "Content-Type:application/zip" --data-binary "@$env:temp\$orbname.zip" "https://api.ukmeteors.co.uk/fireballfiles?orbitfile=$orbname.zip"
+move-item $env:temp\$orbname.zip $curdir -Force
+Write-Output "uploaded at $(get-date)"
+set-location $loc
+conda deactivate